From noreply at buildbot.pypy.org Sat Nov 1 01:02:38 2014 From: noreply at buildbot.pypy.org (arigo) Date: Sat, 1 Nov 2014 01:02:38 +0100 (CET) Subject: [pypy-commit] pypy.org extradoc: update the values Message-ID: <20141101000238.917F41C355E@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: extradoc Changeset: r550:06bbe80dd579 Date: 2014-11-01 01:02 +0100 http://bitbucket.org/pypy/pypy.org/changeset/06bbe80dd579/ Log: update the values diff --git a/don1.html b/don1.html --- a/don1.html +++ b/don1.html @@ -15,7 +15,7 @@ - $58058 of $105000 (55.3%) + $58082 of $105000 (55.3%)
diff --git a/don3.html b/don3.html --- a/don3.html +++ b/don3.html @@ -9,13 +9,13 @@ - $50362 of $60000 (83.9%) + $50904 of $60000 (84.8%)
diff --git a/don4.html b/don4.html --- a/don4.html +++ b/don4.html @@ -9,7 +9,7 @@ @@ -17,7 +17,7 @@ 2nd call: - $19629 of $80000 (24.5%) + $19674 of $80000 (24.6%)
From noreply at buildbot.pypy.org Sat Nov 1 17:42:29 2014 From: noreply at buildbot.pypy.org (arigo) Date: Sat, 1 Nov 2014 17:42:29 +0100 (CET) Subject: [pypy-commit] stmgc hashtable: A random test, and a more precise test and fix following from it. Message-ID: <20141101164229.484691C100F@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: hashtable Changeset: r1489:bb24720e379b Date: 2014-11-01 17:41 +0100 http://bitbucket.org/pypy/stmgc/changeset/bb24720e379b/ Log: A random test, and a more precise test and fix following from it. diff --git a/c7/stm/hashtable.c b/c7/stm/hashtable.c --- a/c7/stm/hashtable.c +++ b/c7/stm/hashtable.c @@ -221,7 +221,9 @@ item in the current table. */ if (rc > 6) { - if (_is_from_same_transaction(hashtableobj)) { + /* we can only enter here once! If we allocate stuff, we may + run the GC, and so 'hashtableobj' might move afterwards. */ + if (_is_young(hashtableobj)) { entry = (stm_hashtable_entry_t *) stm_allocate(sizeof(stm_hashtable_entry_t)); entry->userdata = stm_hashtable_entry_userdata; @@ -296,11 +298,14 @@ } void stm_hashtable_write(object_t *hashtableobj, stm_hashtable_t *hashtable, - uintptr_t index, object_t *nvalue) + uintptr_t index, object_t *nvalue, + stm_thread_local_t *tl) { + STM_PUSH_ROOT(*tl, nvalue); stm_hashtable_entry_t *e = _stm_hashtable_lookup(hashtableobj, hashtable, index); stm_write((object_t *)e); + STM_POP_ROOT(*tl, nvalue); e->object = nvalue; } diff --git a/c7/stmgc.h b/c7/stmgc.h --- a/c7/stmgc.h +++ b/c7/stmgc.h @@ -538,7 +538,7 @@ void stm_hashtable_free(stm_hashtable_t *); object_t *stm_hashtable_read(object_t *, stm_hashtable_t *, uintptr_t key); void stm_hashtable_write(object_t *, stm_hashtable_t *, uintptr_t key, - object_t *nvalue); + object_t *nvalue, stm_thread_local_t *); extern uint32_t stm_hashtable_entry_userdata; void stm_hashtable_tracefn(stm_hashtable_t *, void (object_t **)); diff --git a/c7/test/support.py b/c7/test/support.py --- a/c7/test/support.py +++ b/c7/test/support.py @@ -172,7 +172,7 @@ bool _check_hashtable_read(object_t *, stm_hashtable_t *, uintptr_t key); object_t *hashtable_read_result; bool _check_hashtable_write(object_t *, stm_hashtable_t *, uintptr_t key, - object_t *nvalue); + object_t *nvalue, stm_thread_local_t *tl); uint32_t stm_hashtable_entry_userdata; void stm_hashtable_tracefn(stm_hashtable_t *, void (object_t **)); @@ -261,9 +261,9 @@ } bool _check_hashtable_write(object_t *hobj, stm_hashtable_t *h, uintptr_t key, - object_t *nvalue) + object_t *nvalue, stm_thread_local_t *tl) { - CHECKED(stm_hashtable_write(hobj, h, key, nvalue)); + CHECKED(stm_hashtable_write(hobj, h, key, nvalue, tl)); } #undef CHECKED diff --git a/c7/test/test_hashtable.py b/c7/test/test_hashtable.py --- a/c7/test/test_hashtable.py +++ b/c7/test/test_hashtable.py @@ -10,9 +10,9 @@ raise Conflict return lib.hashtable_read_result -def htset(o, key, nvalue): +def htset(o, key, nvalue, tl): h = get_hashtable(o) - res = lib._check_hashtable_write(o, h, key, nvalue) + res = lib._check_hashtable_write(o, h, key, nvalue, tl) if res: raise Conflict @@ -59,9 +59,10 @@ def test_set_value(self): self.start_transaction() + tl0 = self.tls[self.current_thread] h = self.allocate_hashtable() lp1 = stm_allocate(16) - htset(h, 12345678901, lp1) + htset(h, 12345678901, lp1, tl0) assert htget(h, 12345678901) == lp1 for i in range(64): index = 12345678901 ^ (1 << i) @@ -73,16 +74,17 @@ lp2 = stm_allocate_old(16) # self.start_transaction() + tl0 = self.tls[self.current_thread] h = self.allocate_hashtable() self.push_root(h) stm_set_char(lp1, 'A') - htset(h, 1234, lp1) + htset(h, 1234, lp1, tl0) self.commit_transaction() # self.start_transaction() h = self.pop_root() stm_set_char(lp2, 'B') - htset(h, 9991234, lp2) + htset(h, 9991234, lp2, tl0) # self.switch(1) self.start_transaction() @@ -96,7 +98,7 @@ assert htget(h, 9991234) == lp2 assert stm_get_char(lp2) == 'B' assert htget(h, 1234) == lp1 - htset(h, 1234, ffi.NULL) + htset(h, 1234, ffi.NULL, tl0) self.commit_transaction() # self.start_transaction() @@ -114,11 +116,13 @@ self.start_transaction() h = self.pop_root() self.push_root(h) - htset(h, 1234, lp1) + tl0 = self.tls[self.current_thread] + htset(h, 1234, lp1, tl0) # self.switch(1) self.start_transaction() - py.test.raises(Conflict, "htset(h, 1234, lp2)") + tl1 = self.tls[self.current_thread] + py.test.raises(Conflict, "htset(h, 1234, lp2, tl1)") # self.switch(0) self.pop_root() @@ -131,7 +135,8 @@ self.push_root(h) lp1 = stm_allocate(16) stm_set_char(lp1, 'N') - htset(h, 1234, lp1) + tl0 = self.tls[self.current_thread] + htset(h, 1234, lp1, tl0) stm_minor_collect() h = self.pop_root() lp1b = htget(h, 1234) @@ -146,7 +151,8 @@ h = self.allocate_hashtable() self.push_root(h) stm_set_char(lp1, 'N') - htset(h, 1234, lp1) + tl0 = self.tls[self.current_thread] + htset(h, 1234, lp1, tl0) self.commit_transaction() # self.start_transaction() @@ -158,3 +164,126 @@ # stm_major_collect() # to get rid of the hashtable object self.commit_transaction() + + def test_minor_collect_bug1(self): + self.start_transaction() + lp1 = stm_allocate(32) + self.push_root(lp1) + h = self.allocate_hashtable() + self.push_root(h) + stm_minor_collect() + h = self.pop_root() + lp1 = self.pop_root() + print 'h', h # 0xa040010 + print 'lp1', lp1 # 0xa040040 + tl0 = self.tls[self.current_thread] + htset(h, 1, lp1, tl0) + self.commit_transaction() + # + self.start_transaction() + assert htget(h, 1) == lp1 + stm_major_collect() # to get rid of the hashtable object + + def test_random_single_thread(self): + import random + values = [] + mirror = {} + roots = [] + def push_roots(): + assert roots == [] + for k, hitems in mirror.items(): + assert lib._get_type_id(k) == 421419 + for key, value in hitems.items(): + assert lib._get_type_id(value) < 1000 + self.push_root(value) + roots.append(key) + self.push_root(k) + roots.append(None) + for v in values: + self.push_root(v) + mirror.clear() + # + def pop_roots(): + assert mirror == {} + for i in reversed(range(len(values))): + values[i] = self.pop_root() + assert stm_get_char(values[i]) == chr((i + 1) & 255) + for r in reversed(roots): + obj = self.pop_root() + if r is None: + assert lib._get_type_id(obj) == 421419 + mirror[obj] = curhitems = {} + else: + assert lib._get_type_id(obj) < 1000 + curhitems[r] = obj + del roots[:] + # + for i in range(100): + print "start_transaction" + self.start_transaction() + pop_roots() + for j in range(10): + r = random.random() + if r < 0.05: + h = self.allocate_hashtable() + print "allocate_hashtable ->", h + mirror[h] = {} + elif r < 0.10: + print "stm_minor_collect" + push_roots() + stm_minor_collect() + pop_roots() + elif r < 0.11: + print "stm_major_collect" + push_roots() + stm_major_collect() + pop_roots() + elif r < 0.5: + if not mirror: continue + h = random.choice(mirror.keys()) + if not mirror[h]: continue + key = random.choice(mirror[h].keys()) + value = mirror[h][key] + print "htget(%r, %r) == %r" % (h, key, value) + push_roots() + self.push_root(value) + result = htget(h, key) + value = self.pop_root() + assert result == value + pop_roots() + elif r < 0.6: + if not mirror: continue + h = random.choice(mirror.keys()) + key = random.randrange(0, 40) + if key in mirror[h]: continue + print "htget(%r, %r) == NULL" % (h, key) + push_roots() + assert htget(h, key) == ffi.NULL + pop_roots() + elif r < 0.63: + if not mirror: continue + h, _ = mirror.popitem() + print "popped", h + elif r < 0.75: + obj = stm_allocate(32) + values.append(obj) + stm_set_char(obj, chr(len(values) & 255)) + else: + if not mirror or not values: continue + h = random.choice(mirror.keys()) + key = random.randrange(0, 32) + value = random.choice(values) + print "htset(%r, %r, %r)" % (h, key, value) + push_roots() + tl = self.tls[self.current_thread] + htset(h, key, value, tl) + pop_roots() + mirror[h][key] = value + push_roots() + print "commit_transaction" + self.commit_transaction() + # + self.start_transaction() + self.become_inevitable() + pop_roots() + stm_major_collect() # to get rid of the hashtable objects From noreply at buildbot.pypy.org Sat Nov 1 21:48:29 2014 From: noreply at buildbot.pypy.org (mattip) Date: Sat, 1 Nov 2014 21:48:29 +0100 (CET) Subject: [pypy-commit] pypy nditer-external_loop: whoops, partial commit Message-ID: <20141101204829.93E7F1C026A@cobra.cs.uni-duesseldorf.de> Author: mattip Branch: nditer-external_loop Changeset: r74319:6a505b52a29d Date: 2014-10-31 01:14 +0200 http://bitbucket.org/pypy/pypy/changeset/6a505b52a29d/ Log: whoops, partial commit diff --git a/pypy/module/micronumpy/nditer.py b/pypy/module/micronumpy/nditer.py --- a/pypy/module/micronumpy/nditer.py +++ b/pypy/module/micronumpy/nditer.py @@ -180,7 +180,7 @@ # and in nditer's with 'external_loop' flag can_coalesce = True if it.order == 'F': - indxs = slice(1,None) + fastest = 0 else: fastest = -1 for idim in range(it.ndim - 1): @@ -246,7 +246,7 @@ new_backstrides = backstrides[:-1] _shape = shape[-1] * old_iter.slice_shape # use the operand's iterator's rightmost stride, - # even if it is larger than minimum (for 'F' or swapped axis) + # even if it is not the fastest (for 'F' or swapped axis) _stride = old_iter.slice_stride _backstride = (_shape - 1) * _stride new_iter = SliceIter(old_iter.array, old_iter.size / shape[-1], From noreply at buildbot.pypy.org Sat Nov 1 21:48:38 2014 From: noreply at buildbot.pypy.org (mattip) Date: Sat, 1 Nov 2014 21:48:38 +0100 (CET) Subject: [pypy-commit] pypy nditer-external_loop: merge default into branch Message-ID: <20141101204838.876DD1C026A@cobra.cs.uni-duesseldorf.de> Author: mattip Branch: nditer-external_loop Changeset: r74320:d9fcdce54806 Date: 2014-10-31 01:14 +0200 http://bitbucket.org/pypy/pypy/changeset/d9fcdce54806/ Log: merge default into branch diff too long, truncating to 2000 out of 39286 lines diff --git a/.hgignore b/.hgignore --- a/.hgignore +++ b/.hgignore @@ -89,6 +89,7 @@ ^pypy/doc/image/lattice3\.png$ ^pypy/doc/image/stackless_informal\.png$ ^pypy/doc/image/parsing_example.+\.png$ +^rpython/doc/_build/.*$ ^pypy/module/test_lib_pypy/ctypes_tests/_ctypes_test\.o$ ^compiled ^.git/ diff --git a/README.rst b/README.rst --- a/README.rst +++ b/README.rst @@ -23,6 +23,7 @@ the pypy-dev team + Building ======== diff --git a/ctypes_configure/doc/configure.txt b/ctypes_configure/doc/configure.txt --- a/ctypes_configure/doc/configure.txt +++ b/ctypes_configure/doc/configure.txt @@ -19,6 +19,4 @@ usage ===== -`sample.py`_ explains in details how to use it. - -.. _`sample.py`: http://codespeak.net/svn/pypy/dist/ctypes_configure/doc/sample.py +:source:`sample.py ` explains in details how to use it. diff --git a/pypy/config/pypyoption.py b/pypy/config/pypyoption.py --- a/pypy/config/pypyoption.py +++ b/pypy/config/pypyoption.py @@ -255,10 +255,6 @@ BoolOption("optimized_list_getitem", "special case the 'list[integer]' expressions", default=False), - BoolOption("builtinshortcut", - "a shortcut for operations between built-in types. XXX: " - "deprecated, not really a shortcut any more.", - default=False), BoolOption("getattributeshortcut", "track types that override __getattribute__", default=False, @@ -270,9 +266,6 @@ # weakrefs needed, because of get_subclasses() requires=[("translation.rweakref", True)]), - ChoiceOption("multimethods", "the multimethod implementation to use", - ["doubledispatch", "mrd"], - default="mrd"), BoolOption("withidentitydict", "track types that override __hash__, __eq__ or __cmp__ and use a special dict strategy for those which do not", default=False, diff --git a/pypy/config/test/test_pypyoption.py b/pypy/config/test/test_pypyoption.py --- a/pypy/config/test/test_pypyoption.py +++ b/pypy/config/test/test_pypyoption.py @@ -64,7 +64,7 @@ def check_file_exists(fn): assert configdocdir.join(fn).check() - from pypy.doc.config.confrest import all_optiondescrs + from pypy.doc.config.generate import all_optiondescrs configdocdir = thisdir.dirpath().dirpath().join("doc", "config") for descr in all_optiondescrs: prefix = descr._name diff --git a/pypy/doc/TODO b/pypy/doc/TODO new file mode 100644 --- /dev/null +++ b/pypy/doc/TODO @@ -0,0 +1,40 @@ +Documentation TODO +================== + +General +------- + +* architecture documents don't really show the separation between PyPy and + RPython + * architecture.rst is duplicate (both pypy and rpython) +* Consider moving information from pypy/doc/{build,windows}.rst to rpython/doc + + +Cleanup +~~~~~~~ + +* remove documentation on removed features + * various object spaces +* update / remove dead links + + +Meta +~~~~ + +* work on configuration/options documentation generation + + +PyPy +---- + +* Update coding guide +* Move links from project-documentation.rst to index.rst and consider killing + it. + + +RPython +------- + +* make translation.rst a high-level overview and move details in their own + documents +* redo various outdated pictures in translation.rst diff --git a/pypy/doc/__pypy__-module.rst b/pypy/doc/__pypy__-module.rst --- a/pypy/doc/__pypy__-module.rst +++ b/pypy/doc/__pypy__-module.rst @@ -1,20 +1,17 @@ - .. comment: this document is very incomplete, should we generate it automatically? -======================= The ``__pypy__`` module ======================= The ``__pypy__`` module is the main entry point to special features provided -by PyPy's standard interpreter. Its content depends on `configuration options`_ -which may add new functionality and functions whose existence or non-existence -indicates the presence of such features. +by PyPy's standard interpreter. Its content depends on :doc:`configuration options ` +which may add new functionality and functions whose existence or non-existence +indicates the presence of such features. -.. _`configuration options`: config/index.html Generally available functionality -================================= +--------------------------------- - ``internal_repr(obj)``: return the interpreter-level representation of an object. @@ -22,28 +19,22 @@ It works like a simplified array of characters (actually, depending on the configuration the ``array`` module internally uses this). + Transparent Proxy Functionality -=============================== +------------------------------- -If `transparent proxies`_ are enabled (with :config:`objspace.std.withtproxy`) +If :ref:`transparent proxies ` are enabled (with :config:`objspace.std.withtproxy`) the following functions are put into ``__pypy__``: - ``tproxy(typ, controller)``: Return something that looks like it is of type typ. Its behaviour is completely controlled by the controller. See the docs - about `transparent proxies`_ for detail. - + about :ref:`transparent proxies ` for detail. - ``get_tproxy_controller(obj)``: If obj is really a transparent proxy, return its controller. Otherwise return None. -.. _`transparent proxies`: objspace-proxies.html#tproxy - Functionality available on py.py (not after translation) -======================================================== +-------------------------------------------------------- - ``isfake(obj)``: returns True if ``obj`` is faked. - - ``interp_pdb()``: start a pdb at interpreter-level. - - - diff --git a/pypy/doc/_ref.txt b/pypy/doc/_ref.txt deleted file mode 100644 --- a/pypy/doc/_ref.txt +++ /dev/null @@ -1,114 +0,0 @@ -.. This file is generated automatically by makeref.py script, - which in turn is run manually. - -.. _`ctypes_configure/doc/sample.py`: https://bitbucket.org/pypy/pypy/src/default/ctypes_configure/doc/sample.py -.. _`dotviewer/`: https://bitbucket.org/pypy/pypy/src/default/dotviewer/ -.. _`lib-python/`: https://bitbucket.org/pypy/pypy/src/default/lib-python/ -.. _`lib-python/2.7/dis.py`: https://bitbucket.org/pypy/pypy/src/default/lib-python/2.7/dis.py -.. _`lib_pypy/`: https://bitbucket.org/pypy/pypy/src/default/lib_pypy/ -.. _`lib_pypy/greenlet.py`: https://bitbucket.org/pypy/pypy/src/default/lib_pypy/greenlet.py -.. _`lib_pypy/tputil.py`: https://bitbucket.org/pypy/pypy/src/default/lib_pypy/tputil.py -.. _`pypy/bin/`: https://bitbucket.org/pypy/pypy/src/default/pypy/bin/ -.. _`pypy/bin/pyinteractive.py`: https://bitbucket.org/pypy/pypy/src/default/pypy/bin/pyinteractive.py -.. _`pypy/config/`: https://bitbucket.org/pypy/pypy/src/default/pypy/config/ -.. _`pypy/config/pypyoption.py`: https://bitbucket.org/pypy/pypy/src/default/pypy/config/pypyoption.py -.. _`pypy/doc/`: https://bitbucket.org/pypy/pypy/src/default/pypy/doc/ -.. _`pypy/doc/config/`: https://bitbucket.org/pypy/pypy/src/default/pypy/doc/config/ -.. _`pypy/doc/discussion/`: https://bitbucket.org/pypy/pypy/src/default/pypy/doc/discussion/ -.. _`pypy/goal/`: https://bitbucket.org/pypy/pypy/src/default/pypy/goal/ -.. _`pypy/interpreter`: -.. _`pypy/interpreter/`: https://bitbucket.org/pypy/pypy/src/default/pypy/interpreter/ -.. _`pypy/interpreter/argument.py`: https://bitbucket.org/pypy/pypy/src/default/pypy/interpreter/argument.py -.. _`pypy/interpreter/astcompiler`: -.. _`pypy/interpreter/astcompiler/`: https://bitbucket.org/pypy/pypy/src/default/pypy/interpreter/astcompiler/ -.. _`pypy/interpreter/astcompiler/assemble.py`: https://bitbucket.org/pypy/pypy/src/default/pypy/interpreter/astcompiler/assemble.py -.. _`pypy/interpreter/astcompiler/ast.py`: https://bitbucket.org/pypy/pypy/src/default/pypy/interpreter/astcompiler/ast.py -.. _`pypy/interpreter/astcompiler/astbuilder.py`: https://bitbucket.org/pypy/pypy/src/default/pypy/interpreter/astcompiler/astbuilder.py -.. _`pypy/interpreter/astcompiler/asthelpers.py`: https://bitbucket.org/pypy/pypy/src/default/pypy/interpreter/astcompiler/asthelpers.py -.. _`pypy/interpreter/astcompiler/codegen.py`: https://bitbucket.org/pypy/pypy/src/default/pypy/interpreter/astcompiler/codegen.py -.. _`pypy/interpreter/astcompiler/optimize.py`: https://bitbucket.org/pypy/pypy/src/default/pypy/interpreter/astcompiler/optimize.py -.. _`pypy/interpreter/astcompiler/symtable.py`: https://bitbucket.org/pypy/pypy/src/default/pypy/interpreter/astcompiler/symtable.py -.. _`pypy/interpreter/astcompiler/tools/Python.asdl`: https://bitbucket.org/pypy/pypy/src/default/pypy/interpreter/astcompiler/tools/Python.asdl -.. _`pypy/interpreter/astcompiler/tools/asdl_py.py`: https://bitbucket.org/pypy/pypy/src/default/pypy/interpreter/astcompiler/tools/asdl_py.py -.. _`pypy/interpreter/baseobjspace.py`: https://bitbucket.org/pypy/pypy/src/default/pypy/interpreter/baseobjspace.py -.. _`pypy/interpreter/eval.py`: https://bitbucket.org/pypy/pypy/src/default/pypy/interpreter/eval.py -.. _`pypy/interpreter/executioncontext.py`: https://bitbucket.org/pypy/pypy/src/default/pypy/interpreter/executioncontext.py -.. _`pypy/interpreter/function.py`: https://bitbucket.org/pypy/pypy/src/default/pypy/interpreter/function.py -.. _`pypy/interpreter/gateway.py`: https://bitbucket.org/pypy/pypy/src/default/pypy/interpreter/gateway.py -.. _`pypy/interpreter/mixedmodule.py`: https://bitbucket.org/pypy/pypy/src/default/pypy/interpreter/mixedmodule.py -.. _`pypy/interpreter/module.py`: https://bitbucket.org/pypy/pypy/src/default/pypy/interpreter/module.py -.. _`pypy/interpreter/pyframe.py`: https://bitbucket.org/pypy/pypy/src/default/pypy/interpreter/pyframe.py -.. _`pypy/interpreter/pyopcode.py`: https://bitbucket.org/pypy/pypy/src/default/pypy/interpreter/pyopcode.py -.. _`pypy/interpreter/pyparser`: -.. _`pypy/interpreter/pyparser/`: https://bitbucket.org/pypy/pypy/src/default/pypy/interpreter/pyparser/ -.. _`pypy/interpreter/pyparser/future.py`: https://bitbucket.org/pypy/pypy/src/default/pypy/interpreter/pyparser/future.py -.. _`pypy/interpreter/pyparser/metaparser.py`: https://bitbucket.org/pypy/pypy/src/default/pypy/interpreter/pyparser/metaparser.py -.. _`pypy/interpreter/pyparser/parser.py`: https://bitbucket.org/pypy/pypy/src/default/pypy/interpreter/pyparser/parser.py -.. _`pypy/interpreter/pyparser/pyparse.py`: https://bitbucket.org/pypy/pypy/src/default/pypy/interpreter/pyparser/pyparse.py -.. _`pypy/interpreter/pyparser/pytokenizer.py`: https://bitbucket.org/pypy/pypy/src/default/pypy/interpreter/pyparser/pytokenizer.py -.. _`pypy/interpreter/typedef.py`: https://bitbucket.org/pypy/pypy/src/default/pypy/interpreter/typedef.py -.. _`pypy/module`: -.. _`pypy/module/`: https://bitbucket.org/pypy/pypy/src/default/pypy/module/ -.. _`pypy/module/__builtin__/__init__.py`: https://bitbucket.org/pypy/pypy/src/default/pypy/module/__builtin__/__init__.py -.. _`pypy/module/cppyy/capi/__init__.py`: https://bitbucket.org/pypy/pypy/src/default/pypy/module/cppyy/capi/__init__.py -.. _`pypy/module/cppyy/capi/builtin_capi.py`: https://bitbucket.org/pypy/pypy/src/default/pypy/module/cppyy/capi/builtin_capi.py -.. _`pypy/module/cppyy/include/capi.h`: https://bitbucket.org/pypy/pypy/src/default/pypy/module/cppyy/include/capi.h -.. _`pypy/module/test_lib_pypy/`: https://bitbucket.org/pypy/pypy/src/default/pypy/module/test_lib_pypy/ -.. _`pypy/objspace/`: https://bitbucket.org/pypy/pypy/src/default/pypy/objspace/ -.. _`pypy/objspace/std`: -.. _`pypy/objspace/std/`: https://bitbucket.org/pypy/pypy/src/default/pypy/objspace/std/ -.. _`pypy/objspace/std/bytesobject.py`: https://bitbucket.org/pypy/pypy/src/default/pypy/objspace/std/bytesobject.py -.. _`pypy/objspace/std/multimethod.py`: https://bitbucket.org/pypy/pypy/src/default/pypy/objspace/std/multimethod.py -.. _`pypy/objspace/std/objspace.py`: https://bitbucket.org/pypy/pypy/src/default/pypy/objspace/std/objspace.py -.. _`pypy/objspace/std/proxy_helpers.py`: https://bitbucket.org/pypy/pypy/src/default/pypy/objspace/std/proxy_helpers.py -.. _`pypy/objspace/std/proxyobject.py`: https://bitbucket.org/pypy/pypy/src/default/pypy/objspace/std/proxyobject.py -.. _`pypy/objspace/std/strbufobject.py`: https://bitbucket.org/pypy/pypy/src/default/pypy/objspace/std/strbufobject.py -.. _`pypy/objspace/std/transparent.py`: https://bitbucket.org/pypy/pypy/src/default/pypy/objspace/std/transparent.py -.. _`pypy/tool/`: https://bitbucket.org/pypy/pypy/src/default/pypy/tool/ -.. _`pypy/tool/pytest/`: https://bitbucket.org/pypy/pypy/src/default/pypy/tool/pytest/ -.. _`rpython/annotator`: -.. _`rpython/annotator/`: https://bitbucket.org/pypy/pypy/src/default/rpython/annotator/ -.. _`rpython/annotator/annrpython.py`: https://bitbucket.org/pypy/pypy/src/default/rpython/annotator/annrpython.py -.. _`rpython/annotator/binaryop.py`: https://bitbucket.org/pypy/pypy/src/default/rpython/annotator/binaryop.py -.. _`rpython/annotator/builtin.py`: https://bitbucket.org/pypy/pypy/src/default/rpython/annotator/builtin.py -.. _`rpython/bin/translatorshell.py`: https://bitbucket.org/pypy/pypy/src/default/rpython/bin/translatorshell.py -.. _`rpython/config/`: https://bitbucket.org/pypy/pypy/src/default/rpython/config/ -.. _`rpython/config/translationoption.py`: https://bitbucket.org/pypy/pypy/src/default/rpython/config/translationoption.py -.. _`rpython/flowspace/`: https://bitbucket.org/pypy/pypy/src/default/rpython/flowspace/ -.. _`rpython/flowspace/model.py`: https://bitbucket.org/pypy/pypy/src/default/rpython/flowspace/model.py -.. _`rpython/memory/`: https://bitbucket.org/pypy/pypy/src/default/rpython/memory/ -.. _`rpython/memory/gc/generation.py`: https://bitbucket.org/pypy/pypy/src/default/rpython/memory/gc/generation.py -.. _`rpython/memory/gc/hybrid.py`: https://bitbucket.org/pypy/pypy/src/default/rpython/memory/gc/hybrid.py -.. _`rpython/memory/gc/minimarkpage.py`: https://bitbucket.org/pypy/pypy/src/default/rpython/memory/gc/minimarkpage.py -.. _`rpython/memory/gc/semispace.py`: https://bitbucket.org/pypy/pypy/src/default/rpython/memory/gc/semispace.py -.. _`rpython/rlib`: -.. _`rpython/rlib/`: https://bitbucket.org/pypy/pypy/src/default/rpython/rlib/ -.. _`rpython/rlib/listsort.py`: https://bitbucket.org/pypy/pypy/src/default/rpython/rlib/listsort.py -.. _`rpython/rlib/nonconst.py`: https://bitbucket.org/pypy/pypy/src/default/rpython/rlib/nonconst.py -.. _`rpython/rlib/objectmodel.py`: https://bitbucket.org/pypy/pypy/src/default/rpython/rlib/objectmodel.py -.. _`rpython/rlib/parsing/`: https://bitbucket.org/pypy/pypy/src/default/rpython/rlib/parsing/ -.. _`rpython/rlib/parsing/tree.py`: https://bitbucket.org/pypy/pypy/src/default/rpython/rlib/parsing/tree.py -.. _`rpython/rlib/rarithmetic.py`: https://bitbucket.org/pypy/pypy/src/default/rpython/rlib/rarithmetic.py -.. _`rpython/rlib/rbigint.py`: https://bitbucket.org/pypy/pypy/src/default/rpython/rlib/rbigint.py -.. _`rpython/rlib/rrandom.py`: https://bitbucket.org/pypy/pypy/src/default/rpython/rlib/rrandom.py -.. _`rpython/rlib/rsocket.py`: https://bitbucket.org/pypy/pypy/src/default/rpython/rlib/rsocket.py -.. _`rpython/rlib/streamio.py`: https://bitbucket.org/pypy/pypy/src/default/rpython/rlib/streamio.py -.. _`rpython/rlib/test`: https://bitbucket.org/pypy/pypy/src/default/rpython/rlib/test/ -.. _`rpython/rlib/unroll.py`: https://bitbucket.org/pypy/pypy/src/default/rpython/rlib/unroll.py -.. _`rpython/rtyper`: -.. _`rpython/rtyper/`: https://bitbucket.org/pypy/pypy/src/default/rpython/rtyper/ -.. _`rpython/rtyper/lltypesystem/`: https://bitbucket.org/pypy/pypy/src/default/rpython/rtyper/lltypesystem/ -.. _`rpython/rtyper/lltypesystem/lltype.py`: https://bitbucket.org/pypy/pypy/src/default/rpython/rtyper/lltypesystem/lltype.py -.. _`rpython/rtyper/rint.py`: https://bitbucket.org/pypy/pypy/src/default/rpython/rtyper/rint.py -.. _`rpython/rtyper/rlist.py`: https://bitbucket.org/pypy/pypy/src/default/rpython/rtyper/rlist.py -.. _`rpython/rtyper/rmodel.py`: https://bitbucket.org/pypy/pypy/src/default/rpython/rtyper/rmodel.py -.. _`rpython/rtyper/rtyper.py`: https://bitbucket.org/pypy/pypy/src/default/rpython/rtyper/rtyper.py -.. _`rpython/rtyper/test/test_llinterp.py`: https://bitbucket.org/pypy/pypy/src/default/rpython/rtyper/test/test_llinterp.py -.. _`rpython/tool/algo/`: https://bitbucket.org/pypy/pypy/src/default/rpython/tool/algo/ -.. _`rpython/translator`: -.. _`rpython/translator/`: https://bitbucket.org/pypy/pypy/src/default/rpython/translator/ -.. _`rpython/translator/backendopt/`: https://bitbucket.org/pypy/pypy/src/default/rpython/translator/backendopt/ -.. _`rpython/translator/c/`: https://bitbucket.org/pypy/pypy/src/default/rpython/translator/c/ -.. _`rpython/translator/c/src/stacklet/`: https://bitbucket.org/pypy/pypy/src/default/rpython/translator/c/src/stacklet/ -.. _`rpython/translator/c/src/stacklet/stacklet.h`: https://bitbucket.org/pypy/pypy/src/default/rpython/translator/c/src/stacklet/stacklet.h -.. _`rpython/translator/tool/`: https://bitbucket.org/pypy/pypy/src/default/rpython/translator/tool/ diff --git a/pypy/doc/architecture.rst b/pypy/doc/architecture.rst --- a/pypy/doc/architecture.rst +++ b/pypy/doc/architecture.rst @@ -1,93 +1,29 @@ -================================================== -Goals and Architecture Overview -================================================== +Goals and Architecture Overview +=============================== .. contents:: +This document gives an overview of the goals and architecture of PyPy. If you're +interested in :ref:`using PyPy ` or :ref:`hacking on it `, +have a look at our :ref:`getting started ` section. -This document gives an overview of the goals and architecture of PyPy. -See `getting started`_ for a practical introduction and starting points. -Mission statement -==================== +Mission statement +----------------- -We aim to provide: +We aim to provide a compliant, flexible and fast implementation of the Python_ +Language which uses the RPython toolchain to enable new advanced high-level +features without having to encode the low-level details. We call this PyPy. - * a common translation and support framework for producing - implementations of dynamic languages, emphasizing a clean - separation between language specification and implementation - aspects. We call this the `RPython toolchain`_. +.. _Python: http://docs.python.org/reference/ - * a compliant, flexible and fast implementation of the Python_ Language - which uses the above toolchain to enable new advanced high-level features - without having to encode the low-level details. We call this PyPy. - -By separating concerns in this way, our implementation -of Python - and other dynamic languages - is able to automatically -generate a Just-in-Time compiler for any dynamic language. It also -allows a mix-and-match approach to implementation decisions, including -many that have historically been outside of a user's control, such as -target platform, memory and -threading models, garbage collection strategies, and optimizations applied, -including whether or not to have a JIT in the first place. High Level Goals -============================= - -RPython - the Translation Toolchain ------------------------------------------------ - -Traditionally, language interpreters are written in a target platform language -such as C/Posix, Java or C#. Each implementation provides -a fundamental mapping between application source code and the target -environment. One of -the goals of the "all-encompassing" environments, such as the .NET framework -and to some extent the Java virtual machine, is to provide standardized -and higher level functionalities in order to support language implementers -for writing language implementations. - -PyPy is experimenting with a more ambitious approach. We are using a -subset of the high-level language Python, called RPython_, in which we -write languages as simple interpreters with few references to and -dependencies on lower level details. The `RPython toolchain`_ -produces a concrete virtual machine for the platform of our choice by -inserting appropriate lower level aspects. The result can be customized -by selecting other feature and platform configurations. - -Our goal is to provide a possible solution to the problem of language -implementers: having to write ``l * o * p`` interpreters for ``l`` -dynamic languages and ``p`` platforms with ``o`` crucial design -decisions. PyPy aims at making it possible to change each of these -variables independently such that: - -* ``l``: the language that we analyze can be evolved or entirely replaced; - -* ``o``: we can tweak and optimize the translation process to produce - platform specific code based on different models and trade-offs; - -* ``p``: we can write new translator back-ends to target different - physical and virtual platforms. - -By contrast, a standardized target environment - say .NET - -enforces ``p=1`` as far as it's concerned. This helps making ``o`` a -bit smaller by providing a higher-level base to build upon. Still, -we believe that enforcing the use of one common environment -is not necessary. PyPy's goal is to give weight to this claim - at least -as far as language implementation is concerned - showing an approach -to the ``l * o * p`` problem that does not rely on standardization. - -The most ambitious part of this goal is to `generate Just-In-Time -Compilers`_ in a language-independent way, instead of only translating -the source interpreter into an interpreter for the target platform. -This is an area of language implementation that is commonly considered -very challenging because of the involved complexity. - - -PyPy - the Python Interpreter --------------------------------------------- +---------------- Our main motivation for developing the translation framework is to -provide a full featured, customizable, fast_ and `very compliant`_ Python +provide a full featured, customizable, :ref:`fast ` and +:doc:`very compliant ` Python implementation, working on and interacting with a large variety of platforms and allowing the quick introduction of new advanced language features. @@ -106,88 +42,22 @@ and fully orthogonal to the interpreter source code. -PyPy Architecture -=========================== - -As you would expect from a project implemented using ideas from the world -of `Extreme Programming`_, the architecture of PyPy has evolved over time -and continues to evolve. Nevertheless, the high level architecture is -stable. As described above, there are two rather independent basic -subsystems: the `PyPy Python Interpreter`_ and the `RPython Translation Toolchain`_. - -.. _`translation framework`: - -RPython Translation Toolchain -------------------------- - -The job of the RPython toolchain is to translate RPython_ programs -into an efficient version of that program for one of the various target -platforms, generally one that is considerably lower-level than Python. - -The approach we have taken is to reduce the level of abstraction of the -source RPython program in several steps, from the high level down to the -level of the target platform, whatever that may be. Currently we -support two broad flavours of target platforms: the ones that assume a -C-like memory model with structures and pointers, and the ones that -assume an object-oriented model with classes, instances and methods (as, -for example, the Java and .NET virtual machines do). - -The RPython toolchain never sees the RPython source code or syntax -trees, but rather starts with the *code objects* that define the -behaviour of the function objects one gives it as input. It can be -considered as "freezing" a pre-imported RPython program into an -executable form suitable for the target platform. - -The steps of the translation process can be summarized as follows: - -* The code object of each source functions is converted to a `control - flow graph` by the `Flow Object Space`_. - -* The control flow graphs are processed by the Annotator_, which - performs whole-program type inference to annotate each variable of - the control flow graph with the types it may take at run-time. - -* The information provided by the annotator is used by the RTyper_ to - convert the high level operations of the control flow graphs into - operations closer to the abstraction level of the target platform. - -* Optionally, `various transformations`_ can then be applied which, for - example, perform optimizations such as inlining, add capabilities - such as stackless-style concurrency, or insert code for the - `garbage collector`_. - -* Then, the graphs are converted to source code for the target platform - and compiled into an executable. - -This process is described in much more detail in the `document about -the RPython toolchain`_ and in the paper `Compiling dynamic language -implementations`_. - -.. _`control flow graph`: translation.html#the-flow-model -.. _`Flow Object Space`: objspace.html#the-flow-object-space -.. _Annotator: translation.html#the-annotation-pass -.. _RTyper: rtyper.html#overview -.. _`various transformations`: translation.html#the-optional-transformations -.. _`document about the RPython toolchain`: translation.html -.. _`garbage collector`: garbage_collection.html -.. _`RPython toolchain`: translation.html -.. _`standard interpreter`: -.. _`python interpreter`: +.. _python-interpreter: PyPy Python Interpreter -------------------------------------- +----------------------- PyPy's *Python Interpreter* is written in RPython and implements the full Python language. This interpreter very closely emulates the behavior of CPython. It contains the following key components: -- a bytecode compiler responsible for producing Python code objects +- a bytecode compiler responsible for producing Python code objects from the source code of a user application; -- a `bytecode evaluator`_ responsible for interpreting +- a :doc:`bytecode evaluator ` responsible for interpreting Python code objects; -- a `standard object space`_, responsible for creating and manipulating +- a :ref:`standard object space `, responsible for creating and manipulating the Python objects seen by the application. The *bytecode compiler* is the preprocessing phase that produces a @@ -200,64 +70,6 @@ lists, as well as the operations between them, like addition or truth-value-testing. -This division between bytecode evaluator and object space is very -important, as it gives a lot of flexibility. One can plug in -different `object spaces`_ to get different or enriched behaviours -of the Python objects. Additionally, a special more abstract object -space, the `flow object space`_, allows us to reuse the bytecode -evaluator for our translation framework. - -.. _`bytecode evaluator`: interpreter.html -.. _`standard object space`: objspace.html#the-standard-object-space -.. _`object spaces`: objspace.html -.. _`flow object space`: objspace.html#the-flow-object-space - -.. _`the translation framework`: - - -Further reading -=============== - -All of PyPy's documentation can be reached from the `documentation -index`_. Of particular interest after reading this document might be: - - * `getting-started`_: a hands-on guide to getting involved with the - PyPy source code. - - * `PyPy's approach to virtual machine construction`_: a paper - presented to the Dynamic Languages Symposium attached to OOPSLA - 2006. - - * `The translation document`_: a detailed description of our - translation process. - - * `JIT Generation in PyPy`_, describing how we produce a Just-in-time - Compiler from an interpreter. - - * A tutorial of how to use the `RPython toolchain`_ to `implement your own - interpreter`_. - -.. _`documentation index`: index.html#project-documentation -.. _`getting-started`: getting-started-dev.html -.. _`PyPy's approach to virtual machine construction`: https://bitbucket.org/pypy/extradoc/raw/tip/talk/dls2006/pypy-vm-construction.pdf -.. _`the translation document`: translation.html -.. _`RPython toolchain`: translation.html -.. _`Compiling dynamic language implementations`: https://bitbucket.org/pypy/extradoc/raw/tip/eu-report/D05.1_Publish_on_translating_a_very-high-level_description.pdf -.. _`Technical reports`: index-report.html - -.. _`getting started`: getting-started.html -.. _`Extreme Programming`: http://www.extremeprogramming.org/ - -.. _fast: faq.html#how-fast-is-pypy -.. _`very compliant`: cpython_differences.html - -.. _`RPython`: coding-guide.html#rpython - -.. _Python: http://docs.python.org/reference/ -.. _Psyco: http://psyco.sourceforge.net -.. _`generate Just-In-Time Compilers`: jit/index.html -.. _`JIT Generation in PyPy`: jit/index.html -.. _`implement your own interpreter`: http://morepypy.blogspot.com/2011/04/tutorial-writing-interpreter-with-pypy.html - -.. include:: _ref.txt - +This division between bytecode evaluator and object space gives a lot of +flexibility. One can plug in different :doc:`object spaces ` to get +different or enriched behaviours of the Python objects. diff --git a/pypy/doc/arm.rst b/pypy/doc/arm.rst deleted file mode 100644 --- a/pypy/doc/arm.rst +++ /dev/null @@ -1,166 +0,0 @@ -========================= -Cross-translating for ARM -========================= - - -Here we describe the setup required and the steps needed to follow to translate -an interpreter using the RPython translator to target ARM using a cross -compilation toolchain. - -To translate an RPython program for ARM we can either -translate directly on an ARM device following the normal translation steps. Unfortunately this is not really feasible on most ARM machines. The alternative is to cross-translate using a cross-compilation toolchain. - -To cross-translate we run the translation on a more powerful (usually -x86) machine and generate a binary for ARM using a cross-compiler to compile -the generated C code. There are several constraints when doing this. In -particular we currently only support Linux as translation host and target -platforms (tested on Ubuntu). Also we need a 32-bit environment to run the -translation. This can be done either on a 32bit host or in 32bit chroot. - - -Requirements ------------- - -The tools required to cross translate from a Linux based host to an ARM based Linux target are: - -- A checkout of PyPy (default branch). -- The GCC ARM cross compiler (on Ubuntu it is the ``gcc-arm-linux-gnueabi package``) but other toolchains should also work. -- Scratchbox 2, a cross-compilation engine (``scratchbox2`` Ubuntu package). -- A 32-bit PyPy or Python. -- And the following (or corresponding) packages need to be installed to create an ARM based chroot: - - * ``debootstrap`` - * ``schroot`` - * ``binfmt-support`` - * ``qemu-system`` - * ``qemu-user-static`` - -- The dependencies above are in addition to the ones needed for a regular - translation, `listed here`_. - -.. _`listed here`: getting-started-python.html#translating-the-pypy-python-interpreter - - -Creating a Qemu based ARM chroot --------------------------------- - -First we will need to create a rootfs containing the packages and dependencies -required in order to translate PyPy or other interpreters. We are going to -assume, that the files will be placed in ``/srv/chroot/precise_arm``. - -Create the rootfs by calling: - -:: - - mkdir -p /srv/chroot/precise_arm - qemu-debootstrap --variant=buildd --arch=armel precise /srv/chroot/precise_arm/ http://ports.ubuntu.com/ubuntu-ports/ - -Next, copy the qemu-arm-static binary to the rootfs. - -:: - - cp /usr/bin/qemu-arm-static /srv/chroot/precise_arm/usr/bin/qemu-arm-static - -For easier configuration and management we will create a schroot pointing to -the rootfs. We need to add a configuration block (like the one below) to the -schroot configuration file in /etc/schroot/schroot.conf. - - -:: - - [precise_arm] - directory=/srv/chroot/precise_arm - users=USERNAME - root-users=USERNAME - groups=users - aliases=default - type=directory - - -To verify that everything is working in the chroot, running ``schroot -c -precise_arm`` should start a shell running in the schroot environment using -qemu-arm to execute the ARM binaries. Running ``uname -m`` in the chroot should -yeild a result like ``armv7l``. Showing that we are emulating an ARM system. - -Start the schroot as the user root in order to configure the apt sources and -to install the following packages: - - -:: - - schroot -c precise_arm -u root - echo "deb http://ports.ubuntu.com/ubuntu-ports/ precise main universe restricted" > /etc/apt/sources.list - apt-get update - apt-get install libffi-dev libgc-dev python-dev build-essential libncurses5-dev libbz2-dev - - -Now all dependencies should be in place and we can exit the schroot environment. - - -Configuring scratchbox2 ------------------------ - -To configure the scratchbox we need to cd into the root directory of the rootfs -we created before. From there we can call the sb2 configuration tools which -will take the current directory as the base directory for the scratchbox2 -environment. - -:: - - cd /srv/chroot/precise_arm - sb2-init -c `which qemu-arm` ARM `which arm-linux-gnueabi-gcc` - -This will create a scratchbox2 based environment called ARM that maps calls to -gcc done within the scratchbox to the arm-linux-gnueabi-gcc outside the -scratchbox. Now we should have a working cross compilation toolchain in place -and can start cross-translating programs for ARM. - -Translation ------------ - -Having performed all the preliminary steps we should now be able to cross -translate a program for ARM. You can use this_ minimal -target to test your setup before applying it to a larger project. - -Before starting the translator we need to set two environment variables, so the -translator knows how to use the scratchbox environment. We need to set the -**SB2** environment variable to point to the rootfs and the **SB2OPT** should -contain the command line options for the sb2 command. If our rootfs is in the -folder /srv/chroot/precise_arm and the scratchbox environment is called "ARM", -the variables would be defined as follows. - - -:: - - export SB2=/srv/chroot/precise_arm - export SB2OPT='-t ARM' - -Once this is set, you can call the translator. For example save this file - -:: - - def main(args): - print "Hello World" - return 0 - - def target(*args): - return main, None - -and call the translator - -:: - - pypy ~/path_to_pypy_checkout/rpython/bin/rpython -O1 --platform=arm target.py - -If everything worked correctly this should yield an ARM binary. Running this binary in the ARM chroot or on an ARM device should produce the output ``"Hello World"``. - -To translate the full python pypy interpreter with a jit, you can cd into pypy/goal and call - -:: - - pypy /rpython/bin/rpython -Ojit --platform=arm --gcrootfinder=shadowstack --jit-backend=arm targetpypystandalone.py - -The gcrootfinder option is needed to work around `issue 1377`_ and the jit-backend works around `issue 1376`_ - -.. _`issue 1377`: https://bugs.pypy.org/issue1377 -.. _`issue 1376`: https://bugs.pypy.org/issue1376 diff --git a/pypy/doc/build.rst b/pypy/doc/build.rst new file mode 100644 --- /dev/null +++ b/pypy/doc/build.rst @@ -0,0 +1,170 @@ +Building PyPy from Source +========================= + +For building PyPy, we recommend installing a pre-built PyPy first (see +:doc:`install`). It is possible to build PyPy with CPython, but it will take a +lot longer to run -- depending on your architecture, between two and three +times as long. + +Even when using PyPy to build PyPy, translation is time-consuming -- 30 +minutes on a fast machine -- and RAM-hungry. You will need **at least** 2 GB +of memory on a 32-bit machine and 4GB on a 64-bit machine. + + +Clone the repository +-------------------- + +If you prefer to compile your own PyPy, or if you want to modify it, you +will need to obtain a copy of the sources. This can be done either by +`downloading them from the download page`_ or by checking them out from the +repository using mercurial. We suggest using mercurial if you want to access +the current development. + +.. _downloading them from the download page: http://pypy.org/download.html + +You must issue the following command on your +command line, DOS box, or terminal:: + + hg clone http://bitbucket.org/pypy/pypy pypy + +This will clone the repository and place it into a directory +named ``pypy``, and will get you the PyPy source in ``pypy/pypy`` and +documentation files in ``pypy/pypy/doc``. +We try to ensure that the tip is always stable, but it might +occasionally be broken. You may want to check out `our nightly tests`_: +find a revision (12-chars alphanumeric string, e.g. "963e808156b3") +that passed at least the +``{linux32}`` tests (corresponding to a ``+`` sign on the +line ``success``) and then, in your cloned repository, switch to this revision +using:: + + hg up -r XXXXX + +where XXXXX is the revision id. + +.. _our nightly tests: http://buildbot.pypy.org/summary?branch= + + +Install build-time dependencies +------------------------------- + +To build PyPy on Unix using the C translation backend, you need at least a C +compiler and ``make`` installed. Further, some optional modules have additional +dependencies: + +cffi, ctypes + libffi, pkg-config + +zlib + libz + +bz2 + libbz2 + +lzma (PyPy3 only) + liblzma + +sqlite3 + libsqlite3 + +curses + libncurses + cffi dependencies from above + +pyexpat + libexpat1 + +_ssl + libssl + +Make sure to have these libraries (with development headers) installed before +building PyPy, otherwise the resulting binary will not contain these modules. + +On Debian, this is the command to install all build-time dependencies:: + + apt-get install gcc make libffi-dev pkg-config libz-dev libbz2-dev \ + libsqlite3-dev libncurses-dev libexpat1-dev libssl-dev + +For the optional lzma module on PyPy3 you will also need ``liblzma-dev``. + +On Fedora:: + + yum install gcc make libffi-devel pkgconfig zlib-devel bzip2-devel \ + lib-sqlite3-devel ncurses-devel expat-devel openssl-devel + +For the optional lzma module on PyPy3 you will also need ``xz-devel``. + +On SLES11:: + + zypper install gcc make python-devel pkg-config \ + zlib-devel libopenssl-devel libbz2-devel sqlite3-devel \ + libexpat-devel libffi-devel python-curses + +For the optional lzma module on PyPy3 you will also need ``xz-devel``. + +On Mac OS X, most of these build-time dependencies are installed alongside +the Developer Tools. However, note that in order for the installation to +find them you may need to run:: + + xcode-select --install + + +Run the translation +------------------- + +Translate with JIT:: + + pypy rpython/bin/rpython --opt=jit pypy/goal/targetpypystandalone.py + +Translate without JIT:: + + pypy rpython/bin/rpython --opt=2 pypy/goal/targetpypystandalone.py + +(You can use ``python`` instead of ``pypy`` here, which will take longer +but works too.) + +If everything works correctly this will create an executable ``pypy-c`` in the +current directory. The executable behaves mostly like a normal Python +interpreter (see :doc:`cpython_differences`). + + +.. _translate-pypy: + +Translating with non-standard options +------------------------------------- + +It is possible to have non-standard features enabled for translation, +but they are not really tested any more. Look, for example, at the +:doc:`objspace proxies ` document. + + + +Installation +------------ + +PyPy dynamically finds the location of its libraries depending on the location +of the executable. The directory hierarchy of a typical PyPy installation +looks like this:: + + ./bin/pypy + ./include/ + ./lib_pypy/ + ./lib-python/2.7 + ./site-packages/ + +The hierarchy shown above is relative to a PREFIX directory. PREFIX is +computed by starting from the directory where the executable resides, and +"walking up" the filesystem until we find a directory containing ``lib_pypy`` +and ``lib-python/2.7``. + +To install PyPy system wide on unix-like systems, it is recommended to put the +whole hierarchy alone (e.g. in ``/opt/pypy``) and put a symlink to the +``pypy`` executable into ``/usr/bin`` or ``/usr/local/bin``. + +If the executable fails to find suitable libraries, it will report ``debug: +WARNING: library path not found, using compiled-in sys.path`` and then attempt +to continue normally. If the default path is usable, most code will be fine. +However, the ``sys.prefix`` will be unset and some existing libraries assume +that this is never the case. + + +.. TODO windows diff --git a/pypy/doc/cleanup.rst b/pypy/doc/cleanup.rst deleted file mode 100644 --- a/pypy/doc/cleanup.rst +++ /dev/null @@ -1,11 +0,0 @@ -Old documentation that needs review ------------------------------------ - -.. The following stuff is old (and crufty?), and needs further investigation: - -.. doc-index: This needs merging somehow - -.. toctree:: - - distribution.rst - diff --git a/pypy/doc/coding-guide.rst b/pypy/doc/coding-guide.rst --- a/pypy/doc/coding-guide.rst +++ b/pypy/doc/coding-guide.rst @@ -1,21 +1,17 @@ -==================================== Coding Guide -==================================== +============ .. contents:: This document describes coding requirements and conventions for working with the PyPy code base. Please read it carefully and ask back any questions you might have. The document does not talk -very much about coding style issues. We mostly follow `PEP 8`_ though. +very much about coding style issues. We mostly follow :pep:`8` though. If in doubt, follow the style that is already present in the code base. -.. _`PEP 8`: http://www.python.org/dev/peps/pep-0008/ - -.. _`RPython`: Overview and motivation -======================== +------------------------ We are writing a Python interpreter in Python, using Python's well known ability to step behind the algorithmic problems as a language. At first glance, @@ -25,7 +21,7 @@ CPython vs. PyPy -------------------- +~~~~~~~~~~~~~~~~ Compared to the CPython implementation, Python takes the role of the C Code. We rewrite the CPython interpreter in Python itself. We could @@ -44,20 +40,20 @@ but let's stick with this somewhat canonical approach. -.. _`application-level`: -.. _`interpreter-level`: +.. _application-level: +.. _interpreter-level: Application-level and interpreter-level execution and objects -------------------------------------------------------------- +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Since Python is used for implementing all of our code base, there is a -crucial distinction to be aware of: that between *interpreter-level* objects and +crucial distinction to be aware of: that between *interpreter-level* objects and *application-level* objects. The latter are the ones that you deal with when you write normal python programs. Interpreter-level code, however, cannot invoke operations nor access attributes from application-level objects. You will immediately recognize any interpreter level code in PyPy, because half the variable and object names start with a ``w_``, which -indicates that they are `wrapped`_ application-level values. +indicates that they are `wrapped`_ application-level values. Let's show the difference with a simple example. To sum the contents of two variables ``a`` and ``b``, one would write the simple application-level @@ -80,10 +76,10 @@ interpreting. -.. _`app-preferable`: +.. _app-preferable: -Application level is often preferable -------------------------------------- +Application level is often preferable +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Application-level code is substantially higher-level, and therefore correspondingly easier to write and debug. For example, suppose we want @@ -113,11 +109,11 @@ space.setitem(w_self, w_key, w_value) This interpreter-level implementation looks much more similar to the C -source code. It is still more readable than its C counterpart because -it doesn't contain memory management details and can use Python's native -exception mechanism. +source code. It is still more readable than its C counterpart because +it doesn't contain memory management details and can use Python's native +exception mechanism. -In any case, it should be obvious that the application-level implementation +In any case, it should be obvious that the application-level implementation is definitely more readable, more elegant and more maintainable than the interpreter-level one (and indeed, dict.update is really implemented at applevel in PyPy). @@ -129,10 +125,11 @@ level code is usually preferable. We have an abstraction (called the 'Gateway') which allows the caller of a function to remain ignorant of whether a particular function is implemented at application or -interpreter level. +interpreter level. + Our runtime interpreter is "RPython" ----------------------------------------------- +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ In order to make a C code generator feasible all code on interpreter level has to restrict itself to a subset of the Python language, and we adhere to some @@ -159,7 +156,7 @@ elegant: For the definition of all the opcodes of the Python interpreter, the module ``dis`` is imported and used to initialize our bytecode interpreter. (See ``__initclass__`` in -`pypy/interpreter/pyopcode.py`_). This +:source:`pypy/interpreter/pyopcode.py`). This saves us from adding extra modules to PyPy. The import code is run at startup time, and we are allowed to use the CPython builtin import function. @@ -172,308 +169,13 @@ enables the code generator to emit efficient machine level replacements for pure integer objects, for instance. -RPython -================= - -RPython Definition ------------------- - -RPython is a restricted subset of Python that is amenable to static analysis. -Although there are additions to the language and some things might surprisingly -work, this is a rough list of restrictions that should be considered. Note -that there are tons of special cased restrictions that you'll encounter -as you go. The exact definition is "RPython is everything that our translation -toolchain can accept" :) - -.. _`wrapped object`: coding-guide.html#wrapping-rules - -Flow restrictions -------------------------- - -**variables** - - variables should contain values of at most one type as described in - `Object restrictions`_ at each control flow point, that means for - example that joining control paths using the same variable to - contain both a string and a int must be avoided. It is allowed to - mix None (basically with the role of a null pointer) with many other - types: `wrapped objects`, class instances, lists, dicts, strings, etc. - but *not* with int, floats or tuples. - -**constants** - - all module globals are considered constants. Their binding must not - be changed at run-time. Moreover, global (i.e. prebuilt) lists and - dictionaries are supposed to be immutable: modifying e.g. a global - list will give inconsistent results. However, global instances don't - have this restriction, so if you need mutable global state, store it - in the attributes of some prebuilt singleton instance. - - - -**control structures** - - all allowed, ``for`` loops restricted to builtin types, generators - very restricted. - -**range** - - ``range`` and ``xrange`` are identical. ``range`` does not necessarily create an array, - only if the result is modified. It is allowed everywhere and completely - implemented. The only visible difference to CPython is the inaccessibility - of the ``xrange`` fields start, stop and step. - -**definitions** - - run-time definition of classes or functions is not allowed. - -**generators** - - generators are supported, but their exact scope is very limited. you can't - merge two different generator in one control point. - -**exceptions** - -+ fully supported -+ see below `Exception rules`_ for restrictions on exceptions raised by built-in operations - - -Object restrictions -------------------------- - -We are using - -**integer, float, boolean** - - works. - -**strings** - - a lot of, but not all string methods are supported and those that are - supported, not necesarilly accept all arguments. Indexes can be - negative. In case they are not, then you get slightly more efficient - code if the translator can prove that they are non-negative. When - slicing a string it is necessary to prove that the slice start and - stop indexes are non-negative. There is no implicit str-to-unicode cast - anywhere. Simple string formatting using the ``%`` operator works, as long - as the format string is known at translation time; the only supported - formatting specifiers are ``%s``, ``%d``, ``%x``, ``%o``, ``%f``, plus - ``%r`` but only for user-defined instances. Modifiers such as conversion - flags, precision, length etc. are not supported. Moreover, it is forbidden - to mix unicode and strings when formatting. - -**tuples** - - no variable-length tuples; use them to store or return pairs or n-tuples of - values. Each combination of types for elements and length constitute - a separate and not mixable type. - -**lists** - - lists are used as an allocated array. Lists are over-allocated, so list.append() - is reasonably fast. However, if you use a fixed-size list, the code - is more efficient. Annotator can figure out most of the time that your - list is fixed-size, even when you use list comprehension. - Negative or out-of-bound indexes are only allowed for the - most common operations, as follows: - - - *indexing*: - positive and negative indexes are allowed. Indexes are checked when requested - by an IndexError exception clause. - - - *slicing*: - the slice start must be within bounds. The stop doesn't need to, but it must - not be smaller than the start. All negative indexes are disallowed, except for - the [:-1] special case. No step. Slice deletion follows the same rules. - - - *slice assignment*: - only supports ``lst[x:y] = sublist``, if ``len(sublist) == y - x``. - In other words, slice assignment cannot change the total length of the list, - but just replace items. - - - *other operators*: - ``+``, ``+=``, ``in``, ``*``, ``*=``, ``==``, ``!=`` work as expected. - - - *methods*: - append, index, insert, extend, reverse, pop. The index used in pop() follows - the same rules as for *indexing* above. The index used in insert() must be within - bounds and not negative. - -**dicts** - - dicts with a unique key type only, provided it is hashable. Custom - hash functions and custom equality will not be honored. - Use ``rpython.rlib.objectmodel.r_dict`` for custom hash functions. - - -**list comprehensions** - - May be used to create allocated, initialized arrays. - -**functions** - -+ statically called functions may use defaults and a variable number of - arguments (which may be passed as a list instead of a tuple, so write code - that does not depend on it being a tuple). - -+ dynamic dispatch enforces the use of signatures that are equal for all - possible called function, or at least "compatible enough". This - concerns mainly method calls, when the method is overridden or in any - way given different definitions in different classes. It also concerns - the less common case of explicitly manipulated function objects. - Describing the exact compatibility rules is rather involved (but if you - break them, you should get explicit errors from the rtyper and not - obscure crashes.) - -**builtin functions** - - A number of builtin functions can be used. The precise set can be - found in `rpython/annotator/builtin.py`_ (see ``def builtin_xxx()``). - Some builtin functions may be limited in what they support, though. - - ``int, float, str, ord, chr``... are available as simple conversion - functions. Note that ``int, float, str``... have a special meaning as - a type inside of isinstance only. - -**classes** - -+ methods and other class attributes do not change after startup -+ single inheritance is fully supported -+ use `rpython.rlib.objectmodel.import_from_mixin(M)` in a class - body to copy the whole content of a class `M`. This can be used - to implement mixins: functions and staticmethods are duplicated - (the other class attributes are just copied unmodified). - -+ classes are first-class objects too - -**objects** - - Normal rules apply. The only special methods that are honoured are - ``__init__``, ``__del__``, ``__len__``, ``__getitem__``, ``__setitem__``, - ``__getslice__``, ``__setslice__``, and ``__iter__``. To handle slicing, - ``__getslice__`` and ``__setslice__`` must be used; using ``__getitem__`` and - ``__setitem__`` for slicing isn't supported. Additionally, using negative - indices for slicing is still not support, even when using ``__getslice__``. - -This layout makes the number of types to take care about quite limited. - - -Integer Types -------------------------- - -While implementing the integer type, we stumbled over the problem that -integers are quite in flux in CPython right now. Starting with Python 2.4, -integers mutate into longs on overflow. In contrast, we need -a way to perform wrap-around machine-sized arithmetic by default, while still -being able to check for overflow when we need it explicitly. Moreover, we need -a consistent behavior before and after translation. - -We use normal integers for signed arithmetic. It means that before -translation we get longs in case of overflow, and after translation we get a -silent wrap-around. Whenever we need more control, we use the following -helpers (which live in `rpython/rlib/rarithmetic.py`_): - -**ovfcheck()** - - This special function should only be used with a single arithmetic operation - as its argument, e.g. ``z = ovfcheck(x+y)``. Its intended meaning is to - perform the given operation in overflow-checking mode. - - At run-time, in Python, the ovfcheck() function itself checks the result - and raises OverflowError if it is a ``long``. But the code generators use - ovfcheck() as a hint: they replace the whole ``ovfcheck(x+y)`` expression - with a single overflow-checking addition in C. - -**intmask()** - - This function is used for wrap-around arithmetic. It returns the lower bits - of its argument, masking away anything that doesn't fit in a C "signed long int". - Its purpose is, in Python, to convert from a Python ``long`` that resulted from a - previous operation back to a Python ``int``. The code generators ignore - intmask() entirely, as they are doing wrap-around signed arithmetic all the time - by default anyway. (We have no equivalent of the "int" versus "long int" - distinction of C at the moment and assume "long ints" everywhere.) - -**r_uint** - - In a few cases (e.g. hash table manipulation), we need machine-sized unsigned - arithmetic. For these cases there is the r_uint class, which is a pure - Python implementation of word-sized unsigned integers that silently wrap - around. ("word-sized" and "machine-sized" are used equivalently and mean - the native size, which you get using "unsigned long" in C.) - The purpose of this class (as opposed to helper functions as above) - is consistent typing: both Python and the annotator will propagate r_uint - instances in the program and interpret all the operations between them as - unsigned. Instances of r_uint are special-cased by the code generators to - use the appropriate low-level type and operations. - Mixing of (signed) integers and r_uint in operations produces r_uint that - means unsigned results. To convert back from r_uint to signed integers, use - intmask(). - - -Exception rules ---------------------- - -Exceptions are by default not generated for simple cases.:: - - #!/usr/bin/python - - lst = [1,2,3,4,5] - item = lst[i] # this code is not checked for out-of-bound access - - try: - item = lst[i] - except IndexError: - # complain - -Code with no exception handlers does not raise exceptions (after it has been -translated, that is. When you run it on top of CPython, it may raise -exceptions, of course). By supplying an exception handler, you ask for error -checking. Without, you assure the system that the operation cannot fail. -This rule does not apply to *function calls*: any called function is -assumed to be allowed to raise any exception. - -For example:: - - x = 5.1 - x = x + 1.2 # not checked for float overflow - try: - x = x + 1.2 - except OverflowError: - # float result too big - -But:: - - z = some_function(x, y) # can raise any exception - try: - z = some_other_function(x, y) - except IndexError: - # only catches explicitly-raised IndexErrors in some_other_function() - # other exceptions can be raised, too, and will not be caught here. - -The ovfcheck() function described above follows the same rule: in case of -overflow, it explicitly raise OverflowError, which can be caught anywhere. - -Exceptions explicitly raised or re-raised will always be generated. - -PyPy is debuggable on top of CPython ------------------------------------- - -PyPy has the advantage that it is runnable on standard -CPython. That means, we can run all of PyPy with all exception -handling enabled, so we might catch cases where we failed to -adhere to our implicit assertions. - -.. _`wrapping rules`: -.. _`wrapped`: - - +.. _wrapped: Wrapping rules -============== +-------------- Wrapping ---------- +~~~~~~~~ PyPy is made of Python source code at two levels: there is on the one hand *application-level code* that looks like normal Python code, and that @@ -500,7 +202,7 @@ structure. For example, an application-level Python ``list`` -is implemented by the `standard object space`_ as an +is implemented by the :ref:`standard object space ` as an instance of ``W_ListObject``, which has an instance attribute ``wrappeditems`` (an interpreter-level list which contains the application-level list's items as wrapped objects). @@ -509,7 +211,7 @@ Naming conventions ------------------- +~~~~~~~~~~~~~~~~~~ * ``space``: the object space is only visible at interpreter-level code, where it is by convention passed around by the name @@ -529,14 +231,14 @@ Operations on ``w_xxx`` ------------------------ +~~~~~~~~~~~~~~~~~~~~~~~ The core bytecode interpreter considers wrapped objects as black boxes. It is not allowed to inspect them directly. The allowed operations are all implemented on the object space: they are called ``space.xxx()``, where ``xxx`` is a standard operation name (``add``, ``getattr``, ``call``, ``eq``...). They are documented in the -`object space document`_. +:ref:`object space document `. A short warning: **don't do** ``w_x == w_y`` or ``w_x is w_y``! rationale for this rule is that there is no reason that two @@ -548,12 +250,11 @@ use ``space.is_true(space.is_(w_x, w_y))`` or better ``space.is_w(w_x, w_y)``. -.. _`object space document`: objspace.html#interface -.. _`applevel-exceptions`: +.. _applevel-exceptions: Application-level exceptions ----------------------------- +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Interpreter-level code can use exceptions freely. However, all application-level exceptions are represented as an @@ -585,10 +286,10 @@ instances of subclasses. -.. _`modules`: +.. _modules: Modules in PyPy -=============== +--------------- Modules visible from application programs are imported from interpreter or application level files. PyPy reuses almost all python @@ -597,18 +298,19 @@ because they rely on implementation details of CPython. If we don't just modify an original CPython module but need to rewrite -it from scratch we put it into `lib_pypy/`_ as a pure application level +it from scratch we put it into :source:`lib_pypy/` as a pure application level module. When we need access to interpreter-level objects we put the module into -`pypy/module`_. Such modules use a `mixed module mechanism`_ +:source:`pypy/module`. Such modules use a `mixed module mechanism`_ which makes it convenient to use both interpreter- and application-level parts for the implementation. Note that there is no extra facility for pure-interpreter level modules, you just write a mixed module and leave the application-level part empty. + Determining the location of a module implementation ---------------------------------------------------- +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ You can interactively find out where a module comes from, when running py.py. here are examples for the possible locations:: @@ -626,8 +328,9 @@ '/home/hpk/pypy-dist/lib-python/2.7/os.py' >>>> + Module directories / Import order ---------------------------------- +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Here is the order in which PyPy looks up Python modules: @@ -650,10 +353,11 @@ The modified CPython library. -.. _`modify modules`: + +.. _modify modules: Modifying a CPython library module or regression test -------------------------------------------------------- +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Although PyPy is very compatible with CPython we sometimes need to change modules contained in our copy of the standard library, @@ -665,34 +369,31 @@ to see what is changed we have a branch called `vendor/stdlib` wich contains the unmodified cpython stdlib -.. _`mixed module mechanism`: -.. _`mixed modules`: + +.. _mixed module mechanism: +.. _mixed-modules: Implementing a mixed interpreter/application level Module ---------------------------------------------------------- +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ If a module needs to access PyPy's interpreter level then it is implemented as a mixed module. -Mixed modules are directories in `pypy/module`_ with an `__init__.py` +Mixed modules are directories in :source:`pypy/module` with an `__init__.py` file containing specifications where each name in a module comes from. Only specified names will be exported to a Mixed Module's applevel namespace. -Sometimes it is necessary to really write some functions in C (or -whatever target language). See `rffi`_ and `external functions -documentation`_ for details. The latter approach is cumbersome and -being phased out and former has currently quite a few rough edges. +Sometimes it is necessary to really write some functions in C (or whatever +target language). See :ref:`rffi ` details. -.. _`rffi`: rffi.html -.. _`external functions documentation`: translation.html#extfunccalls application level definitions -............................. +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Application level specifications are found in the `appleveldefs` dictionary found in ``__init__.py`` files of directories in ``pypy/module``. -For example, in `pypy/module/__builtin__/__init__.py`_ you find the following +For example, in :source:`pypy/module/__builtin__/__init__.py` you find the following entry specifying where ``__builtin__.locals`` comes from:: ... @@ -703,12 +404,13 @@ interpreted at application level and the wrapped function value for ``locals`` will be extracted accordingly. + interpreter level definitions -............................. +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Interpreter level specifications are found in the ``interpleveldefs`` dictionary found in ``__init__.py`` files of directories in ``pypy/module``. -For example, in `pypy/module/__builtin__/__init__.py`_ the following +For example, in :source:`pypy/module/__builtin__/__init__.py` the following entry specifies where ``__builtin__.len`` comes from:: ... @@ -724,7 +426,7 @@ return space.len(w_obj) Exposed interpreter level functions usually take a ``space`` argument -and some wrapped values (see `wrapping rules`_) . +and some wrapped values (see `Wrapping rules`_) . You can also use a convenient shortcut in ``interpleveldefs`` dictionaries: namely an expression in parentheses to specify an interpreter level @@ -743,37 +445,40 @@ --withoutmod-mymodule (the latter being the default)) for py.py and translate.py. + Testing modules in ``lib_pypy/`` --------------------------------- +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -You can go to the `pypy/module/test_lib_pypy/`_ directory and invoke the testing tool +You can go to the :source:`pypy/module/test_lib_pypy/` directory and invoke the testing tool ("py.test" or "python ../../pypy/test_all.py") to run tests against the -lib_pypy hierarchy. Note, that tests in `pypy/module/test_lib_pypy/`_ are allowed +lib_pypy hierarchy. Note, that tests in :source:`pypy/module/test_lib_pypy/` are allowed and encouraged to let their tests run at interpreter level although -`lib_pypy/`_ modules eventually live at PyPy's application level. +:source:`lib_pypy/` modules eventually live at PyPy's application level. This allows us to quickly test our python-coded reimplementations against CPython. + Testing modules in ``pypy/module`` ----------------------------------- +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Simply change to ``pypy/module`` or to a subdirectory and `run the tests as usual`_. Testing modules in ``lib-python`` ------------------------------------ +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ In order to let CPython's regression tests run against PyPy -you can switch to the `lib-python/`_ directory and run +you can switch to the :source:`lib-python/` directory and run the testing tool in order to start compliance tests. (XXX check windows compatibility for producing test reports). + Naming conventions and directory layout -=========================================== +--------------------------------------- Directory and File Naming -------------------------- +~~~~~~~~~~~~~~~~~~~~~~~~~ - directories/modules/namespaces are always **lowercase** @@ -786,8 +491,9 @@ - keep filenames concise and completion-friendly. + Naming of python objects ------------------------- +~~~~~~~~~~~~~~~~~~~~~~~~ - class names are **CamelCase** @@ -803,8 +509,9 @@ includes w_self. Don't use ``w_`` in application level python only code. + Committing & Branching to the repository ------------------------------------------------------ +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - write good log messages because several people are reading the diffs. @@ -815,22 +522,23 @@ ``try1`` doesn't already exists) you should do:: hg branch try1 - + The branch will be recorded in the repository only after a commit. To switch back to the default branch:: - + hg update default - + For further details use the help or refer to the `official wiki`_:: - + hg help branch -.. _`official wiki`: http://mercurial.selenic.com/wiki/Branch +.. _official wiki: http://mercurial.selenic.com/wiki/Branch -.. _`using development tracker`: + +.. _using-development-tracker: Using the development bug/feature tracker -========================================= +----------------------------------------- We have a `development tracker`_, based on Richard Jones' `roundup`_ application. You can file bugs, @@ -838,15 +546,14 @@ for the next milestone, both from an E-Mail and from a web interface. -.. _`development tracker`: https://bugs.pypy.org/ -.. _`roundup`: http://roundup.sourceforge.net/ +.. _development tracker: https://bugs.pypy.org/ +.. _roundup: http://roundup.sourceforge.net/ -.. _`testing in PyPy`: -.. _`test-design`: +.. _testing: Testing in PyPy -=============== +--------------- Our tests are based on the `py.test`_ tool which lets you write unittests without boilerplate. All tests of modules @@ -859,12 +566,11 @@ - **Application Level tests**. They run at application level which means that they look like straight python code but they are interpreted by PyPy. -.. _`standard object space`: objspace.html#standard-object-space -.. _`objectspace`: objspace.html -.. _`py.test`: http://pytest.org/ +.. _py.test: http://pytest.org/ + Interpreter level tests ------------------------ +~~~~~~~~~~~~~~~~~~~~~~~ You can write test functions and methods like this:: @@ -880,8 +586,9 @@ module global level and use plain 'assert' statements thanks to the usage of the `py.test`_ tool. + Application Level tests ------------------------ +~~~~~~~~~~~~~~~~~~~~~~~ For testing the conformance and well-behavedness of PyPy it is often sufficient to write "normal" application-level @@ -916,7 +623,8 @@ assert self.d["a"] == 1 assert self.d["b"] == 2 -.. _`run the tests as usual`: + +.. _run the tests as usual: Another possibility is to use cls.space.appexec, for example:: @@ -959,7 +667,7 @@ Command line tool test_all --------------------------- +~~~~~~~~~~~~~~~~~~~~~~~~~~ You can run almost all of PyPy's tests by invoking:: @@ -969,8 +677,9 @@ located in the ``py/bin/`` directory. For switches to modify test execution pass the ``-h`` option. + Coverage reports ----------------- +~~~~~~~~~~~~~~~~ In order to get coverage reports the `pytest-cov`_ plugin is included. it adds some extra requirements ( coverage_ and `cov-core`_ ) @@ -978,12 +687,13 @@ python test_all.py --cov file_or_direcory_to_cover file_or_directory -.. _`pytest-cov`: http://pypi.python.org/pypi/pytest-cov -.. _`coverage`: http://pypi.python.org/pypi/coverage -.. _`cov-core`: http://pypi.python.org/pypi/cov-core +.. _pytest-cov: http://pypi.python.org/pypi/pytest-cov +.. _coverage: http://pypi.python.org/pypi/coverage +.. _cov-core: http://pypi.python.org/pypi/cov-core + Test conventions ----------------- +~~~~~~~~~~~~~~~~ - adding features requires adding appropriate tests. (It often even makes sense to first write the tests so that you are sure that they @@ -993,29 +703,28 @@ which contain unit tests. Such scripts can usually be executed directly or are collectively run by pypy/test_all.py -.. _`change documentation and website`: + +.. _change documentation and website: Changing documentation and website -================================== +---------------------------------- documentation/website files in your local checkout ---------------------------------------------------- +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Most of the PyPy's documentation is kept in `pypy/doc`. You can simply edit or add '.rst' files which contain ReST-markuped files. Here is a `ReST quickstart`_ but you can also just look at the existing documentation and see how things work. -.. _`ReST quickstart`: http://docutils.sourceforge.net/docs/user/rst/quickref.html - Note that the web site of http://pypy.org/ is maintained separately. For now it is in the repository https://bitbucket.org/pypy/pypy.org +.. _ReST quickstart: http://docutils.sourceforge.net/docs/user/rst/quickref.html + + Automatically test documentation/website changes ------------------------------------------------- - -.. _`sphinx home page`: -.. _`sphinx`: http://sphinx.pocoo.org/ +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ We automatically check referential integrity and ReST-conformance. In order to run the tests you need sphinx_ installed. Then go to the local checkout @@ -1035,5 +744,4 @@ which will check that remote URLs are reachable. - -.. include:: _ref.txt +.. _sphinx: http://sphinx.pocoo.org/ diff --git a/pypy/doc/conf.py b/pypy/doc/conf.py --- a/pypy/doc/conf.py +++ b/pypy/doc/conf.py @@ -40,9 +40,9 @@ # Add any Sphinx extension module names here, as strings. They can be extensions # coming with Sphinx (named 'sphinx.ext.*') or your custom ones. -extensions = ['sphinx.ext.autodoc', 'sphinx.ext.doctest', 'sphinx.ext.intersphinx', - 'sphinx.ext.todo', 'sphinx.ext.ifconfig', 'sphinx.ext.graphviz', - 'pypyconfig'] +extensions = ['sphinx.ext.autodoc', 'sphinx.ext.doctest', 'sphinx.ext.extlinks', + 'sphinx.ext.intersphinx', 'sphinx.ext.todo', 'sphinx.ext.ifconfig', + 'sphinx.ext.graphviz', 'pypyconfig'] # Add any paths that contain templates here, relative to this directory. templates_path = ['_templates'] @@ -215,7 +215,8 @@ # Example configuration for intersphinx: refer to the Python standard library. -intersphinx_mapping = {'http://docs.python.org/': None} +intersphinx_mapping = {'python': ('http://docs.python.org/2.7', None), + 'rpython': ('http://rpython.readthedocs.org/en/latest/', None)} # -- Options for manpage output------------------------------------------------- @@ -225,3 +226,4 @@ u'The PyPy Project', 1) ] +extlinks = {'source': ('https://bitbucket.org/pypy/pypy/src/default/%s', '')} diff --git a/pypy/doc/config/confrest.py b/pypy/doc/config/confrest.py deleted file mode 100644 --- a/pypy/doc/config/confrest.py +++ /dev/null @@ -1,62 +0,0 @@ -from pypy.doc.confrest import * -from pypy.config.makerestdoc import make_cmdline_overview -from pypy.config import pypyoption -from rpython.config.config import Config -from rpython.config import translationoption - - -all_optiondescrs = [pypyoption.pypy_optiondescription, - translationoption.translation_optiondescription, - ] -start_to_descr = dict([(descr._name, descr) for descr in all_optiondescrs]) - -class PyPyPage(PyPyPage): - def fill(self): - super(PyPyPage, self).fill() - self.menubar[:] = html.div( - html.a("general documentation", href="../index.html", - class_="menu"), " ", - html.a("config index", href="index.html", - class_="menu"), " ", - html.a("command-line overview", href="commandline.html", - class_="menu"), " ", - " ", id="menubar") - -class Project(Project): - stylesheet = "../style.css" - title = "PyPy Configuration" - prefix_title = "PyPy Configuration" - Page = PyPyPage - - def get_content(self, txtpath, encoding): - if txtpath.basename == "commandline.rst": - result = [] - for line in txtpath.read().splitlines(): - if line.startswith('.. GENERATE:'): - start = line[len('.. GENERATE:'):].strip() - descr = start_to_descr[start] - line = make_cmdline_overview(descr, title=False).text() - result.append(line) - return "\n".join(result) - fullpath = txtpath.purebasename - start = fullpath.split(".")[0] - path = fullpath.rsplit(".", 1)[0] - basedescr = start_to_descr.get(start) - if basedescr is None: - return txtpath.read() - if fullpath.count(".") == 0: - descr = basedescr - path = "" - else: - conf = Config(basedescr) - subconf, step = conf._cfgimpl_get_home_by_path( - fullpath.split(".", 1)[1]) - descr = getattr(subconf._cfgimpl_descr, step) - text = unicode(descr.make_rest_doc(path).text()) - if txtpath.check(file=True): - content = txtpath.read() - if content: - text += "\nDescription\n===========" - return u"%s\n\n%s" % (text, unicode(txtpath.read(), encoding)) - return text - diff --git a/pypy/doc/config/generate.py b/pypy/doc/config/generate.py --- a/pypy/doc/config/generate.py +++ b/pypy/doc/config/generate.py @@ -1,6 +1,5 @@ import py from pypy.config import pypyoption, makerestdoc -from pypy.doc.config.confrest import all_optiondescrs from rpython.config import translationoption, config all_optiondescrs = [pypyoption.pypy_optiondescription, diff --git a/pypy/doc/config/objspace.std.builtinshortcut.txt b/pypy/doc/config/objspace.std.builtinshortcut.txt deleted file mode 100644 --- a/pypy/doc/config/objspace.std.builtinshortcut.txt +++ /dev/null @@ -1,5 +0,0 @@ -A shortcut speeding up primitive operations between built-in types. - -This is a space-time trade-off: at the moment, this option makes a -translated pypy-c executable bigger by about 1.7 MB. (This can probably -be improved with careful analysis.) diff --git a/pypy/doc/config/objspace.std.multimethods.txt b/pypy/doc/config/objspace.std.multimethods.txt deleted file mode 100644 --- a/pypy/doc/config/objspace.std.multimethods.txt +++ /dev/null @@ -1,8 +0,0 @@ -Choose the multimethod implementation. - -* ``doubledispatch`` turns - a multimethod call into a sequence of normal method calls. - -* ``mrd`` uses a technique known as Multiple Row Displacement - which precomputes a few compact tables of numbers and - function pointers. diff --git a/pypy/doc/configuration.rst b/pypy/doc/configuration.rst --- a/pypy/doc/configuration.rst +++ b/pypy/doc/configuration.rst @@ -1,4 +1,3 @@ -============================= PyPy's Configuration Handling ============================= @@ -6,16 +5,17 @@ hand the necessary options to where they are actually used and even more annoying to add new options. To circumvent these problems configuration management was introduced. There all the necessary options are stored in a -configuration object, which is available nearly everywhere in the `RPython +configuration object, which is available nearly everywhere in the `RPython toolchain`_ and in the standard interpreter so that adding new options becomes trivial. Options are organized into a tree. Configuration objects can be created in different ways, there is support for creating an optparse command line parser automatically. -_`RPython toolchain`: translation.html +.. _RPython toolchain: https://rpython.readthedocs.org/ + Main Assumption -=============== +--------------- Configuration objects are produced at the entry points and handed down to where they are actually used. This keeps configuration local but available @@ -24,7 +24,7 @@ API Details -=========== +----------- The handling of options is split into two parts: the description of which options are available, what their possible values and defaults are and how they @@ -40,8 +40,9 @@ option group. The parts of the full name of the option are separated by dots: e.g. ``config.translation.thread``. + Description of Options ----------------------- +~~~~~~~~~~~~~~~~~~~~~~ All the constructors take a ``name`` and a ``doc`` argument as first arguments to give the option or option group a name and to document it. Most constructors @@ -70,6 +71,7 @@ ``children`` is a list of option descriptions (including ``OptionDescription`` instances for nested namespaces). + ``ChoiceOption`` ++++++++++++++++ @@ -81,10 +83,11 @@ ``requires`` is a dictionary mapping values to lists of of two-element tuples. + ``BoolOption`` ++++++++++++++ -Represents a choice between ``True`` and ``False``. +Represents a choice between ``True`` and ``False``. ``__init__(self, name, doc, default=None, requires=None, suggests=None, cmdline=DEFAULT, negation=True)`` ``default`` specifies the default value of the option. ``requires`` is @@ -103,7 +106,6 @@ Represents a choice of an integer. ``__init__(self, name, doc, default=None, cmdline=DEFAULT)`` - ``FloatOption`` @@ -112,7 +114,6 @@ Represents a choice of a floating point number. ``__init__(self, name, doc, default=None, cmdline=DEFAULT)`` - ``StrOption`` @@ -121,12 +122,10 @@ Represents the choice of a string. From noreply at buildbot.pypy.org Sat Nov 1 21:48:39 2014 From: noreply at buildbot.pypy.org (mattip) Date: Sat, 1 Nov 2014 21:48:39 +0100 (CET) Subject: [pypy-commit] pypy nditer-external_loop: cleanup, document branch to be merged Message-ID: <20141101204839.B7C481C026A@cobra.cs.uni-duesseldorf.de> Author: mattip Branch: nditer-external_loop Changeset: r74321:418e458600c7 Date: 2014-10-31 08:27 +0200 http://bitbucket.org/pypy/pypy/changeset/418e458600c7/ Log: cleanup, document branch to be merged diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -39,3 +39,7 @@ .. branch: kill-multimethod Kill multimethod machinery, all multimethods were removed earlier. + +.. branch nditer-external_loop + +Implement `external_loop` arguement to numpy's nditer diff --git a/pypy/module/micronumpy/nditer.py b/pypy/module/micronumpy/nditer.py --- a/pypy/module/micronumpy/nditer.py +++ b/pypy/module/micronumpy/nditer.py @@ -197,11 +197,8 @@ astrides = op_it.array.strides[:indx] # does op_it iters over array "naturally" if astrides != op_it.strides: - print 'cannot coalesce array strides', op_it.array.strides, - print 'with iter strides', op_it.strides can_coalesce = False break - # is the resulting slice array contiguous if can_coalesce: for i in range(len(it.iters)): old_iter = it.iters[i][0] From noreply at buildbot.pypy.org Sat Nov 1 21:48:40 2014 From: noreply at buildbot.pypy.org (mattip) Date: Sat, 1 Nov 2014 21:48:40 +0100 (CET) Subject: [pypy-commit] pypy nditer-external_loop: close branch to be merged Message-ID: <20141101204840.EF1491C026A@cobra.cs.uni-duesseldorf.de> Author: mattip Branch: nditer-external_loop Changeset: r74322:f44932374644 Date: 2014-10-31 08:28 +0200 http://bitbucket.org/pypy/pypy/changeset/f44932374644/ Log: close branch to be merged From noreply at buildbot.pypy.org Sat Nov 1 21:48:42 2014 From: noreply at buildbot.pypy.org (mattip) Date: Sat, 1 Nov 2014 21:48:42 +0100 (CET) Subject: [pypy-commit] pypy default: merge nditer-external_loop which implements numpy's nditer external_loop argument Message-ID: <20141101204842.2F7B51C026A@cobra.cs.uni-duesseldorf.de> Author: mattip Branch: Changeset: r74323:55056cc539e0 Date: 2014-10-31 08:30 +0200 http://bitbucket.org/pypy/pypy/changeset/55056cc539e0/ Log: merge nditer-external_loop which implements numpy's nditer external_loop argument diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -39,3 +39,7 @@ .. branch: kill-multimethod Kill multimethod machinery, all multimethods were removed earlier. + +.. branch nditer-external_loop + +Implement `external_loop` arguement to numpy's nditer diff --git a/pypy/module/micronumpy/concrete.py b/pypy/module/micronumpy/concrete.py --- a/pypy/module/micronumpy/concrete.py +++ b/pypy/module/micronumpy/concrete.py @@ -449,7 +449,7 @@ strides.reverse() backstrides.reverse() new_shape.reverse() - return SliceArray(self.start, strides, backstrides, new_shape, + return self.__class__(self.start, strides, backstrides, new_shape, self, orig_array) new_strides = calc_new_strides(new_shape, self.get_shape(), self.get_strides(), @@ -460,10 +460,16 @@ new_backstrides = [0] * len(new_shape) for nd in range(len(new_shape)): new_backstrides[nd] = (new_shape[nd] - 1) * new_strides[nd] - return SliceArray(self.start, new_strides, new_backstrides, new_shape, + return self.__class__(self.start, new_strides, new_backstrides, new_shape, self, orig_array) +class NonWritableSliceArray(SliceArray): + def descr_setitem(self, space, orig_array, w_index, w_value): + raise OperationError(space.w_ValueError, space.wrap( + "assignment destination is read-only")) + + class VoidBoxStorage(BaseConcreteArray): def __init__(self, size, dtype): self.storage = alloc_raw_storage(size) diff --git a/pypy/module/micronumpy/iterators.py b/pypy/module/micronumpy/iterators.py --- a/pypy/module/micronumpy/iterators.py +++ b/pypy/module/micronumpy/iterators.py @@ -8,8 +8,8 @@ At which byte in x.data does the item x[3,4] begin? if x.strides==[1,5]: pData = x.pData + (x.start + 3*1 + 4*5)*sizeof(x.pData[0]) - pData = x.pData + (x.start + 24) * sizeof(x.pData[0]) -so the offset of the element is 24 elements after the first + pData = x.pData + (x.start + 23) * sizeof(x.pData[0]) +so the offset of the element is 23 elements after the first What is the next element in x after coordinates [3,4]? if x.order =='C': @@ -33,7 +33,7 @@ which is x.strides[1] * (x.shape[1] - 1) + x.strides[0] so if we precalculate the overflow backstride as [x.strides[i] * (x.shape[i] - 1) for i in range(len(x.shape))] -we can go faster. +we can do only addition while iterating All the calculations happen in next() """ from rpython.rlib import jit @@ -41,6 +41,16 @@ from pypy.module.micronumpy.base import W_NDimArray from pypy.module.micronumpy.flagsobj import _update_contiguous_flags +class OpFlag(object): + def __init__(self): + self.rw = '' + self.broadcast = True + self.force_contig = False + self.force_align = False + self.native_byte_order = False + self.tmp_copy = '' + self.allocate = False + class PureShapeIter(object): def __init__(self, shape, idx_w): @@ -89,11 +99,13 @@ class ArrayIter(object): _immutable_fields_ = ['contiguous', 'array', 'size', 'ndim_m1', 'shape_m1[*]', 'strides[*]', 'backstrides[*]', 'factors[*]', - 'track_index'] + 'slice_shape', 'slice_stride', 'slice_backstride', + 'track_index', 'operand_type', 'slice_operand_type'] track_index = True - def __init__(self, array, size, shape, strides, backstrides): + def __init__(self, array, size, shape, strides, backstrides, op_flags=OpFlag()): + from pypy.module.micronumpy import concrete assert len(shape) == len(strides) == len(backstrides) _update_contiguous_flags(array) self.contiguous = (array.flags & NPY.ARRAY_C_CONTIGUOUS and @@ -105,6 +117,12 @@ self.shape_m1 = [s - 1 for s in shape] self.strides = strides self.backstrides = backstrides + self.slice_shape = 1 + self.slice_stride = -1 + if strides: + self.slice_stride = strides[-1] + self.slice_backstride = 1 + self.slice_operand_type = concrete.SliceArray ndim = len(shape) factors = [0] * ndim @@ -114,6 +132,10 @@ else: factors[ndim-i-1] = factors[ndim-i] * shape[ndim-i] self.factors = factors + if op_flags.rw == 'r': + self.operand_type = concrete.ConcreteNonWritableArrayWithBase + else: + self.operand_type = concrete.ConcreteArrayWithBase @jit.unroll_safe def reset(self, state=None): @@ -193,6 +215,12 @@ assert state.iterator is self self.array.setitem(state.offset, elem) + def getoperand(self, st, base): + impl = self.operand_type + res = impl([], self.array.dtype, self.array.order, [], [], + self.array.storage, base) + res.start = st.offset + return res def AxisIter(array, shape, axis, cumulative): strides = array.get_strides() @@ -216,3 +244,42 @@ size /= shape[axis] shape[axis] = backstrides[axis] = 0 return ArrayIter(array, size, shape, array.strides, backstrides) + +class SliceIter(ArrayIter): + ''' + used with external loops, getitem and setitem return a SliceArray + view into the original array + ''' + _immutable_fields_ = ['base', 'slice_shape[*]', 'slice_stride[*]', 'slice_backstride[*]'] + + def __init__(self, array, size, shape, strides, backstrides, slice_shape, + slice_stride, slice_backstride, op_flags, base): + from pypy.module.micronumpy import concrete + ArrayIter.__init__(self, array, size, shape, strides, backstrides, op_flags) + self.slice_shape = slice_shape + self.slice_stride = slice_stride + self.slice_backstride = slice_backstride + self.base = base + if op_flags.rw == 'r': + self.slice_operand_type = concrete.NonWritableSliceArray + else: + self.slice_operand_type = concrete.SliceArray + + def getitem(self, state): + # XXX cannot be called - must return a boxed value + assert False + + def getitem_bool(self, state): + # XXX cannot be called - must return a boxed value + assert False + + def setitem(self, state, elem): + # XXX cannot be called - must return a boxed value + assert False + + def getoperand(self, state, base): + assert state.iterator is self + impl = self.slice_operand_type + arr = impl(state.offset, [self.slice_stride], [self.slice_backstride], + [self.slice_shape], self.array, self.base) + return arr diff --git a/pypy/module/micronumpy/ndarray.py b/pypy/module/micronumpy/ndarray.py --- a/pypy/module/micronumpy/ndarray.py +++ b/pypy/module/micronumpy/ndarray.py @@ -83,8 +83,12 @@ raise OperationError(space.w_AttributeError, space.wrap( "Cannot delete array dtype")) + def ndims(self): + return len(self.get_shape()) + ndims._always_inline_ = True + def descr_get_ndim(self, space): - return space.wrap(len(self.get_shape())) + return space.wrap(self.ndims()) def descr_get_itemsize(self, space): return space.wrap(self.get_dtype().elsize) @@ -103,14 +107,14 @@ return space.wrap(loop.tostring(space, self)) def getitem_filter(self, space, arr): - if len(arr.get_shape()) > 1 and arr.get_shape() != self.get_shape(): + if arr.ndims() > 1 and arr.get_shape() != self.get_shape(): raise OperationError(space.w_ValueError, space.wrap( "boolean index array should have 1 dimension")) if arr.get_size() > self.get_size(): raise OperationError(space.w_ValueError, space.wrap( "index out of range for array")) size = loop.count_all_true(arr) - if len(arr.get_shape()) == 1: + if arr.ndims() == 1: res_shape = [size] + self.get_shape()[1:] else: res_shape = [size] @@ -119,7 +123,7 @@ return loop.getitem_filter(w_res, self, arr) def setitem_filter(self, space, idx, val): - if len(idx.get_shape()) > 1 and idx.get_shape() != self.get_shape(): + if idx.ndims() > 1 and idx.get_shape() != self.get_shape(): raise OperationError(space.w_ValueError, space.wrap( "boolean index array should have 1 dimension")) if idx.get_size() > self.get_size(): @@ -210,7 +214,7 @@ if space.is_w(w_idx, space.w_Ellipsis): return self elif isinstance(w_idx, W_NDimArray) and w_idx.get_dtype().is_bool() \ - and len(w_idx.get_shape()) > 0: + and w_idx.ndims() > 0: return self.getitem_filter(space, w_idx) try: return self.implementation.descr_getitem(space, self, w_idx) @@ -228,7 +232,7 @@ self.implementation.setslice(space, convert_to_array(space, w_value)) return elif isinstance(w_idx, W_NDimArray) and w_idx.get_dtype().is_bool() \ - and len(w_idx.get_shape()) > 0: + and w_idx.ndims() > 0: self.setitem_filter(space, w_idx, convert_to_array(space, w_value)) return try: @@ -289,7 +293,7 @@ shape=shape, backward_broadcast=backward_broadcast) def is_scalar(self): - return len(self.get_shape()) == 0 + return self.ndims() == 0 def set_scalar_value(self, w_val): return self.implementation.setitem(self.implementation.start, w_val) @@ -408,7 +412,7 @@ """ if axis1 == axis2: return self - n = len(self.get_shape()) + n = self.ndims() if n <= 1: return self if axis1 < 0: @@ -426,7 +430,7 @@ return self.implementation.nonzero(space, index_type) def descr_tolist(self, space): - if len(self.get_shape()) == 0: + if self.ndims() == 0: return self.get_scalar_value().item(space) l_w = [] for i in range(self.get_shape()[0]): @@ -514,7 +518,7 @@ if len(args_w) == 0: raise OperationError(space.w_ValueError, space.wrap( "itemset must have at least one argument")) - if len(args_w) != len(self.get_shape()) + 1: + if len(args_w) != self.ndims() + 1: raise OperationError(space.w_ValueError, space.wrap( "incorrect number of indices for array")) self.descr_setitem(space, space.newtuple(args_w[:-1]), args_w[-1]) @@ -647,14 +651,14 @@ @unwrap_spec(offset=int, axis1=int, axis2=int) def descr_diagonal(self, space, offset=0, axis1=0, axis2=1): - if len(self.get_shape()) < 2: + if self.ndims() < 2: raise OperationError(space.w_ValueError, space.wrap( "need at least 2 dimensions for diagonal")) - if (axis1 < 0 or axis2 < 0 or axis1 >= len(self.get_shape()) or - axis2 >= len(self.get_shape())): + if (axis1 < 0 or axis2 < 0 or axis1 >= self.ndims() or + axis2 >= self.ndims()): raise oefmt(space.w_ValueError, "axis1(=%d) and axis2(=%d) must be withing range " - "(ndim=%d)", axis1, axis2, len(self.get_shape())) + "(ndim=%d)", axis1, axis2, self.ndims()) if axis1 == axis2: raise OperationError(space.w_ValueError, space.wrap( "axis1 and axis2 cannot be the same")) @@ -733,7 +737,7 @@ raise OperationError(space.w_NotImplementedError, space.wrap( 'sorter not supported in searchsort')) side = searchside_converter(space, w_side) - if len(self.get_shape()) != 1: + if self.ndims() != 1: raise oefmt(space.w_ValueError, "a must be a 1-d array") v = convert_to_array(space, w_v) ret = W_NDimArray.from_shape( @@ -972,7 +976,7 @@ if other.is_scalar(): #Note: w_out is not modified, this is numpy compliant. return self.descr_mul(space, other) - elif len(self.get_shape()) < 2 and len(other.get_shape()) < 2: + elif self.ndims() < 2 and other.ndims() < 2: w_res = self.descr_mul(space, other) assert isinstance(w_res, W_NDimArray) return w_res.descr_sum(space, space.wrap(-1), out) @@ -989,7 +993,7 @@ matches = False elif not out.implementation.order == "C": matches = False - elif len(out.get_shape()) != len(out_shape): + elif out.ndims() != len(out_shape): matches = False else: for i in range(len(out_shape)): diff --git a/pypy/module/micronumpy/nditer.py b/pypy/module/micronumpy/nditer.py --- a/pypy/module/micronumpy/nditer.py +++ b/pypy/module/micronumpy/nditer.py @@ -5,7 +5,7 @@ from pypy.module.micronumpy import ufuncs, support, concrete from pypy.module.micronumpy.base import W_NDimArray, convert_to_array from pypy.module.micronumpy.descriptor import decode_w_dtype -from pypy.module.micronumpy.iterators import ArrayIter +from pypy.module.micronumpy.iterators import ArrayIter, SliceIter, OpFlag from pypy.module.micronumpy.strides import (calculate_broadcast_strides, shape_agreement, shape_agreement_multiple) @@ -35,17 +35,6 @@ return ret -class OpFlag(object): - def __init__(self): - self.rw = '' - self.broadcast = True - self.force_contig = False - self.force_align = False - self.native_byte_order = False - self.tmp_copy = '' - self.allocate = False - - def parse_op_flag(space, lst): op_flag = OpFlag() for w_item in lst: @@ -71,17 +60,17 @@ elif item == 'allocate': op_flag.allocate = True elif item == 'no_subtype': - raise OperationError(space.w_NotImplementedError, space.wrap( - '"no_subtype" op_flag not implemented yet')) + raise oefmt(space.w_NotImplementedError, + '"no_subtype" op_flag not implemented yet') elif item == 'arraymask': - raise OperationError(space.w_NotImplementedError, space.wrap( - '"arraymask" op_flag not implemented yet')) + raise oefmt(space.w_NotImplementedError, + '"arraymask" op_flag not implemented yet') elif item == 'writemask': - raise OperationError(space.w_NotImplementedError, space.wrap( - '"writemask" op_flag not implemented yet')) + raise oefmt(space.w_NotImplementedError, + '"writemask" op_flag not implemented yet') else: - raise OperationError(space.w_ValueError, space.wrap( - 'op_flags must be a tuple or array of per-op flag-tuples')) + raise oefmt(space.w_ValueError, + 'op_flags must be a tuple or array of per-op flag-tuples') if op_flag.rw == '': raise oefmt(space.w_ValueError, "None of the iterator flags READWRITE, READONLY, or " @@ -94,8 +83,8 @@ return elif not space.isinstance_w(w_flags, space.w_tuple) and not \ space.isinstance_w(w_flags, space.w_list): - raise OperationError(space.w_ValueError, space.wrap( - 'Iter global flags must be a list or tuple of strings')) + raise oefmt(space.w_ValueError, + 'Iter global flags must be a list or tuple of strings') lst = space.listview(w_flags) for w_item in lst: if not space.isinstance_w(w_item, space.w_str) and not \ @@ -106,12 +95,10 @@ typename) item = space.str_w(w_item) if item == 'external_loop': - raise OperationError(space.w_NotImplementedError, space.wrap( - 'nditer external_loop not implemented yet')) nditer.external_loop = True elif item == 'buffered': - raise OperationError(space.w_NotImplementedError, space.wrap( - 'nditer buffered not implemented yet')) + raise oefmt(space.w_NotImplementedError, + 'nditer buffered not implemented yet') # For numpy compatability nditer.buffered = True elif item == 'c_index': @@ -131,8 +118,8 @@ elif item == 'refs_ok': nditer.refs_ok = True elif item == 'reduce_ok': - raise OperationError(space.w_NotImplementedError, space.wrap( - 'nditer reduce_ok not implemented yet')) + raise oefmt(space.w_NotImplementedError, + 'nditer reduce_ok not implemented yet') nditer.reduce_ok = True elif item == 'zerosize_ok': nditer.zerosize_ok = True @@ -141,9 +128,9 @@ 'Unexpected iterator global flag "%s"', item) if nditer.tracked_index and nditer.external_loop: - raise OperationError(space.w_ValueError, space.wrap( + raise oefmt(space.w_ValueError, 'Iterator flag EXTERNAL_LOOP cannot be used if an index or ' - 'multi-index is being tracked')) + 'multi-index is being tracked') def is_backward(imp, order): @@ -155,11 +142,11 @@ raise NotImplementedError('not implemented yet') -def get_iter(space, order, arr, shape, dtype): +def get_iter(space, order, arr, shape, dtype, op_flags): imp = arr.implementation backward = is_backward(imp, order) if arr.is_scalar(): - return ArrayIter(imp, 1, [], [], []) + return ArrayIter(imp, 1, [], [], [], op_flags=op_flags) if (imp.strides[0] < imp.strides[-1] and not backward) or \ (imp.strides[0] > imp.strides[-1] and backward): # flip the strides. Is this always true for multidimension? @@ -174,8 +161,103 @@ backstrides = imp.backstrides r = calculate_broadcast_strides(strides, backstrides, imp.shape, shape, backward) - return ArrayIter(imp, imp.get_size(), shape, r[0], r[1]) + return ArrayIter(imp, imp.get_size(), shape, r[0], r[1], op_flags=op_flags) +def calculate_ndim(op_in, oa_ndim): + if oa_ndim >=0: + return oa_ndim + else: + ndim = 0 + for op in op_in: + if op is None: + continue + assert isinstance(op, W_NDimArray) + ndim = max(ndim, op.ndims()) + return ndim + +def coalesce_axes(it, space): + # Copy logic from npyiter_coalesce_axes, used in ufunc iterators + # and in nditer's with 'external_loop' flag + can_coalesce = True + if it.order == 'F': + fastest = 0 + else: + fastest = -1 + for idim in range(it.ndim - 1): + for op_it, _ in it.iters: + if op_it is None: + continue + assert isinstance(op_it, ArrayIter) + indx = len(op_it.strides) + if it.order == 'F': + indx = len(op_it.array.strides) - indx + assert indx >=0 + astrides = op_it.array.strides[indx:] + else: + astrides = op_it.array.strides[:indx] + # does op_it iters over array "naturally" + if astrides != op_it.strides: + can_coalesce = False + break + if can_coalesce: + for i in range(len(it.iters)): + old_iter = it.iters[i][0] + shape = [s+1 for s in old_iter.shape_m1] + strides = old_iter.strides + backstrides = old_iter.backstrides + if it.order == 'F': + new_shape = shape[1:] + new_strides = strides[1:] + new_backstrides = backstrides[1:] + _stride = min(strides[0], old_iter.slice_stride) + else: + new_shape = shape[:-1] + new_strides = strides[:-1] + new_backstrides = backstrides[:-1] + _stride = old_iter.slice_stride + # We always want the "fastest" iterator in external loops + _shape = shape[fastest] * old_iter.slice_shape + _backstride = (_shape - 1) * _stride + new_iter = SliceIter(old_iter.array, old_iter.size / shape[fastest], + new_shape, new_strides, new_backstrides, + _shape, _stride, _backstride, + it.op_flags[i], it) + it.iters[i] = (new_iter, new_iter.reset()) + if len(it.shape) > 1: + if it.order == 'F': + it.shape = it.shape[1:] + else: + it.shape = it.shape[:-1] + else: + it.shape = [1] + else: + break + # Always coalesce at least one + for i in range(len(it.iters)): + old_iter = it.iters[i][0] + shape = [s+1 for s in old_iter.shape_m1] + strides = old_iter.strides + backstrides = old_iter.backstrides + new_shape = shape[:-1] + new_strides = strides[:-1] + new_backstrides = backstrides[:-1] + _shape = shape[-1] * old_iter.slice_shape + # use the operand's iterator's rightmost stride, + # even if it is not the fastest (for 'F' or swapped axis) + _stride = old_iter.slice_stride + _backstride = (_shape - 1) * _stride + new_iter = SliceIter(old_iter.array, old_iter.size / shape[-1], + new_shape, new_strides, new_backstrides, + _shape, _stride, _backstride, + it.op_flags[i], it) + it.iters[i] = (new_iter, new_iter.reset()) + if len(it.shape) > 1: + if it.order == 'F': + it.shape = it.shape[1:] + else: + it.shape = it.shape[:-1] + else: + it.shape = [1] class IndexIterator(object): def __init__(self, shape, backward=False): @@ -205,6 +287,7 @@ class W_NDIter(W_Root): + _immutable_fields_ = ['ndim', ] def __init__(self, space, w_seq, w_flags, w_op_flags, w_op_dtypes, w_casting, w_op_axes, w_itershape, w_buffersize, order): self.order = order @@ -236,28 +319,29 @@ self.op_flags = parse_op_arg(space, 'op_flags', w_op_flags, len(self.seq), parse_op_flag) # handle w_op_axes + oa_ndim = -1 if not space.is_none(w_op_axes): - self.set_op_axes(space, w_op_axes) + oa_ndim = self.set_op_axes(space, w_op_axes) + self.ndim = calculate_ndim(self.seq, oa_ndim) # handle w_op_dtypes part 1: creating self.dtypes list from input if not space.is_none(w_op_dtypes): w_seq_as_list = space.listview(w_op_dtypes) self.dtypes = [decode_w_dtype(space, w_elem) for w_elem in w_seq_as_list] if len(self.dtypes) != len(self.seq): - raise OperationError(space.w_ValueError, space.wrap( - "op_dtypes must be a tuple/list matching the number of ops")) + raise oefmt(space.w_ValueError, + "op_dtypes must be a tuple/list matching the number of ops") else: self.dtypes = [] # handle None or writable operands, calculate my shape - self.iters = [] outargs = [i for i in range(len(self.seq)) if self.seq[i] is None or self.op_flags[i].rw == 'w'] if len(outargs) > 0: out_shape = shape_agreement_multiple(space, [self.seq[i] for i in outargs]) else: out_shape = None - self.shape = iter_shape = shape_agreement_multiple(space, self.seq, + self.shape = shape_agreement_multiple(space, self.seq, shape=out_shape) if len(outargs) > 0: # Make None operands writeonly and flagged for allocation @@ -276,11 +360,11 @@ for i in outargs: if self.seq[i] is None: # XXX can we postpone allocation to later? - self.seq[i] = W_NDimArray.from_shape(space, iter_shape, out_dtype) + self.seq[i] = W_NDimArray.from_shape(space, self.shape, out_dtype) else: if not self.op_flags[i].broadcast: # Raises if ooutput cannot be broadcast - shape_agreement(space, iter_shape, self.seq[i], False) + shape_agreement(space, self.shape, self.seq[i], False) if self.tracked_index != "": if self.order == "K": @@ -289,7 +373,7 @@ backward = False else: backward = self.order != self.tracked_index - self.index_iter = IndexIterator(iter_shape, backward=backward) + self.index_iter = IndexIterator(self.shape, backward=backward) # handle w_op_dtypes part 2: copy where needed if possible if len(self.dtypes) > 0: @@ -311,49 +395,49 @@ self.dtypes = [s.get_dtype() for s in self.seq] # create an iterator for each operand + self.iters = [] for i in range(len(self.seq)): - it = get_iter(space, self.order, self.seq[i], iter_shape, self.dtypes[i]) + it = get_iter(space, self.order, self.seq[i], self.shape, + self.dtypes[i], self.op_flags[i]) it.contiguous = False self.iters.append((it, it.reset())) + if self.external_loop: + coalesce_axes(self, space) + def set_op_axes(self, space, w_op_axes): if space.len_w(w_op_axes) != len(self.seq): raise oefmt(space.w_ValueError, "op_axes must be a tuple/list matching the number of ops") op_axes = space.listview(w_op_axes) - l = -1 + oa_ndim = -1 for w_axis in op_axes: if not space.is_none(w_axis): axis_len = space.len_w(w_axis) - if l == -1: - l = axis_len - elif axis_len != l: + if oa_ndim == -1: + oa_ndim = axis_len + elif axis_len != oa_ndim: raise oefmt(space.w_ValueError, "Each entry of op_axes must have the same size") self.op_axes.append([space.int_w(x) if not space.is_none(x) else -1 for x in space.listview(w_axis)]) - if l == -1: + if oa_ndim == -1: raise oefmt(space.w_ValueError, "If op_axes is provided, at least one list of axes " "must be contained within it") - raise Exception('xxx TODO') + raise oefmt(space.w_NotImplementedError, "op_axis not finished yet") # Check that values make sense: # - in bounds for each operand # ValueError: Iterator input op_axes[0][3] (==3) is not a valid axis of op[0], which has 2 dimensions # - no repeat axis # ValueError: The 'op_axes' provided to the iterator constructor for operand 1 contained duplicate value 0 + return oa_ndim def descr_iter(self, space): return space.wrap(self) - def getitem(self, it, st, op_flags): - if op_flags.rw == 'r': - impl = concrete.ConcreteNonWritableArrayWithBase - else: - impl = concrete.ConcreteArrayWithBase - res = impl([], it.array.dtype, it.array.order, [], [], - it.array.storage, self) - res.start = st.offset + def getitem(self, it, st): + res = it.getoperand(st, self) return W_NDimArray(res) def descr_getitem(self, space, w_idx): @@ -363,7 +447,7 @@ except IndexError: raise oefmt(space.w_IndexError, "Iterator operand index %d is out of bounds", idx) - return self.getitem(it, st, self.op_flags[idx]) + return self.getitem(it, st) def descr_setitem(self, space, w_idx, w_value): raise oefmt(space.w_NotImplementedError, "not implemented yet") @@ -385,7 +469,7 @@ else: self.first_next = False for i, (it, st) in enumerate(self.iters): - res.append(self.getitem(it, st, self.op_flags[i])) + res.append(self.getitem(it, st)) self.iters[i] = (it, it.next(st)) if len(res) < 2: return res[0] @@ -477,7 +561,7 @@ raise oefmt(space.w_NotImplementedError, "not implemented yet") def descr_get_ndim(self, space): - raise oefmt(space.w_NotImplementedError, "not implemented yet") + return space.wrap(self.ndim) def descr_get_nop(self, space): raise oefmt(space.w_NotImplementedError, "not implemented yet") diff --git a/pypy/module/micronumpy/test/test_nditer.py b/pypy/module/micronumpy/test/test_nditer.py --- a/pypy/module/micronumpy/test/test_nditer.py +++ b/pypy/module/micronumpy/test/test_nditer.py @@ -63,9 +63,6 @@ from numpy import arange, nditer, array a = arange(24).reshape(2, 3, 4) import sys - if '__pypy__' in sys.builtin_module_names: - raises(NotImplementedError, nditer, a, flags=['external_loop']) - skip('nditer external_loop not implmented') r = [] n = 0 for x in nditer(a, flags=['external_loop']): @@ -79,7 +76,9 @@ r.append(x) n += 1 assert n == 12 - assert (array(r) == [[ 0, 12], [ 4, 16], [ 8, 20], [ 1, 13], [ 5, 17], [ 9, 21], [ 2, 14], [ 6, 18], [10, 22], [ 3, 15], [ 7, 19], [11, 23]]).all() + assert (array(r) == [[ 0, 12], [ 4, 16], [ 8, 20], [ 1, 13], [ 5, 17], [ 9, 21], + [ 2, 14], [ 6, 18], [10, 22], [ 3, 15], [ 7, 19], [11, 23], + ]).all() e = raises(ValueError, 'r[0][0] = 0') assert str(e.value) == 'assignment destination is read-only' r = [] @@ -222,9 +221,6 @@ def test_outarg(self): from numpy import nditer, zeros, arange import sys - if '__pypy__' in sys.builtin_module_names: - raises(NotImplementedError, nditer, [1, 2], flags=['external_loop']) - skip('nditer external_loop not implmented') def square1(a): it = nditer([a, None]) @@ -233,6 +229,9 @@ return it.operands[1] assert (square1([1, 2, 3]) == [1, 4, 9]).all() + if '__pypy__' in sys.builtin_module_names: + raises(NotImplementedError, nditer, [1, 2], flags=['buffered']) + skip('nditer buffered not implmented') def square2(a, out=None): it = nditer([a, out], flags=['external_loop', 'buffered'], op_flags=[['readonly'], @@ -252,10 +251,11 @@ from numpy import nditer, arange a = arange(3) import sys + b = arange(8).reshape(2,4) if '__pypy__' in sys.builtin_module_names: - raises(NotImplementedError, nditer, a, flags=['external_loop']) - skip('nditer external_loop not implmented') - b = arange(8).reshape(2,4) + raises(NotImplementedError, nditer, [a, b, None], flags=['external_loop'], + op_axes=[[0, -1, -1], [-1, 0, 1], None]) + skip('nditer op_axes not implemented yet') it = nditer([a, b, None], flags=['external_loop'], op_axes=[[0, -1, -1], [-1, 0, 1], None]) for x, y, z in it: From noreply at buildbot.pypy.org Sat Nov 1 21:48:45 2014 From: noreply at buildbot.pypy.org (mattip) Date: Sat, 1 Nov 2014 21:48:45 +0100 (CET) Subject: [pypy-commit] pypy ufuncapi: merge default into branch Message-ID: <20141101204845.4F0A61C026A@cobra.cs.uni-duesseldorf.de> Author: mattip Branch: ufuncapi Changeset: r74324:a8d365caf86e Date: 2014-10-31 14:01 +0200 http://bitbucket.org/pypy/pypy/changeset/a8d365caf86e/ Log: merge default into branch diff too long, truncating to 2000 out of 40806 lines diff --git a/.hgignore b/.hgignore --- a/.hgignore +++ b/.hgignore @@ -89,6 +89,7 @@ ^pypy/doc/image/lattice3\.png$ ^pypy/doc/image/stackless_informal\.png$ ^pypy/doc/image/parsing_example.+\.png$ +^rpython/doc/_build/.*$ ^pypy/module/test_lib_pypy/ctypes_tests/_ctypes_test\.o$ ^compiled ^.git/ diff --git a/README.rst b/README.rst --- a/README.rst +++ b/README.rst @@ -23,6 +23,7 @@ the pypy-dev team + Building ======== diff --git a/ctypes_configure/doc/configure.txt b/ctypes_configure/doc/configure.txt --- a/ctypes_configure/doc/configure.txt +++ b/ctypes_configure/doc/configure.txt @@ -19,6 +19,4 @@ usage ===== -`sample.py`_ explains in details how to use it. - -.. _`sample.py`: http://codespeak.net/svn/pypy/dist/ctypes_configure/doc/sample.py +:source:`sample.py ` explains in details how to use it. diff --git a/pypy/config/pypyoption.py b/pypy/config/pypyoption.py --- a/pypy/config/pypyoption.py +++ b/pypy/config/pypyoption.py @@ -255,10 +255,6 @@ BoolOption("optimized_list_getitem", "special case the 'list[integer]' expressions", default=False), - BoolOption("builtinshortcut", - "a shortcut for operations between built-in types. XXX: " - "deprecated, not really a shortcut any more.", - default=False), BoolOption("getattributeshortcut", "track types that override __getattribute__", default=False, @@ -270,9 +266,6 @@ # weakrefs needed, because of get_subclasses() requires=[("translation.rweakref", True)]), - ChoiceOption("multimethods", "the multimethod implementation to use", - ["doubledispatch", "mrd"], - default="mrd"), BoolOption("withidentitydict", "track types that override __hash__, __eq__ or __cmp__ and use a special dict strategy for those which do not", default=False, diff --git a/pypy/config/test/test_pypyoption.py b/pypy/config/test/test_pypyoption.py --- a/pypy/config/test/test_pypyoption.py +++ b/pypy/config/test/test_pypyoption.py @@ -64,7 +64,7 @@ def check_file_exists(fn): assert configdocdir.join(fn).check() - from pypy.doc.config.confrest import all_optiondescrs + from pypy.doc.config.generate import all_optiondescrs configdocdir = thisdir.dirpath().dirpath().join("doc", "config") for descr in all_optiondescrs: prefix = descr._name diff --git a/pypy/doc/TODO b/pypy/doc/TODO new file mode 100644 --- /dev/null +++ b/pypy/doc/TODO @@ -0,0 +1,40 @@ +Documentation TODO +================== + +General +------- + +* architecture documents don't really show the separation between PyPy and + RPython + * architecture.rst is duplicate (both pypy and rpython) +* Consider moving information from pypy/doc/{build,windows}.rst to rpython/doc + + +Cleanup +~~~~~~~ + +* remove documentation on removed features + * various object spaces +* update / remove dead links + + +Meta +~~~~ + +* work on configuration/options documentation generation + + +PyPy +---- + +* Update coding guide +* Move links from project-documentation.rst to index.rst and consider killing + it. + + +RPython +------- + +* make translation.rst a high-level overview and move details in their own + documents +* redo various outdated pictures in translation.rst diff --git a/pypy/doc/__pypy__-module.rst b/pypy/doc/__pypy__-module.rst --- a/pypy/doc/__pypy__-module.rst +++ b/pypy/doc/__pypy__-module.rst @@ -1,20 +1,17 @@ - .. comment: this document is very incomplete, should we generate it automatically? -======================= The ``__pypy__`` module ======================= The ``__pypy__`` module is the main entry point to special features provided -by PyPy's standard interpreter. Its content depends on `configuration options`_ -which may add new functionality and functions whose existence or non-existence -indicates the presence of such features. +by PyPy's standard interpreter. Its content depends on :doc:`configuration options ` +which may add new functionality and functions whose existence or non-existence +indicates the presence of such features. -.. _`configuration options`: config/index.html Generally available functionality -================================= +--------------------------------- - ``internal_repr(obj)``: return the interpreter-level representation of an object. @@ -22,28 +19,22 @@ It works like a simplified array of characters (actually, depending on the configuration the ``array`` module internally uses this). + Transparent Proxy Functionality -=============================== +------------------------------- -If `transparent proxies`_ are enabled (with :config:`objspace.std.withtproxy`) +If :ref:`transparent proxies ` are enabled (with :config:`objspace.std.withtproxy`) the following functions are put into ``__pypy__``: - ``tproxy(typ, controller)``: Return something that looks like it is of type typ. Its behaviour is completely controlled by the controller. See the docs - about `transparent proxies`_ for detail. - + about :ref:`transparent proxies ` for detail. - ``get_tproxy_controller(obj)``: If obj is really a transparent proxy, return its controller. Otherwise return None. -.. _`transparent proxies`: objspace-proxies.html#tproxy - Functionality available on py.py (not after translation) -======================================================== +-------------------------------------------------------- - ``isfake(obj)``: returns True if ``obj`` is faked. - - ``interp_pdb()``: start a pdb at interpreter-level. - - - diff --git a/pypy/doc/_ref.txt b/pypy/doc/_ref.txt deleted file mode 100644 --- a/pypy/doc/_ref.txt +++ /dev/null @@ -1,114 +0,0 @@ -.. This file is generated automatically by makeref.py script, - which in turn is run manually. - -.. _`ctypes_configure/doc/sample.py`: https://bitbucket.org/pypy/pypy/src/default/ctypes_configure/doc/sample.py -.. _`dotviewer/`: https://bitbucket.org/pypy/pypy/src/default/dotviewer/ -.. _`lib-python/`: https://bitbucket.org/pypy/pypy/src/default/lib-python/ -.. _`lib-python/2.7/dis.py`: https://bitbucket.org/pypy/pypy/src/default/lib-python/2.7/dis.py -.. _`lib_pypy/`: https://bitbucket.org/pypy/pypy/src/default/lib_pypy/ -.. _`lib_pypy/greenlet.py`: https://bitbucket.org/pypy/pypy/src/default/lib_pypy/greenlet.py -.. _`lib_pypy/tputil.py`: https://bitbucket.org/pypy/pypy/src/default/lib_pypy/tputil.py -.. _`pypy/bin/`: https://bitbucket.org/pypy/pypy/src/default/pypy/bin/ -.. _`pypy/bin/pyinteractive.py`: https://bitbucket.org/pypy/pypy/src/default/pypy/bin/pyinteractive.py -.. _`pypy/config/`: https://bitbucket.org/pypy/pypy/src/default/pypy/config/ -.. _`pypy/config/pypyoption.py`: https://bitbucket.org/pypy/pypy/src/default/pypy/config/pypyoption.py -.. _`pypy/doc/`: https://bitbucket.org/pypy/pypy/src/default/pypy/doc/ -.. _`pypy/doc/config/`: https://bitbucket.org/pypy/pypy/src/default/pypy/doc/config/ -.. _`pypy/doc/discussion/`: https://bitbucket.org/pypy/pypy/src/default/pypy/doc/discussion/ -.. _`pypy/goal/`: https://bitbucket.org/pypy/pypy/src/default/pypy/goal/ -.. _`pypy/interpreter`: -.. _`pypy/interpreter/`: https://bitbucket.org/pypy/pypy/src/default/pypy/interpreter/ -.. _`pypy/interpreter/argument.py`: https://bitbucket.org/pypy/pypy/src/default/pypy/interpreter/argument.py -.. _`pypy/interpreter/astcompiler`: -.. _`pypy/interpreter/astcompiler/`: https://bitbucket.org/pypy/pypy/src/default/pypy/interpreter/astcompiler/ -.. _`pypy/interpreter/astcompiler/assemble.py`: https://bitbucket.org/pypy/pypy/src/default/pypy/interpreter/astcompiler/assemble.py -.. _`pypy/interpreter/astcompiler/ast.py`: https://bitbucket.org/pypy/pypy/src/default/pypy/interpreter/astcompiler/ast.py -.. _`pypy/interpreter/astcompiler/astbuilder.py`: https://bitbucket.org/pypy/pypy/src/default/pypy/interpreter/astcompiler/astbuilder.py -.. _`pypy/interpreter/astcompiler/asthelpers.py`: https://bitbucket.org/pypy/pypy/src/default/pypy/interpreter/astcompiler/asthelpers.py -.. _`pypy/interpreter/astcompiler/codegen.py`: https://bitbucket.org/pypy/pypy/src/default/pypy/interpreter/astcompiler/codegen.py -.. _`pypy/interpreter/astcompiler/optimize.py`: https://bitbucket.org/pypy/pypy/src/default/pypy/interpreter/astcompiler/optimize.py -.. _`pypy/interpreter/astcompiler/symtable.py`: https://bitbucket.org/pypy/pypy/src/default/pypy/interpreter/astcompiler/symtable.py -.. _`pypy/interpreter/astcompiler/tools/Python.asdl`: https://bitbucket.org/pypy/pypy/src/default/pypy/interpreter/astcompiler/tools/Python.asdl -.. _`pypy/interpreter/astcompiler/tools/asdl_py.py`: https://bitbucket.org/pypy/pypy/src/default/pypy/interpreter/astcompiler/tools/asdl_py.py -.. _`pypy/interpreter/baseobjspace.py`: https://bitbucket.org/pypy/pypy/src/default/pypy/interpreter/baseobjspace.py -.. _`pypy/interpreter/eval.py`: https://bitbucket.org/pypy/pypy/src/default/pypy/interpreter/eval.py -.. _`pypy/interpreter/executioncontext.py`: https://bitbucket.org/pypy/pypy/src/default/pypy/interpreter/executioncontext.py -.. _`pypy/interpreter/function.py`: https://bitbucket.org/pypy/pypy/src/default/pypy/interpreter/function.py -.. _`pypy/interpreter/gateway.py`: https://bitbucket.org/pypy/pypy/src/default/pypy/interpreter/gateway.py -.. _`pypy/interpreter/mixedmodule.py`: https://bitbucket.org/pypy/pypy/src/default/pypy/interpreter/mixedmodule.py -.. _`pypy/interpreter/module.py`: https://bitbucket.org/pypy/pypy/src/default/pypy/interpreter/module.py -.. _`pypy/interpreter/pyframe.py`: https://bitbucket.org/pypy/pypy/src/default/pypy/interpreter/pyframe.py -.. _`pypy/interpreter/pyopcode.py`: https://bitbucket.org/pypy/pypy/src/default/pypy/interpreter/pyopcode.py -.. _`pypy/interpreter/pyparser`: -.. _`pypy/interpreter/pyparser/`: https://bitbucket.org/pypy/pypy/src/default/pypy/interpreter/pyparser/ -.. _`pypy/interpreter/pyparser/future.py`: https://bitbucket.org/pypy/pypy/src/default/pypy/interpreter/pyparser/future.py -.. _`pypy/interpreter/pyparser/metaparser.py`: https://bitbucket.org/pypy/pypy/src/default/pypy/interpreter/pyparser/metaparser.py -.. _`pypy/interpreter/pyparser/parser.py`: https://bitbucket.org/pypy/pypy/src/default/pypy/interpreter/pyparser/parser.py -.. _`pypy/interpreter/pyparser/pyparse.py`: https://bitbucket.org/pypy/pypy/src/default/pypy/interpreter/pyparser/pyparse.py -.. _`pypy/interpreter/pyparser/pytokenizer.py`: https://bitbucket.org/pypy/pypy/src/default/pypy/interpreter/pyparser/pytokenizer.py -.. _`pypy/interpreter/typedef.py`: https://bitbucket.org/pypy/pypy/src/default/pypy/interpreter/typedef.py -.. _`pypy/module`: -.. _`pypy/module/`: https://bitbucket.org/pypy/pypy/src/default/pypy/module/ -.. _`pypy/module/__builtin__/__init__.py`: https://bitbucket.org/pypy/pypy/src/default/pypy/module/__builtin__/__init__.py -.. _`pypy/module/cppyy/capi/__init__.py`: https://bitbucket.org/pypy/pypy/src/default/pypy/module/cppyy/capi/__init__.py -.. _`pypy/module/cppyy/capi/builtin_capi.py`: https://bitbucket.org/pypy/pypy/src/default/pypy/module/cppyy/capi/builtin_capi.py -.. _`pypy/module/cppyy/include/capi.h`: https://bitbucket.org/pypy/pypy/src/default/pypy/module/cppyy/include/capi.h -.. _`pypy/module/test_lib_pypy/`: https://bitbucket.org/pypy/pypy/src/default/pypy/module/test_lib_pypy/ -.. _`pypy/objspace/`: https://bitbucket.org/pypy/pypy/src/default/pypy/objspace/ -.. _`pypy/objspace/std`: -.. _`pypy/objspace/std/`: https://bitbucket.org/pypy/pypy/src/default/pypy/objspace/std/ -.. _`pypy/objspace/std/bytesobject.py`: https://bitbucket.org/pypy/pypy/src/default/pypy/objspace/std/bytesobject.py -.. _`pypy/objspace/std/multimethod.py`: https://bitbucket.org/pypy/pypy/src/default/pypy/objspace/std/multimethod.py -.. _`pypy/objspace/std/objspace.py`: https://bitbucket.org/pypy/pypy/src/default/pypy/objspace/std/objspace.py -.. _`pypy/objspace/std/proxy_helpers.py`: https://bitbucket.org/pypy/pypy/src/default/pypy/objspace/std/proxy_helpers.py -.. _`pypy/objspace/std/proxyobject.py`: https://bitbucket.org/pypy/pypy/src/default/pypy/objspace/std/proxyobject.py -.. _`pypy/objspace/std/strbufobject.py`: https://bitbucket.org/pypy/pypy/src/default/pypy/objspace/std/strbufobject.py -.. _`pypy/objspace/std/transparent.py`: https://bitbucket.org/pypy/pypy/src/default/pypy/objspace/std/transparent.py -.. _`pypy/tool/`: https://bitbucket.org/pypy/pypy/src/default/pypy/tool/ -.. _`pypy/tool/pytest/`: https://bitbucket.org/pypy/pypy/src/default/pypy/tool/pytest/ -.. _`rpython/annotator`: -.. _`rpython/annotator/`: https://bitbucket.org/pypy/pypy/src/default/rpython/annotator/ -.. _`rpython/annotator/annrpython.py`: https://bitbucket.org/pypy/pypy/src/default/rpython/annotator/annrpython.py -.. _`rpython/annotator/binaryop.py`: https://bitbucket.org/pypy/pypy/src/default/rpython/annotator/binaryop.py -.. _`rpython/annotator/builtin.py`: https://bitbucket.org/pypy/pypy/src/default/rpython/annotator/builtin.py -.. _`rpython/bin/translatorshell.py`: https://bitbucket.org/pypy/pypy/src/default/rpython/bin/translatorshell.py -.. _`rpython/config/`: https://bitbucket.org/pypy/pypy/src/default/rpython/config/ -.. _`rpython/config/translationoption.py`: https://bitbucket.org/pypy/pypy/src/default/rpython/config/translationoption.py -.. _`rpython/flowspace/`: https://bitbucket.org/pypy/pypy/src/default/rpython/flowspace/ -.. _`rpython/flowspace/model.py`: https://bitbucket.org/pypy/pypy/src/default/rpython/flowspace/model.py -.. _`rpython/memory/`: https://bitbucket.org/pypy/pypy/src/default/rpython/memory/ -.. _`rpython/memory/gc/generation.py`: https://bitbucket.org/pypy/pypy/src/default/rpython/memory/gc/generation.py -.. _`rpython/memory/gc/hybrid.py`: https://bitbucket.org/pypy/pypy/src/default/rpython/memory/gc/hybrid.py -.. _`rpython/memory/gc/minimarkpage.py`: https://bitbucket.org/pypy/pypy/src/default/rpython/memory/gc/minimarkpage.py -.. _`rpython/memory/gc/semispace.py`: https://bitbucket.org/pypy/pypy/src/default/rpython/memory/gc/semispace.py -.. _`rpython/rlib`: -.. _`rpython/rlib/`: https://bitbucket.org/pypy/pypy/src/default/rpython/rlib/ -.. _`rpython/rlib/listsort.py`: https://bitbucket.org/pypy/pypy/src/default/rpython/rlib/listsort.py -.. _`rpython/rlib/nonconst.py`: https://bitbucket.org/pypy/pypy/src/default/rpython/rlib/nonconst.py -.. _`rpython/rlib/objectmodel.py`: https://bitbucket.org/pypy/pypy/src/default/rpython/rlib/objectmodel.py -.. _`rpython/rlib/parsing/`: https://bitbucket.org/pypy/pypy/src/default/rpython/rlib/parsing/ -.. _`rpython/rlib/parsing/tree.py`: https://bitbucket.org/pypy/pypy/src/default/rpython/rlib/parsing/tree.py -.. _`rpython/rlib/rarithmetic.py`: https://bitbucket.org/pypy/pypy/src/default/rpython/rlib/rarithmetic.py -.. _`rpython/rlib/rbigint.py`: https://bitbucket.org/pypy/pypy/src/default/rpython/rlib/rbigint.py -.. _`rpython/rlib/rrandom.py`: https://bitbucket.org/pypy/pypy/src/default/rpython/rlib/rrandom.py -.. _`rpython/rlib/rsocket.py`: https://bitbucket.org/pypy/pypy/src/default/rpython/rlib/rsocket.py -.. _`rpython/rlib/streamio.py`: https://bitbucket.org/pypy/pypy/src/default/rpython/rlib/streamio.py -.. _`rpython/rlib/test`: https://bitbucket.org/pypy/pypy/src/default/rpython/rlib/test/ -.. _`rpython/rlib/unroll.py`: https://bitbucket.org/pypy/pypy/src/default/rpython/rlib/unroll.py -.. _`rpython/rtyper`: -.. _`rpython/rtyper/`: https://bitbucket.org/pypy/pypy/src/default/rpython/rtyper/ -.. _`rpython/rtyper/lltypesystem/`: https://bitbucket.org/pypy/pypy/src/default/rpython/rtyper/lltypesystem/ -.. _`rpython/rtyper/lltypesystem/lltype.py`: https://bitbucket.org/pypy/pypy/src/default/rpython/rtyper/lltypesystem/lltype.py -.. _`rpython/rtyper/rint.py`: https://bitbucket.org/pypy/pypy/src/default/rpython/rtyper/rint.py -.. _`rpython/rtyper/rlist.py`: https://bitbucket.org/pypy/pypy/src/default/rpython/rtyper/rlist.py -.. _`rpython/rtyper/rmodel.py`: https://bitbucket.org/pypy/pypy/src/default/rpython/rtyper/rmodel.py -.. _`rpython/rtyper/rtyper.py`: https://bitbucket.org/pypy/pypy/src/default/rpython/rtyper/rtyper.py -.. _`rpython/rtyper/test/test_llinterp.py`: https://bitbucket.org/pypy/pypy/src/default/rpython/rtyper/test/test_llinterp.py -.. _`rpython/tool/algo/`: https://bitbucket.org/pypy/pypy/src/default/rpython/tool/algo/ -.. _`rpython/translator`: -.. _`rpython/translator/`: https://bitbucket.org/pypy/pypy/src/default/rpython/translator/ -.. _`rpython/translator/backendopt/`: https://bitbucket.org/pypy/pypy/src/default/rpython/translator/backendopt/ -.. _`rpython/translator/c/`: https://bitbucket.org/pypy/pypy/src/default/rpython/translator/c/ -.. _`rpython/translator/c/src/stacklet/`: https://bitbucket.org/pypy/pypy/src/default/rpython/translator/c/src/stacklet/ -.. _`rpython/translator/c/src/stacklet/stacklet.h`: https://bitbucket.org/pypy/pypy/src/default/rpython/translator/c/src/stacklet/stacklet.h -.. _`rpython/translator/tool/`: https://bitbucket.org/pypy/pypy/src/default/rpython/translator/tool/ diff --git a/pypy/doc/architecture.rst b/pypy/doc/architecture.rst --- a/pypy/doc/architecture.rst +++ b/pypy/doc/architecture.rst @@ -1,93 +1,29 @@ -================================================== -Goals and Architecture Overview -================================================== +Goals and Architecture Overview +=============================== .. contents:: +This document gives an overview of the goals and architecture of PyPy. If you're +interested in :ref:`using PyPy ` or :ref:`hacking on it `, +have a look at our :ref:`getting started ` section. -This document gives an overview of the goals and architecture of PyPy. -See `getting started`_ for a practical introduction and starting points. -Mission statement -==================== +Mission statement +----------------- -We aim to provide: +We aim to provide a compliant, flexible and fast implementation of the Python_ +Language which uses the RPython toolchain to enable new advanced high-level +features without having to encode the low-level details. We call this PyPy. - * a common translation and support framework for producing - implementations of dynamic languages, emphasizing a clean - separation between language specification and implementation - aspects. We call this the `RPython toolchain`_. +.. _Python: http://docs.python.org/reference/ - * a compliant, flexible and fast implementation of the Python_ Language - which uses the above toolchain to enable new advanced high-level features - without having to encode the low-level details. We call this PyPy. - -By separating concerns in this way, our implementation -of Python - and other dynamic languages - is able to automatically -generate a Just-in-Time compiler for any dynamic language. It also -allows a mix-and-match approach to implementation decisions, including -many that have historically been outside of a user's control, such as -target platform, memory and -threading models, garbage collection strategies, and optimizations applied, -including whether or not to have a JIT in the first place. High Level Goals -============================= - -RPython - the Translation Toolchain ------------------------------------------------ - -Traditionally, language interpreters are written in a target platform language -such as C/Posix, Java or C#. Each implementation provides -a fundamental mapping between application source code and the target -environment. One of -the goals of the "all-encompassing" environments, such as the .NET framework -and to some extent the Java virtual machine, is to provide standardized -and higher level functionalities in order to support language implementers -for writing language implementations. - -PyPy is experimenting with a more ambitious approach. We are using a -subset of the high-level language Python, called RPython_, in which we -write languages as simple interpreters with few references to and -dependencies on lower level details. The `RPython toolchain`_ -produces a concrete virtual machine for the platform of our choice by -inserting appropriate lower level aspects. The result can be customized -by selecting other feature and platform configurations. - -Our goal is to provide a possible solution to the problem of language -implementers: having to write ``l * o * p`` interpreters for ``l`` -dynamic languages and ``p`` platforms with ``o`` crucial design -decisions. PyPy aims at making it possible to change each of these -variables independently such that: - -* ``l``: the language that we analyze can be evolved or entirely replaced; - -* ``o``: we can tweak and optimize the translation process to produce - platform specific code based on different models and trade-offs; - -* ``p``: we can write new translator back-ends to target different - physical and virtual platforms. - -By contrast, a standardized target environment - say .NET - -enforces ``p=1`` as far as it's concerned. This helps making ``o`` a -bit smaller by providing a higher-level base to build upon. Still, -we believe that enforcing the use of one common environment -is not necessary. PyPy's goal is to give weight to this claim - at least -as far as language implementation is concerned - showing an approach -to the ``l * o * p`` problem that does not rely on standardization. - -The most ambitious part of this goal is to `generate Just-In-Time -Compilers`_ in a language-independent way, instead of only translating -the source interpreter into an interpreter for the target platform. -This is an area of language implementation that is commonly considered -very challenging because of the involved complexity. - - -PyPy - the Python Interpreter --------------------------------------------- +---------------- Our main motivation for developing the translation framework is to -provide a full featured, customizable, fast_ and `very compliant`_ Python +provide a full featured, customizable, :ref:`fast ` and +:doc:`very compliant ` Python implementation, working on and interacting with a large variety of platforms and allowing the quick introduction of new advanced language features. @@ -106,88 +42,22 @@ and fully orthogonal to the interpreter source code. -PyPy Architecture -=========================== - -As you would expect from a project implemented using ideas from the world -of `Extreme Programming`_, the architecture of PyPy has evolved over time -and continues to evolve. Nevertheless, the high level architecture is -stable. As described above, there are two rather independent basic -subsystems: the `PyPy Python Interpreter`_ and the `RPython Translation Toolchain`_. - -.. _`translation framework`: - -RPython Translation Toolchain -------------------------- - -The job of the RPython toolchain is to translate RPython_ programs -into an efficient version of that program for one of the various target -platforms, generally one that is considerably lower-level than Python. - -The approach we have taken is to reduce the level of abstraction of the -source RPython program in several steps, from the high level down to the -level of the target platform, whatever that may be. Currently we -support two broad flavours of target platforms: the ones that assume a -C-like memory model with structures and pointers, and the ones that -assume an object-oriented model with classes, instances and methods (as, -for example, the Java and .NET virtual machines do). - -The RPython toolchain never sees the RPython source code or syntax -trees, but rather starts with the *code objects* that define the -behaviour of the function objects one gives it as input. It can be -considered as "freezing" a pre-imported RPython program into an -executable form suitable for the target platform. - -The steps of the translation process can be summarized as follows: - -* The code object of each source functions is converted to a `control - flow graph` by the `Flow Object Space`_. - -* The control flow graphs are processed by the Annotator_, which - performs whole-program type inference to annotate each variable of - the control flow graph with the types it may take at run-time. - -* The information provided by the annotator is used by the RTyper_ to - convert the high level operations of the control flow graphs into - operations closer to the abstraction level of the target platform. - -* Optionally, `various transformations`_ can then be applied which, for - example, perform optimizations such as inlining, add capabilities - such as stackless-style concurrency, or insert code for the - `garbage collector`_. - -* Then, the graphs are converted to source code for the target platform - and compiled into an executable. - -This process is described in much more detail in the `document about -the RPython toolchain`_ and in the paper `Compiling dynamic language -implementations`_. - -.. _`control flow graph`: translation.html#the-flow-model -.. _`Flow Object Space`: objspace.html#the-flow-object-space -.. _Annotator: translation.html#the-annotation-pass -.. _RTyper: rtyper.html#overview -.. _`various transformations`: translation.html#the-optional-transformations -.. _`document about the RPython toolchain`: translation.html -.. _`garbage collector`: garbage_collection.html -.. _`RPython toolchain`: translation.html -.. _`standard interpreter`: -.. _`python interpreter`: +.. _python-interpreter: PyPy Python Interpreter -------------------------------------- +----------------------- PyPy's *Python Interpreter* is written in RPython and implements the full Python language. This interpreter very closely emulates the behavior of CPython. It contains the following key components: -- a bytecode compiler responsible for producing Python code objects +- a bytecode compiler responsible for producing Python code objects from the source code of a user application; -- a `bytecode evaluator`_ responsible for interpreting +- a :doc:`bytecode evaluator ` responsible for interpreting Python code objects; -- a `standard object space`_, responsible for creating and manipulating +- a :ref:`standard object space `, responsible for creating and manipulating the Python objects seen by the application. The *bytecode compiler* is the preprocessing phase that produces a @@ -200,64 +70,6 @@ lists, as well as the operations between them, like addition or truth-value-testing. -This division between bytecode evaluator and object space is very -important, as it gives a lot of flexibility. One can plug in -different `object spaces`_ to get different or enriched behaviours -of the Python objects. Additionally, a special more abstract object -space, the `flow object space`_, allows us to reuse the bytecode -evaluator for our translation framework. - -.. _`bytecode evaluator`: interpreter.html -.. _`standard object space`: objspace.html#the-standard-object-space -.. _`object spaces`: objspace.html -.. _`flow object space`: objspace.html#the-flow-object-space - -.. _`the translation framework`: - - -Further reading -=============== - -All of PyPy's documentation can be reached from the `documentation -index`_. Of particular interest after reading this document might be: - - * `getting-started`_: a hands-on guide to getting involved with the - PyPy source code. - - * `PyPy's approach to virtual machine construction`_: a paper - presented to the Dynamic Languages Symposium attached to OOPSLA - 2006. - - * `The translation document`_: a detailed description of our - translation process. - - * `JIT Generation in PyPy`_, describing how we produce a Just-in-time - Compiler from an interpreter. - - * A tutorial of how to use the `RPython toolchain`_ to `implement your own - interpreter`_. - -.. _`documentation index`: index.html#project-documentation -.. _`getting-started`: getting-started-dev.html -.. _`PyPy's approach to virtual machine construction`: https://bitbucket.org/pypy/extradoc/raw/tip/talk/dls2006/pypy-vm-construction.pdf -.. _`the translation document`: translation.html -.. _`RPython toolchain`: translation.html -.. _`Compiling dynamic language implementations`: https://bitbucket.org/pypy/extradoc/raw/tip/eu-report/D05.1_Publish_on_translating_a_very-high-level_description.pdf -.. _`Technical reports`: index-report.html - -.. _`getting started`: getting-started.html -.. _`Extreme Programming`: http://www.extremeprogramming.org/ - -.. _fast: faq.html#how-fast-is-pypy -.. _`very compliant`: cpython_differences.html - -.. _`RPython`: coding-guide.html#rpython - -.. _Python: http://docs.python.org/reference/ -.. _Psyco: http://psyco.sourceforge.net -.. _`generate Just-In-Time Compilers`: jit/index.html -.. _`JIT Generation in PyPy`: jit/index.html -.. _`implement your own interpreter`: http://morepypy.blogspot.com/2011/04/tutorial-writing-interpreter-with-pypy.html - -.. include:: _ref.txt - +This division between bytecode evaluator and object space gives a lot of +flexibility. One can plug in different :doc:`object spaces ` to get +different or enriched behaviours of the Python objects. diff --git a/pypy/doc/arm.rst b/pypy/doc/arm.rst deleted file mode 100644 --- a/pypy/doc/arm.rst +++ /dev/null @@ -1,166 +0,0 @@ -========================= -Cross-translating for ARM -========================= - - -Here we describe the setup required and the steps needed to follow to translate -an interpreter using the RPython translator to target ARM using a cross -compilation toolchain. - -To translate an RPython program for ARM we can either -translate directly on an ARM device following the normal translation steps. Unfortunately this is not really feasible on most ARM machines. The alternative is to cross-translate using a cross-compilation toolchain. - -To cross-translate we run the translation on a more powerful (usually -x86) machine and generate a binary for ARM using a cross-compiler to compile -the generated C code. There are several constraints when doing this. In -particular we currently only support Linux as translation host and target -platforms (tested on Ubuntu). Also we need a 32-bit environment to run the -translation. This can be done either on a 32bit host or in 32bit chroot. - - -Requirements ------------- - -The tools required to cross translate from a Linux based host to an ARM based Linux target are: - -- A checkout of PyPy (default branch). -- The GCC ARM cross compiler (on Ubuntu it is the ``gcc-arm-linux-gnueabi package``) but other toolchains should also work. -- Scratchbox 2, a cross-compilation engine (``scratchbox2`` Ubuntu package). -- A 32-bit PyPy or Python. -- And the following (or corresponding) packages need to be installed to create an ARM based chroot: - - * ``debootstrap`` - * ``schroot`` - * ``binfmt-support`` - * ``qemu-system`` - * ``qemu-user-static`` - -- The dependencies above are in addition to the ones needed for a regular - translation, `listed here`_. - -.. _`listed here`: getting-started-python.html#translating-the-pypy-python-interpreter - - -Creating a Qemu based ARM chroot --------------------------------- - -First we will need to create a rootfs containing the packages and dependencies -required in order to translate PyPy or other interpreters. We are going to -assume, that the files will be placed in ``/srv/chroot/precise_arm``. - -Create the rootfs by calling: - -:: - - mkdir -p /srv/chroot/precise_arm - qemu-debootstrap --variant=buildd --arch=armel precise /srv/chroot/precise_arm/ http://ports.ubuntu.com/ubuntu-ports/ - -Next, copy the qemu-arm-static binary to the rootfs. - -:: - - cp /usr/bin/qemu-arm-static /srv/chroot/precise_arm/usr/bin/qemu-arm-static - -For easier configuration and management we will create a schroot pointing to -the rootfs. We need to add a configuration block (like the one below) to the -schroot configuration file in /etc/schroot/schroot.conf. - - -:: - - [precise_arm] - directory=/srv/chroot/precise_arm - users=USERNAME - root-users=USERNAME - groups=users - aliases=default - type=directory - - -To verify that everything is working in the chroot, running ``schroot -c -precise_arm`` should start a shell running in the schroot environment using -qemu-arm to execute the ARM binaries. Running ``uname -m`` in the chroot should -yeild a result like ``armv7l``. Showing that we are emulating an ARM system. - -Start the schroot as the user root in order to configure the apt sources and -to install the following packages: - - -:: - - schroot -c precise_arm -u root - echo "deb http://ports.ubuntu.com/ubuntu-ports/ precise main universe restricted" > /etc/apt/sources.list - apt-get update - apt-get install libffi-dev libgc-dev python-dev build-essential libncurses5-dev libbz2-dev - - -Now all dependencies should be in place and we can exit the schroot environment. - - -Configuring scratchbox2 ------------------------ - -To configure the scratchbox we need to cd into the root directory of the rootfs -we created before. From there we can call the sb2 configuration tools which -will take the current directory as the base directory for the scratchbox2 -environment. - -:: - - cd /srv/chroot/precise_arm - sb2-init -c `which qemu-arm` ARM `which arm-linux-gnueabi-gcc` - -This will create a scratchbox2 based environment called ARM that maps calls to -gcc done within the scratchbox to the arm-linux-gnueabi-gcc outside the -scratchbox. Now we should have a working cross compilation toolchain in place -and can start cross-translating programs for ARM. - -Translation ------------ - -Having performed all the preliminary steps we should now be able to cross -translate a program for ARM. You can use this_ minimal -target to test your setup before applying it to a larger project. - -Before starting the translator we need to set two environment variables, so the -translator knows how to use the scratchbox environment. We need to set the -**SB2** environment variable to point to the rootfs and the **SB2OPT** should -contain the command line options for the sb2 command. If our rootfs is in the -folder /srv/chroot/precise_arm and the scratchbox environment is called "ARM", -the variables would be defined as follows. - - -:: - - export SB2=/srv/chroot/precise_arm - export SB2OPT='-t ARM' - -Once this is set, you can call the translator. For example save this file - -:: - - def main(args): - print "Hello World" - return 0 - - def target(*args): - return main, None - -and call the translator - -:: - - pypy ~/path_to_pypy_checkout/rpython/bin/rpython -O1 --platform=arm target.py - -If everything worked correctly this should yield an ARM binary. Running this binary in the ARM chroot or on an ARM device should produce the output ``"Hello World"``. - -To translate the full python pypy interpreter with a jit, you can cd into pypy/goal and call - -:: - - pypy /rpython/bin/rpython -Ojit --platform=arm --gcrootfinder=shadowstack --jit-backend=arm targetpypystandalone.py - -The gcrootfinder option is needed to work around `issue 1377`_ and the jit-backend works around `issue 1376`_ - -.. _`issue 1377`: https://bugs.pypy.org/issue1377 -.. _`issue 1376`: https://bugs.pypy.org/issue1376 diff --git a/pypy/doc/build.rst b/pypy/doc/build.rst new file mode 100644 --- /dev/null +++ b/pypy/doc/build.rst @@ -0,0 +1,170 @@ +Building PyPy from Source +========================= + +For building PyPy, we recommend installing a pre-built PyPy first (see +:doc:`install`). It is possible to build PyPy with CPython, but it will take a +lot longer to run -- depending on your architecture, between two and three +times as long. + +Even when using PyPy to build PyPy, translation is time-consuming -- 30 +minutes on a fast machine -- and RAM-hungry. You will need **at least** 2 GB +of memory on a 32-bit machine and 4GB on a 64-bit machine. + + +Clone the repository +-------------------- + +If you prefer to compile your own PyPy, or if you want to modify it, you +will need to obtain a copy of the sources. This can be done either by +`downloading them from the download page`_ or by checking them out from the +repository using mercurial. We suggest using mercurial if you want to access +the current development. + +.. _downloading them from the download page: http://pypy.org/download.html + +You must issue the following command on your +command line, DOS box, or terminal:: + + hg clone http://bitbucket.org/pypy/pypy pypy + +This will clone the repository and place it into a directory +named ``pypy``, and will get you the PyPy source in ``pypy/pypy`` and +documentation files in ``pypy/pypy/doc``. +We try to ensure that the tip is always stable, but it might +occasionally be broken. You may want to check out `our nightly tests`_: +find a revision (12-chars alphanumeric string, e.g. "963e808156b3") +that passed at least the +``{linux32}`` tests (corresponding to a ``+`` sign on the +line ``success``) and then, in your cloned repository, switch to this revision +using:: + + hg up -r XXXXX + +where XXXXX is the revision id. + +.. _our nightly tests: http://buildbot.pypy.org/summary?branch= + + +Install build-time dependencies +------------------------------- + +To build PyPy on Unix using the C translation backend, you need at least a C +compiler and ``make`` installed. Further, some optional modules have additional +dependencies: + +cffi, ctypes + libffi, pkg-config + +zlib + libz + +bz2 + libbz2 + +lzma (PyPy3 only) + liblzma + +sqlite3 + libsqlite3 + +curses + libncurses + cffi dependencies from above + +pyexpat + libexpat1 + +_ssl + libssl + +Make sure to have these libraries (with development headers) installed before +building PyPy, otherwise the resulting binary will not contain these modules. + +On Debian, this is the command to install all build-time dependencies:: + + apt-get install gcc make libffi-dev pkg-config libz-dev libbz2-dev \ + libsqlite3-dev libncurses-dev libexpat1-dev libssl-dev + +For the optional lzma module on PyPy3 you will also need ``liblzma-dev``. + +On Fedora:: + + yum install gcc make libffi-devel pkgconfig zlib-devel bzip2-devel \ + lib-sqlite3-devel ncurses-devel expat-devel openssl-devel + +For the optional lzma module on PyPy3 you will also need ``xz-devel``. + +On SLES11:: + + zypper install gcc make python-devel pkg-config \ + zlib-devel libopenssl-devel libbz2-devel sqlite3-devel \ + libexpat-devel libffi-devel python-curses + +For the optional lzma module on PyPy3 you will also need ``xz-devel``. + +On Mac OS X, most of these build-time dependencies are installed alongside +the Developer Tools. However, note that in order for the installation to +find them you may need to run:: + + xcode-select --install + + +Run the translation +------------------- + +Translate with JIT:: + + pypy rpython/bin/rpython --opt=jit pypy/goal/targetpypystandalone.py + +Translate without JIT:: + + pypy rpython/bin/rpython --opt=2 pypy/goal/targetpypystandalone.py + +(You can use ``python`` instead of ``pypy`` here, which will take longer +but works too.) + +If everything works correctly this will create an executable ``pypy-c`` in the +current directory. The executable behaves mostly like a normal Python +interpreter (see :doc:`cpython_differences`). + + +.. _translate-pypy: + +Translating with non-standard options +------------------------------------- + +It is possible to have non-standard features enabled for translation, +but they are not really tested any more. Look, for example, at the +:doc:`objspace proxies ` document. + + + +Installation +------------ + +PyPy dynamically finds the location of its libraries depending on the location +of the executable. The directory hierarchy of a typical PyPy installation +looks like this:: + + ./bin/pypy + ./include/ + ./lib_pypy/ + ./lib-python/2.7 + ./site-packages/ + +The hierarchy shown above is relative to a PREFIX directory. PREFIX is +computed by starting from the directory where the executable resides, and +"walking up" the filesystem until we find a directory containing ``lib_pypy`` +and ``lib-python/2.7``. + +To install PyPy system wide on unix-like systems, it is recommended to put the +whole hierarchy alone (e.g. in ``/opt/pypy``) and put a symlink to the +``pypy`` executable into ``/usr/bin`` or ``/usr/local/bin``. + +If the executable fails to find suitable libraries, it will report ``debug: +WARNING: library path not found, using compiled-in sys.path`` and then attempt +to continue normally. If the default path is usable, most code will be fine. +However, the ``sys.prefix`` will be unset and some existing libraries assume +that this is never the case. + + +.. TODO windows diff --git a/pypy/doc/cleanup.rst b/pypy/doc/cleanup.rst deleted file mode 100644 --- a/pypy/doc/cleanup.rst +++ /dev/null @@ -1,11 +0,0 @@ -Old documentation that needs review ------------------------------------ - -.. The following stuff is old (and crufty?), and needs further investigation: - -.. doc-index: This needs merging somehow - -.. toctree:: - - distribution.rst - diff --git a/pypy/doc/coding-guide.rst b/pypy/doc/coding-guide.rst --- a/pypy/doc/coding-guide.rst +++ b/pypy/doc/coding-guide.rst @@ -1,21 +1,17 @@ -==================================== Coding Guide -==================================== +============ .. contents:: This document describes coding requirements and conventions for working with the PyPy code base. Please read it carefully and ask back any questions you might have. The document does not talk -very much about coding style issues. We mostly follow `PEP 8`_ though. +very much about coding style issues. We mostly follow :pep:`8` though. If in doubt, follow the style that is already present in the code base. -.. _`PEP 8`: http://www.python.org/dev/peps/pep-0008/ - -.. _`RPython`: Overview and motivation -======================== +------------------------ We are writing a Python interpreter in Python, using Python's well known ability to step behind the algorithmic problems as a language. At first glance, @@ -25,7 +21,7 @@ CPython vs. PyPy -------------------- +~~~~~~~~~~~~~~~~ Compared to the CPython implementation, Python takes the role of the C Code. We rewrite the CPython interpreter in Python itself. We could @@ -44,20 +40,20 @@ but let's stick with this somewhat canonical approach. -.. _`application-level`: -.. _`interpreter-level`: +.. _application-level: +.. _interpreter-level: Application-level and interpreter-level execution and objects -------------------------------------------------------------- +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Since Python is used for implementing all of our code base, there is a -crucial distinction to be aware of: that between *interpreter-level* objects and +crucial distinction to be aware of: that between *interpreter-level* objects and *application-level* objects. The latter are the ones that you deal with when you write normal python programs. Interpreter-level code, however, cannot invoke operations nor access attributes from application-level objects. You will immediately recognize any interpreter level code in PyPy, because half the variable and object names start with a ``w_``, which -indicates that they are `wrapped`_ application-level values. +indicates that they are `wrapped`_ application-level values. Let's show the difference with a simple example. To sum the contents of two variables ``a`` and ``b``, one would write the simple application-level @@ -80,10 +76,10 @@ interpreting. -.. _`app-preferable`: +.. _app-preferable: -Application level is often preferable -------------------------------------- +Application level is often preferable +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Application-level code is substantially higher-level, and therefore correspondingly easier to write and debug. For example, suppose we want @@ -113,11 +109,11 @@ space.setitem(w_self, w_key, w_value) This interpreter-level implementation looks much more similar to the C -source code. It is still more readable than its C counterpart because -it doesn't contain memory management details and can use Python's native -exception mechanism. +source code. It is still more readable than its C counterpart because +it doesn't contain memory management details and can use Python's native +exception mechanism. -In any case, it should be obvious that the application-level implementation +In any case, it should be obvious that the application-level implementation is definitely more readable, more elegant and more maintainable than the interpreter-level one (and indeed, dict.update is really implemented at applevel in PyPy). @@ -129,10 +125,11 @@ level code is usually preferable. We have an abstraction (called the 'Gateway') which allows the caller of a function to remain ignorant of whether a particular function is implemented at application or -interpreter level. +interpreter level. + Our runtime interpreter is "RPython" ----------------------------------------------- +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ In order to make a C code generator feasible all code on interpreter level has to restrict itself to a subset of the Python language, and we adhere to some @@ -159,7 +156,7 @@ elegant: For the definition of all the opcodes of the Python interpreter, the module ``dis`` is imported and used to initialize our bytecode interpreter. (See ``__initclass__`` in -`pypy/interpreter/pyopcode.py`_). This +:source:`pypy/interpreter/pyopcode.py`). This saves us from adding extra modules to PyPy. The import code is run at startup time, and we are allowed to use the CPython builtin import function. @@ -172,308 +169,13 @@ enables the code generator to emit efficient machine level replacements for pure integer objects, for instance. -RPython -================= - -RPython Definition ------------------- - -RPython is a restricted subset of Python that is amenable to static analysis. -Although there are additions to the language and some things might surprisingly -work, this is a rough list of restrictions that should be considered. Note -that there are tons of special cased restrictions that you'll encounter -as you go. The exact definition is "RPython is everything that our translation -toolchain can accept" :) - -.. _`wrapped object`: coding-guide.html#wrapping-rules - -Flow restrictions -------------------------- - -**variables** - - variables should contain values of at most one type as described in - `Object restrictions`_ at each control flow point, that means for - example that joining control paths using the same variable to - contain both a string and a int must be avoided. It is allowed to - mix None (basically with the role of a null pointer) with many other - types: `wrapped objects`, class instances, lists, dicts, strings, etc. - but *not* with int, floats or tuples. - -**constants** - - all module globals are considered constants. Their binding must not - be changed at run-time. Moreover, global (i.e. prebuilt) lists and - dictionaries are supposed to be immutable: modifying e.g. a global - list will give inconsistent results. However, global instances don't - have this restriction, so if you need mutable global state, store it - in the attributes of some prebuilt singleton instance. - - - -**control structures** - - all allowed, ``for`` loops restricted to builtin types, generators - very restricted. - -**range** - - ``range`` and ``xrange`` are identical. ``range`` does not necessarily create an array, - only if the result is modified. It is allowed everywhere and completely - implemented. The only visible difference to CPython is the inaccessibility - of the ``xrange`` fields start, stop and step. - -**definitions** - - run-time definition of classes or functions is not allowed. - -**generators** - - generators are supported, but their exact scope is very limited. you can't - merge two different generator in one control point. - -**exceptions** - -+ fully supported -+ see below `Exception rules`_ for restrictions on exceptions raised by built-in operations - - -Object restrictions -------------------------- - -We are using - -**integer, float, boolean** - - works. - -**strings** - - a lot of, but not all string methods are supported and those that are - supported, not necesarilly accept all arguments. Indexes can be - negative. In case they are not, then you get slightly more efficient - code if the translator can prove that they are non-negative. When - slicing a string it is necessary to prove that the slice start and - stop indexes are non-negative. There is no implicit str-to-unicode cast - anywhere. Simple string formatting using the ``%`` operator works, as long - as the format string is known at translation time; the only supported - formatting specifiers are ``%s``, ``%d``, ``%x``, ``%o``, ``%f``, plus - ``%r`` but only for user-defined instances. Modifiers such as conversion - flags, precision, length etc. are not supported. Moreover, it is forbidden - to mix unicode and strings when formatting. - -**tuples** - - no variable-length tuples; use them to store or return pairs or n-tuples of - values. Each combination of types for elements and length constitute - a separate and not mixable type. - -**lists** - - lists are used as an allocated array. Lists are over-allocated, so list.append() - is reasonably fast. However, if you use a fixed-size list, the code - is more efficient. Annotator can figure out most of the time that your - list is fixed-size, even when you use list comprehension. - Negative or out-of-bound indexes are only allowed for the - most common operations, as follows: - - - *indexing*: - positive and negative indexes are allowed. Indexes are checked when requested - by an IndexError exception clause. - - - *slicing*: - the slice start must be within bounds. The stop doesn't need to, but it must - not be smaller than the start. All negative indexes are disallowed, except for - the [:-1] special case. No step. Slice deletion follows the same rules. - - - *slice assignment*: - only supports ``lst[x:y] = sublist``, if ``len(sublist) == y - x``. - In other words, slice assignment cannot change the total length of the list, - but just replace items. - - - *other operators*: - ``+``, ``+=``, ``in``, ``*``, ``*=``, ``==``, ``!=`` work as expected. - - - *methods*: - append, index, insert, extend, reverse, pop. The index used in pop() follows - the same rules as for *indexing* above. The index used in insert() must be within - bounds and not negative. - -**dicts** - - dicts with a unique key type only, provided it is hashable. Custom - hash functions and custom equality will not be honored. - Use ``rpython.rlib.objectmodel.r_dict`` for custom hash functions. - - -**list comprehensions** - - May be used to create allocated, initialized arrays. - -**functions** - -+ statically called functions may use defaults and a variable number of - arguments (which may be passed as a list instead of a tuple, so write code - that does not depend on it being a tuple). - -+ dynamic dispatch enforces the use of signatures that are equal for all - possible called function, or at least "compatible enough". This - concerns mainly method calls, when the method is overridden or in any - way given different definitions in different classes. It also concerns - the less common case of explicitly manipulated function objects. - Describing the exact compatibility rules is rather involved (but if you - break them, you should get explicit errors from the rtyper and not - obscure crashes.) - -**builtin functions** - - A number of builtin functions can be used. The precise set can be - found in `rpython/annotator/builtin.py`_ (see ``def builtin_xxx()``). - Some builtin functions may be limited in what they support, though. - - ``int, float, str, ord, chr``... are available as simple conversion - functions. Note that ``int, float, str``... have a special meaning as - a type inside of isinstance only. - -**classes** - -+ methods and other class attributes do not change after startup -+ single inheritance is fully supported -+ use `rpython.rlib.objectmodel.import_from_mixin(M)` in a class - body to copy the whole content of a class `M`. This can be used - to implement mixins: functions and staticmethods are duplicated - (the other class attributes are just copied unmodified). - -+ classes are first-class objects too - -**objects** - - Normal rules apply. The only special methods that are honoured are - ``__init__``, ``__del__``, ``__len__``, ``__getitem__``, ``__setitem__``, - ``__getslice__``, ``__setslice__``, and ``__iter__``. To handle slicing, - ``__getslice__`` and ``__setslice__`` must be used; using ``__getitem__`` and - ``__setitem__`` for slicing isn't supported. Additionally, using negative - indices for slicing is still not support, even when using ``__getslice__``. - -This layout makes the number of types to take care about quite limited. - - -Integer Types -------------------------- - -While implementing the integer type, we stumbled over the problem that -integers are quite in flux in CPython right now. Starting with Python 2.4, -integers mutate into longs on overflow. In contrast, we need -a way to perform wrap-around machine-sized arithmetic by default, while still -being able to check for overflow when we need it explicitly. Moreover, we need -a consistent behavior before and after translation. - -We use normal integers for signed arithmetic. It means that before -translation we get longs in case of overflow, and after translation we get a -silent wrap-around. Whenever we need more control, we use the following -helpers (which live in `rpython/rlib/rarithmetic.py`_): - -**ovfcheck()** - - This special function should only be used with a single arithmetic operation - as its argument, e.g. ``z = ovfcheck(x+y)``. Its intended meaning is to - perform the given operation in overflow-checking mode. - - At run-time, in Python, the ovfcheck() function itself checks the result - and raises OverflowError if it is a ``long``. But the code generators use - ovfcheck() as a hint: they replace the whole ``ovfcheck(x+y)`` expression - with a single overflow-checking addition in C. - -**intmask()** - - This function is used for wrap-around arithmetic. It returns the lower bits - of its argument, masking away anything that doesn't fit in a C "signed long int". - Its purpose is, in Python, to convert from a Python ``long`` that resulted from a - previous operation back to a Python ``int``. The code generators ignore - intmask() entirely, as they are doing wrap-around signed arithmetic all the time - by default anyway. (We have no equivalent of the "int" versus "long int" - distinction of C at the moment and assume "long ints" everywhere.) - -**r_uint** - - In a few cases (e.g. hash table manipulation), we need machine-sized unsigned - arithmetic. For these cases there is the r_uint class, which is a pure - Python implementation of word-sized unsigned integers that silently wrap - around. ("word-sized" and "machine-sized" are used equivalently and mean - the native size, which you get using "unsigned long" in C.) - The purpose of this class (as opposed to helper functions as above) - is consistent typing: both Python and the annotator will propagate r_uint - instances in the program and interpret all the operations between them as - unsigned. Instances of r_uint are special-cased by the code generators to - use the appropriate low-level type and operations. - Mixing of (signed) integers and r_uint in operations produces r_uint that - means unsigned results. To convert back from r_uint to signed integers, use - intmask(). - - -Exception rules ---------------------- - -Exceptions are by default not generated for simple cases.:: - - #!/usr/bin/python - - lst = [1,2,3,4,5] - item = lst[i] # this code is not checked for out-of-bound access - - try: - item = lst[i] - except IndexError: - # complain - -Code with no exception handlers does not raise exceptions (after it has been -translated, that is. When you run it on top of CPython, it may raise -exceptions, of course). By supplying an exception handler, you ask for error -checking. Without, you assure the system that the operation cannot fail. -This rule does not apply to *function calls*: any called function is -assumed to be allowed to raise any exception. - -For example:: - - x = 5.1 - x = x + 1.2 # not checked for float overflow - try: - x = x + 1.2 - except OverflowError: - # float result too big - -But:: - - z = some_function(x, y) # can raise any exception - try: - z = some_other_function(x, y) - except IndexError: - # only catches explicitly-raised IndexErrors in some_other_function() - # other exceptions can be raised, too, and will not be caught here. - -The ovfcheck() function described above follows the same rule: in case of -overflow, it explicitly raise OverflowError, which can be caught anywhere. - -Exceptions explicitly raised or re-raised will always be generated. - -PyPy is debuggable on top of CPython ------------------------------------- - -PyPy has the advantage that it is runnable on standard -CPython. That means, we can run all of PyPy with all exception -handling enabled, so we might catch cases where we failed to -adhere to our implicit assertions. - -.. _`wrapping rules`: -.. _`wrapped`: - - +.. _wrapped: Wrapping rules -============== +-------------- Wrapping ---------- +~~~~~~~~ PyPy is made of Python source code at two levels: there is on the one hand *application-level code* that looks like normal Python code, and that @@ -500,7 +202,7 @@ structure. For example, an application-level Python ``list`` -is implemented by the `standard object space`_ as an +is implemented by the :ref:`standard object space ` as an instance of ``W_ListObject``, which has an instance attribute ``wrappeditems`` (an interpreter-level list which contains the application-level list's items as wrapped objects). @@ -509,7 +211,7 @@ Naming conventions ------------------- +~~~~~~~~~~~~~~~~~~ * ``space``: the object space is only visible at interpreter-level code, where it is by convention passed around by the name @@ -529,14 +231,14 @@ Operations on ``w_xxx`` ------------------------ +~~~~~~~~~~~~~~~~~~~~~~~ The core bytecode interpreter considers wrapped objects as black boxes. It is not allowed to inspect them directly. The allowed operations are all implemented on the object space: they are called ``space.xxx()``, where ``xxx`` is a standard operation name (``add``, ``getattr``, ``call``, ``eq``...). They are documented in the -`object space document`_. +:ref:`object space document `. A short warning: **don't do** ``w_x == w_y`` or ``w_x is w_y``! rationale for this rule is that there is no reason that two @@ -548,12 +250,11 @@ use ``space.is_true(space.is_(w_x, w_y))`` or better ``space.is_w(w_x, w_y)``. -.. _`object space document`: objspace.html#interface -.. _`applevel-exceptions`: +.. _applevel-exceptions: Application-level exceptions ----------------------------- +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Interpreter-level code can use exceptions freely. However, all application-level exceptions are represented as an @@ -585,10 +286,10 @@ instances of subclasses. -.. _`modules`: +.. _modules: Modules in PyPy -=============== +--------------- Modules visible from application programs are imported from interpreter or application level files. PyPy reuses almost all python @@ -597,18 +298,19 @@ because they rely on implementation details of CPython. If we don't just modify an original CPython module but need to rewrite -it from scratch we put it into `lib_pypy/`_ as a pure application level +it from scratch we put it into :source:`lib_pypy/` as a pure application level module. When we need access to interpreter-level objects we put the module into -`pypy/module`_. Such modules use a `mixed module mechanism`_ +:source:`pypy/module`. Such modules use a `mixed module mechanism`_ which makes it convenient to use both interpreter- and application-level parts for the implementation. Note that there is no extra facility for pure-interpreter level modules, you just write a mixed module and leave the application-level part empty. + Determining the location of a module implementation ---------------------------------------------------- +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ You can interactively find out where a module comes from, when running py.py. here are examples for the possible locations:: @@ -626,8 +328,9 @@ '/home/hpk/pypy-dist/lib-python/2.7/os.py' >>>> + Module directories / Import order ---------------------------------- +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Here is the order in which PyPy looks up Python modules: @@ -650,10 +353,11 @@ The modified CPython library. -.. _`modify modules`: + +.. _modify modules: Modifying a CPython library module or regression test -------------------------------------------------------- +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Although PyPy is very compatible with CPython we sometimes need to change modules contained in our copy of the standard library, @@ -665,34 +369,31 @@ to see what is changed we have a branch called `vendor/stdlib` wich contains the unmodified cpython stdlib -.. _`mixed module mechanism`: -.. _`mixed modules`: + +.. _mixed module mechanism: +.. _mixed-modules: Implementing a mixed interpreter/application level Module ---------------------------------------------------------- +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ If a module needs to access PyPy's interpreter level then it is implemented as a mixed module. -Mixed modules are directories in `pypy/module`_ with an `__init__.py` +Mixed modules are directories in :source:`pypy/module` with an `__init__.py` file containing specifications where each name in a module comes from. Only specified names will be exported to a Mixed Module's applevel namespace. -Sometimes it is necessary to really write some functions in C (or -whatever target language). See `rffi`_ and `external functions -documentation`_ for details. The latter approach is cumbersome and -being phased out and former has currently quite a few rough edges. +Sometimes it is necessary to really write some functions in C (or whatever +target language). See :ref:`rffi ` details. -.. _`rffi`: rffi.html -.. _`external functions documentation`: translation.html#extfunccalls application level definitions -............................. +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Application level specifications are found in the `appleveldefs` dictionary found in ``__init__.py`` files of directories in ``pypy/module``. -For example, in `pypy/module/__builtin__/__init__.py`_ you find the following +For example, in :source:`pypy/module/__builtin__/__init__.py` you find the following entry specifying where ``__builtin__.locals`` comes from:: ... @@ -703,12 +404,13 @@ interpreted at application level and the wrapped function value for ``locals`` will be extracted accordingly. + interpreter level definitions -............................. +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Interpreter level specifications are found in the ``interpleveldefs`` dictionary found in ``__init__.py`` files of directories in ``pypy/module``. -For example, in `pypy/module/__builtin__/__init__.py`_ the following +For example, in :source:`pypy/module/__builtin__/__init__.py` the following entry specifies where ``__builtin__.len`` comes from:: ... @@ -724,7 +426,7 @@ return space.len(w_obj) Exposed interpreter level functions usually take a ``space`` argument -and some wrapped values (see `wrapping rules`_) . +and some wrapped values (see `Wrapping rules`_) . You can also use a convenient shortcut in ``interpleveldefs`` dictionaries: namely an expression in parentheses to specify an interpreter level @@ -743,37 +445,40 @@ --withoutmod-mymodule (the latter being the default)) for py.py and translate.py. + Testing modules in ``lib_pypy/`` --------------------------------- +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -You can go to the `pypy/module/test_lib_pypy/`_ directory and invoke the testing tool +You can go to the :source:`pypy/module/test_lib_pypy/` directory and invoke the testing tool ("py.test" or "python ../../pypy/test_all.py") to run tests against the -lib_pypy hierarchy. Note, that tests in `pypy/module/test_lib_pypy/`_ are allowed +lib_pypy hierarchy. Note, that tests in :source:`pypy/module/test_lib_pypy/` are allowed and encouraged to let their tests run at interpreter level although -`lib_pypy/`_ modules eventually live at PyPy's application level. +:source:`lib_pypy/` modules eventually live at PyPy's application level. This allows us to quickly test our python-coded reimplementations against CPython. + Testing modules in ``pypy/module`` ----------------------------------- +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Simply change to ``pypy/module`` or to a subdirectory and `run the tests as usual`_. Testing modules in ``lib-python`` ------------------------------------ +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ In order to let CPython's regression tests run against PyPy -you can switch to the `lib-python/`_ directory and run +you can switch to the :source:`lib-python/` directory and run the testing tool in order to start compliance tests. (XXX check windows compatibility for producing test reports). + Naming conventions and directory layout -=========================================== +--------------------------------------- Directory and File Naming -------------------------- +~~~~~~~~~~~~~~~~~~~~~~~~~ - directories/modules/namespaces are always **lowercase** @@ -786,8 +491,9 @@ - keep filenames concise and completion-friendly. + Naming of python objects ------------------------- +~~~~~~~~~~~~~~~~~~~~~~~~ - class names are **CamelCase** @@ -803,8 +509,9 @@ includes w_self. Don't use ``w_`` in application level python only code. + Committing & Branching to the repository ------------------------------------------------------ +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - write good log messages because several people are reading the diffs. @@ -815,22 +522,23 @@ ``try1`` doesn't already exists) you should do:: hg branch try1 - + The branch will be recorded in the repository only after a commit. To switch back to the default branch:: - + hg update default - + For further details use the help or refer to the `official wiki`_:: - + hg help branch -.. _`official wiki`: http://mercurial.selenic.com/wiki/Branch +.. _official wiki: http://mercurial.selenic.com/wiki/Branch -.. _`using development tracker`: + +.. _using-development-tracker: Using the development bug/feature tracker -========================================= +----------------------------------------- We have a `development tracker`_, based on Richard Jones' `roundup`_ application. You can file bugs, @@ -838,15 +546,14 @@ for the next milestone, both from an E-Mail and from a web interface. -.. _`development tracker`: https://bugs.pypy.org/ -.. _`roundup`: http://roundup.sourceforge.net/ +.. _development tracker: https://bugs.pypy.org/ +.. _roundup: http://roundup.sourceforge.net/ -.. _`testing in PyPy`: -.. _`test-design`: +.. _testing: Testing in PyPy -=============== +--------------- Our tests are based on the `py.test`_ tool which lets you write unittests without boilerplate. All tests of modules @@ -859,12 +566,11 @@ - **Application Level tests**. They run at application level which means that they look like straight python code but they are interpreted by PyPy. -.. _`standard object space`: objspace.html#standard-object-space -.. _`objectspace`: objspace.html -.. _`py.test`: http://pytest.org/ +.. _py.test: http://pytest.org/ + Interpreter level tests ------------------------ +~~~~~~~~~~~~~~~~~~~~~~~ You can write test functions and methods like this:: @@ -880,8 +586,9 @@ module global level and use plain 'assert' statements thanks to the usage of the `py.test`_ tool. + Application Level tests ------------------------ +~~~~~~~~~~~~~~~~~~~~~~~ For testing the conformance and well-behavedness of PyPy it is often sufficient to write "normal" application-level @@ -916,7 +623,8 @@ assert self.d["a"] == 1 assert self.d["b"] == 2 -.. _`run the tests as usual`: + +.. _run the tests as usual: Another possibility is to use cls.space.appexec, for example:: @@ -959,7 +667,7 @@ Command line tool test_all --------------------------- +~~~~~~~~~~~~~~~~~~~~~~~~~~ You can run almost all of PyPy's tests by invoking:: @@ -969,8 +677,9 @@ located in the ``py/bin/`` directory. For switches to modify test execution pass the ``-h`` option. + Coverage reports ----------------- +~~~~~~~~~~~~~~~~ In order to get coverage reports the `pytest-cov`_ plugin is included. it adds some extra requirements ( coverage_ and `cov-core`_ ) @@ -978,12 +687,13 @@ python test_all.py --cov file_or_direcory_to_cover file_or_directory -.. _`pytest-cov`: http://pypi.python.org/pypi/pytest-cov -.. _`coverage`: http://pypi.python.org/pypi/coverage -.. _`cov-core`: http://pypi.python.org/pypi/cov-core +.. _pytest-cov: http://pypi.python.org/pypi/pytest-cov +.. _coverage: http://pypi.python.org/pypi/coverage +.. _cov-core: http://pypi.python.org/pypi/cov-core + Test conventions ----------------- +~~~~~~~~~~~~~~~~ - adding features requires adding appropriate tests. (It often even makes sense to first write the tests so that you are sure that they @@ -993,29 +703,28 @@ which contain unit tests. Such scripts can usually be executed directly or are collectively run by pypy/test_all.py -.. _`change documentation and website`: + +.. _change documentation and website: Changing documentation and website -================================== +---------------------------------- documentation/website files in your local checkout ---------------------------------------------------- +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Most of the PyPy's documentation is kept in `pypy/doc`. You can simply edit or add '.rst' files which contain ReST-markuped files. Here is a `ReST quickstart`_ but you can also just look at the existing documentation and see how things work. -.. _`ReST quickstart`: http://docutils.sourceforge.net/docs/user/rst/quickref.html - Note that the web site of http://pypy.org/ is maintained separately. For now it is in the repository https://bitbucket.org/pypy/pypy.org +.. _ReST quickstart: http://docutils.sourceforge.net/docs/user/rst/quickref.html + + Automatically test documentation/website changes ------------------------------------------------- - -.. _`sphinx home page`: -.. _`sphinx`: http://sphinx.pocoo.org/ +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ We automatically check referential integrity and ReST-conformance. In order to run the tests you need sphinx_ installed. Then go to the local checkout @@ -1035,5 +744,4 @@ which will check that remote URLs are reachable. - -.. include:: _ref.txt +.. _sphinx: http://sphinx.pocoo.org/ diff --git a/pypy/doc/conf.py b/pypy/doc/conf.py --- a/pypy/doc/conf.py +++ b/pypy/doc/conf.py @@ -40,9 +40,9 @@ # Add any Sphinx extension module names here, as strings. They can be extensions # coming with Sphinx (named 'sphinx.ext.*') or your custom ones. -extensions = ['sphinx.ext.autodoc', 'sphinx.ext.doctest', 'sphinx.ext.intersphinx', - 'sphinx.ext.todo', 'sphinx.ext.ifconfig', 'sphinx.ext.graphviz', - 'pypyconfig'] +extensions = ['sphinx.ext.autodoc', 'sphinx.ext.doctest', 'sphinx.ext.extlinks', + 'sphinx.ext.intersphinx', 'sphinx.ext.todo', 'sphinx.ext.ifconfig', + 'sphinx.ext.graphviz', 'pypyconfig'] # Add any paths that contain templates here, relative to this directory. templates_path = ['_templates'] @@ -215,7 +215,8 @@ # Example configuration for intersphinx: refer to the Python standard library. -intersphinx_mapping = {'http://docs.python.org/': None} +intersphinx_mapping = {'python': ('http://docs.python.org/2.7', None), + 'rpython': ('http://rpython.readthedocs.org/en/latest/', None)} # -- Options for manpage output------------------------------------------------- @@ -225,3 +226,4 @@ u'The PyPy Project', 1) ] +extlinks = {'source': ('https://bitbucket.org/pypy/pypy/src/default/%s', '')} diff --git a/pypy/doc/config/confrest.py b/pypy/doc/config/confrest.py deleted file mode 100644 --- a/pypy/doc/config/confrest.py +++ /dev/null @@ -1,62 +0,0 @@ -from pypy.doc.confrest import * -from pypy.config.makerestdoc import make_cmdline_overview -from pypy.config import pypyoption -from rpython.config.config import Config -from rpython.config import translationoption - - -all_optiondescrs = [pypyoption.pypy_optiondescription, - translationoption.translation_optiondescription, - ] -start_to_descr = dict([(descr._name, descr) for descr in all_optiondescrs]) - -class PyPyPage(PyPyPage): - def fill(self): - super(PyPyPage, self).fill() - self.menubar[:] = html.div( - html.a("general documentation", href="../index.html", - class_="menu"), " ", - html.a("config index", href="index.html", - class_="menu"), " ", - html.a("command-line overview", href="commandline.html", - class_="menu"), " ", - " ", id="menubar") - -class Project(Project): - stylesheet = "../style.css" - title = "PyPy Configuration" - prefix_title = "PyPy Configuration" - Page = PyPyPage - - def get_content(self, txtpath, encoding): - if txtpath.basename == "commandline.rst": - result = [] - for line in txtpath.read().splitlines(): - if line.startswith('.. GENERATE:'): - start = line[len('.. GENERATE:'):].strip() - descr = start_to_descr[start] - line = make_cmdline_overview(descr, title=False).text() - result.append(line) - return "\n".join(result) - fullpath = txtpath.purebasename - start = fullpath.split(".")[0] - path = fullpath.rsplit(".", 1)[0] - basedescr = start_to_descr.get(start) - if basedescr is None: - return txtpath.read() - if fullpath.count(".") == 0: - descr = basedescr - path = "" - else: - conf = Config(basedescr) - subconf, step = conf._cfgimpl_get_home_by_path( - fullpath.split(".", 1)[1]) - descr = getattr(subconf._cfgimpl_descr, step) - text = unicode(descr.make_rest_doc(path).text()) - if txtpath.check(file=True): - content = txtpath.read() - if content: - text += "\nDescription\n===========" - return u"%s\n\n%s" % (text, unicode(txtpath.read(), encoding)) - return text - diff --git a/pypy/doc/config/generate.py b/pypy/doc/config/generate.py --- a/pypy/doc/config/generate.py +++ b/pypy/doc/config/generate.py @@ -1,6 +1,5 @@ import py from pypy.config import pypyoption, makerestdoc -from pypy.doc.config.confrest import all_optiondescrs from rpython.config import translationoption, config all_optiondescrs = [pypyoption.pypy_optiondescription, diff --git a/pypy/doc/config/objspace.std.builtinshortcut.txt b/pypy/doc/config/objspace.std.builtinshortcut.txt deleted file mode 100644 --- a/pypy/doc/config/objspace.std.builtinshortcut.txt +++ /dev/null @@ -1,5 +0,0 @@ -A shortcut speeding up primitive operations between built-in types. - -This is a space-time trade-off: at the moment, this option makes a -translated pypy-c executable bigger by about 1.7 MB. (This can probably -be improved with careful analysis.) diff --git a/pypy/doc/config/objspace.std.multimethods.txt b/pypy/doc/config/objspace.std.multimethods.txt deleted file mode 100644 --- a/pypy/doc/config/objspace.std.multimethods.txt +++ /dev/null @@ -1,8 +0,0 @@ -Choose the multimethod implementation. - -* ``doubledispatch`` turns - a multimethod call into a sequence of normal method calls. - -* ``mrd`` uses a technique known as Multiple Row Displacement - which precomputes a few compact tables of numbers and - function pointers. diff --git a/pypy/doc/configuration.rst b/pypy/doc/configuration.rst --- a/pypy/doc/configuration.rst +++ b/pypy/doc/configuration.rst @@ -1,4 +1,3 @@ -============================= PyPy's Configuration Handling ============================= @@ -6,16 +5,17 @@ hand the necessary options to where they are actually used and even more annoying to add new options. To circumvent these problems configuration management was introduced. There all the necessary options are stored in a -configuration object, which is available nearly everywhere in the `RPython +configuration object, which is available nearly everywhere in the `RPython toolchain`_ and in the standard interpreter so that adding new options becomes trivial. Options are organized into a tree. Configuration objects can be created in different ways, there is support for creating an optparse command line parser automatically. -_`RPython toolchain`: translation.html +.. _RPython toolchain: https://rpython.readthedocs.org/ + Main Assumption -=============== +--------------- Configuration objects are produced at the entry points and handed down to where they are actually used. This keeps configuration local but available @@ -24,7 +24,7 @@ API Details -=========== +----------- The handling of options is split into two parts: the description of which options are available, what their possible values and defaults are and how they @@ -40,8 +40,9 @@ option group. The parts of the full name of the option are separated by dots: e.g. ``config.translation.thread``. + Description of Options ----------------------- +~~~~~~~~~~~~~~~~~~~~~~ All the constructors take a ``name`` and a ``doc`` argument as first arguments to give the option or option group a name and to document it. Most constructors @@ -70,6 +71,7 @@ ``children`` is a list of option descriptions (including ``OptionDescription`` instances for nested namespaces). + ``ChoiceOption`` ++++++++++++++++ @@ -81,10 +83,11 @@ ``requires`` is a dictionary mapping values to lists of of two-element tuples. + ``BoolOption`` ++++++++++++++ -Represents a choice between ``True`` and ``False``. +Represents a choice between ``True`` and ``False``. ``__init__(self, name, doc, default=None, requires=None, suggests=None, cmdline=DEFAULT, negation=True)`` ``default`` specifies the default value of the option. ``requires`` is @@ -103,7 +106,6 @@ Represents a choice of an integer. ``__init__(self, name, doc, default=None, cmdline=DEFAULT)`` - ``FloatOption`` @@ -112,7 +114,6 @@ Represents a choice of a floating point number. ``__init__(self, name, doc, default=None, cmdline=DEFAULT)`` - ``StrOption`` @@ -121,12 +122,10 @@ Represents the choice of a string. From noreply at buildbot.pypy.org Sat Nov 1 21:48:46 2014 From: noreply at buildbot.pypy.org (mattip) Date: Sat, 1 Nov 2014 21:48:46 +0100 (CET) Subject: [pypy-commit] pypy default: merge heads Message-ID: <20141101204846.89AA11C026A@cobra.cs.uni-duesseldorf.de> Author: mattip Branch: Changeset: r74325:eb7acf87bbaf Date: 2014-11-01 22:48 +0200 http://bitbucket.org/pypy/pypy/changeset/eb7acf87bbaf/ Log: merge heads diff --git a/pypy/objspace/std/formatting.py b/pypy/objspace/std/formatting.py --- a/pypy/objspace/std/formatting.py +++ b/pypy/objspace/std/formatting.py @@ -1,15 +1,15 @@ -""" -String formatting routines. -""" +"""String formatting routines""" import sys -from pypy.interpreter.error import OperationError, oefmt + from rpython.rlib import jit -from rpython.rlib.rfloat import formatd, DTSF_ALT, isnan, isinf +from rpython.rlib.rarithmetic import INT_MAX +from rpython.rlib.rfloat import DTSF_ALT, formatd, isnan, isinf from rpython.rlib.rstring import StringBuilder, UnicodeBuilder from rpython.rlib.unroll import unrolling_iterable -from rpython.rlib.rarithmetic import INT_MAX from rpython.tool.sourcetools import func_with_new_name +from pypy.interpreter.error import OperationError, oefmt + class BaseStringFormatter(object): def __init__(self, space, values_w, w_valuedict): diff --git a/pypy/objspace/std/objectobject.py b/pypy/objspace/std/objectobject.py --- a/pypy/objspace/std/objectobject.py +++ b/pypy/objspace/std/objectobject.py @@ -1,7 +1,10 @@ +"""The builtin object type implementation""" + from pypy.interpreter.baseobjspace import W_Root from pypy.interpreter.error import OperationError, oefmt from pypy.interpreter.gateway import applevel, interp2app, unwrap_spec -from pypy.interpreter.typedef import GetSetProperty, default_identity_hash, TypeDef +from pypy.interpreter.typedef import ( + GetSetProperty, TypeDef, default_identity_hash) from pypy.objspace.descroperation import Object @@ -25,7 +28,7 @@ else: args = getnewargs() if not isinstance(args, tuple): - raise TypeError, "__getnewargs__ should return a tuple" + raise TypeError("__getnewargs__ should return a tuple") try: getstate = obj.__getstate__ @@ -46,15 +49,8 @@ else: state = getstate() - if isinstance(obj, list): - listitems = iter(obj) - else: - listitems = None - - if isinstance(obj, dict): - dictitems = obj.iteritems() - else: - dictitems = None + listitems = iter(obj) if isinstance(obj, list) else None + dictitems = obj.iteritems() if isinstance(obj, dict) else None import copy_reg newobj = copy_reg.__newobj__ @@ -74,7 +70,7 @@ import copy_reg slotnames = copy_reg._slotnames(cls) if not isinstance(slotnames, list) and slotnames is not None: - raise TypeError, "copy_reg._slotnames didn't return a list or None" + raise TypeError("copy_reg._slotnames didn't return a list or None") return slotnames ''', filename=__file__) @@ -94,14 +90,13 @@ # don't allow arguments if the default object.__init__() is about # to be called w_type = _precheck_for_new(space, w_type) - w_parentinit, w_ignored = w_type.lookup_where('__init__') + w_parentinit, _ = w_type.lookup_where('__init__') if w_parentinit is space.w_object: try: __args__.fixedunpack(0) except ValueError: - raise OperationError(space.w_TypeError, - space.wrap("default __new__ takes " - "no parameters")) + raise oefmt(space.w_TypeError, + "default __new__ takes no parameters") if w_type.is_abstract(): _abstract_method_error(space, w_type) w_obj = space.allocate_instance(W_ObjectObject, w_type) @@ -120,8 +115,8 @@ try: __args__.fixedunpack(0) except ValueError: - raise OperationError(space.w_TypeError, - space.wrap("object.__init__() takes no parameters")) + raise oefmt(space.w_TypeError, + "object.__init__() takes no parameters") def descr_get___class__(space, w_obj): @@ -135,11 +130,12 @@ "__class__ must be set to new-style class, not '%T' " "object", w_newcls) if not w_newcls.is_heaptype(): - raise OperationError(space.w_TypeError, - space.wrap("__class__ assignment: only for heap types")) + raise oefmt(space.w_TypeError, + "__class__ assignment: only for heap types") w_oldcls = space.type(w_obj) assert isinstance(w_oldcls, W_TypeObject) - if w_oldcls.get_full_instance_layout() == w_newcls.get_full_instance_layout(): + if (w_oldcls.get_full_instance_layout() == + w_newcls.get_full_instance_layout()): w_obj.setclass(space, w_newcls) else: raise oefmt(space.w_TypeError, @@ -167,8 +163,8 @@ w_type = space.type(w_obj) w_impl = w_type.lookup("__repr__") if w_impl is None: - raise OperationError(space.w_TypeError, # can it really occur? - space.wrap("operand does not support unary str")) + # can it really occur? + raise oefmt(space.w_TypeError, "operand does not support unary str") return space.get_and_call_function(w_impl, w_obj) diff --git a/pypy/objspace/std/util.py b/pypy/objspace/std/util.py --- a/pypy/objspace/std/util.py +++ b/pypy/objspace/std/util.py @@ -1,6 +1,7 @@ -from pypy.interpreter.error import oefmt, OperationError from rpython.rlib.rstring import InvalidBaseError +from pypy.interpreter.error import OperationError, oefmt + IDTAG_INT = 1 IDTAG_LONG = 3 diff --git a/rpython/annotator/description.py b/rpython/annotator/description.py --- a/rpython/annotator/description.py +++ b/rpython/annotator/description.py @@ -1,6 +1,7 @@ from __future__ import absolute_import -import types, py -from rpython.annotator.signature import enforce_signature_args, enforce_signature_return +import types +from rpython.annotator.signature import ( + enforce_signature_args, enforce_signature_return) from rpython.flowspace.model import Constant, FunctionGraph from rpython.flowspace.bytecode import cpython_code_signature from rpython.annotator.argument import rawshape, ArgErr @@ -15,10 +16,10 @@ 'd1~d2 if d1 and d2 might be called at the same call site'. """ normalized = False - modified = True + modified = True def __init__(self, desc): - self.descs = { desc: True } + self.descs = {desc: True} self.calltables = {} # see calltable_lookup_row() self.total_calltable_size = 0 @@ -29,7 +30,7 @@ for shape, table in other.calltables.items(): for row in table: self.calltable_add_row(shape, row) - absorb = update # UnionFind API + absorb = update # UnionFind API def calltable_lookup_row(self, callshape, row): # this code looks up a table of which graph to @@ -68,7 +69,7 @@ self.descs.update(other.descs) self.read_locations.update(other.read_locations) self.attrs.update(other.attrs) - absorb = update # UnionFind API + absorb = update # UnionFind API def get_s_value(self, attrname): try: @@ -97,7 +98,7 @@ def __init__(self, desc): from rpython.annotator.model import s_ImpossibleValue - self.descs = { desc: True } + self.descs = {desc: True} self.read_locations = {} # set of position_keys self.s_value = s_ImpossibleValue # union of possible values @@ -106,7 +107,7 @@ self.descs.update(other.descs) self.read_locations.update(other.read_locations) self.s_value = unionof(self.s_value, other.s_value) - absorb = update # UnionFind API + absorb = update # UnionFind API def get_s_value(self, attrname): return self.s_value @@ -164,9 +165,9 @@ def bind_under(self, classdef, name): return self + @staticmethod def simplify_desc_set(descs): pass - simplify_desc_set = staticmethod(simplify_desc_set) class NoStandardGraph(Exception): @@ -218,7 +219,7 @@ [graph] = self._cache.values() relax_sig_check = getattr(self.pyobj, "relax_sig_check", False) if (graph.signature != self.signature or - graph.defaults != self.defaults) and not relax_sig_check: + graph.defaults != self.defaults) and not relax_sig_check: raise NoStandardGraph(self) return graph @@ -229,7 +230,7 @@ def nameof(thing): if isinstance(thing, str): return thing - elif hasattr(thing, '__name__'): # mostly types and functions + elif hasattr(thing, '__name__'): # mostly types and functions return thing.__name__ elif hasattr(thing, 'name') and isinstance(thing.name, str): return thing.name # mostly ClassDescs @@ -240,7 +241,7 @@ if key is not None and alt_name is None: postfix = valid_identifier(nameof(key)) - alt_name = "%s__%s"%(self.name, postfix) + alt_name = "%s__%s" % (self.name, postfix) graph = self.buildgraph(alt_name, builder) self._cache[key] = graph return graph @@ -268,7 +269,7 @@ def specialize(self, inputcells, op=None): if (op is None and - getattr(self.bookkeeper, "position_key", None) is not None): + getattr(self.bookkeeper, "position_key", None) is not None): _, block, i = self.bookkeeper.position_key op = block.operations[i] if self.specializer is None: @@ -280,15 +281,16 @@ enforceargs = getattr(self.pyobj, '_annenforceargs_', None) signature = getattr(self.pyobj, '_signature_', None) if enforceargs and signature: - raise Exception("%r: signature and enforceargs cannot both be used" % (self,)) + raise Exception("%r: signature and enforceargs cannot both be " + "used" % (self,)) if enforceargs: if not callable(enforceargs): from rpython.annotator.signature import Sig enforceargs = Sig(*enforceargs) self.pyobj._annenforceargs_ = enforceargs - enforceargs(self, inputcells) # can modify inputcells in-place + enforceargs(self, inputcells) # can modify inputcells in-place if signature: - enforce_signature_args(self, signature[0], inputcells) # mutates inputcells + enforce_signature_args(self, signature[0], inputcells) # mutates inputcells if getattr(self.pyobj, '_annspecialcase_', '').endswith("call_location"): return self.specializer(self, inputcells, op) else: @@ -309,7 +311,8 @@ if signature: sigresult = enforce_signature_return(self, signature[1], result) if sigresult is not None: - self.bookkeeper.annotator.addpendingblock(graph, graph.returnblock, [sigresult]) + self.bookkeeper.annotator.addpendingblock( + graph, graph.returnblock, [sigresult]) result = sigresult # Some specializations may break the invariant of returning # annotations that are always more general than the previous time. @@ -325,12 +328,13 @@ None, # selfclassdef name) + @staticmethod def consider_call_site(bookkeeper, family, descs, args, s_result, op): shape = rawshape(args) row = FunctionDesc.row_to_consider(descs, args, op) family.calltable_add_row(shape, row) - consider_call_site = staticmethod(consider_call_site) + @staticmethod def variant_for_call_site(bookkeeper, family, descs, args, op): shape = rawshape(args) bookkeeper.enter(None) @@ -340,11 +344,11 @@ bookkeeper.leave() index = family.calltable_lookup_row(shape, row) return shape, index - variant_for_call_site = staticmethod(variant_for_call_site) def rowkey(self): return self + @staticmethod def row_to_consider(descs, args, op): # see comments in CallFamily from rpython.annotator.model import s_ImpossibleValue @@ -356,7 +360,6 @@ desc.pycall(enlist, args, s_ImpossibleValue, op) assert row return row - row_to_consider = staticmethod(row_to_consider) def get_s_signatures(self, shape): family = self.getcallfamily() @@ -385,6 +388,9 @@ return s_sigs +def is_mixin(cls): + return cls.__dict__.get('_mixin_', False) + NODEFAULT = object() class ClassDesc(Desc): @@ -394,95 +400,92 @@ settled = False _detect_invalid_attrs = None - def __init__(self, bookkeeper, pyobj=None, + def __init__(self, bookkeeper, cls, name=None, basedesc=None, classdict=None, specialize=None): - super(ClassDesc, self).__init__(bookkeeper, pyobj) + super(ClassDesc, self).__init__(bookkeeper, cls) if name is None: - name = pyobj.__module__ + '.' + pyobj.__name__ + name = cls.__module__ + '.' + cls.__name__ self.name = name self.basedesc = basedesc if classdict is None: classdict = {} # populated below self.classdict = classdict # {attr: Constant-or-Desc} if specialize is None: - specialize = pyobj.__dict__.get('_annspecialcase_', '') + specialize = cls.__dict__.get('_annspecialcase_', '') self.specialize = specialize self._classdefs = {} - if pyobj is not None: - assert pyobj.__module__ != '__builtin__' - cls = pyobj - base = object - baselist = list(cls.__bases__) + if is_mixin(cls): + raise AnnotatorError("cannot use directly the class %r because " + "it is a _mixin_" % (cls,)) - if cls.__dict__.get('_mixin_', False): - raise AnnotatorError("cannot use directly the class %r because " - "it is a _mixin_" % (cls,)) + assert cls.__module__ != '__builtin__' + baselist = list(cls.__bases__) - # special case: skip BaseException in Python 2.5, and pretend - # that all exceptions ultimately inherit from Exception instead - # of BaseException (XXX hack) - if cls is Exception: - baselist = [] - elif baselist == [py.builtin.BaseException]: - baselist = [Exception] + # special case: skip BaseException, and pretend + # that all exceptions ultimately inherit from Exception instead + # of BaseException (XXX hack) + if cls is Exception: + baselist = [] + elif baselist == [BaseException]: + baselist = [Exception] - mixins_before = [] - mixins_after = [] - for b1 in baselist: - if b1 is object: - continue - if b1.__dict__.get('_mixin_', False): - if base is object: - mixins_before.append(b1) - else: - mixins_after.append(b1) + mixins_before = [] + mixins_after = [] + base = object + for b1 in baselist: + if b1 is object: + continue + if is_mixin(b1): + if base is object: + mixins_before.append(b1) else: - assert base is object, ("multiple inheritance only supported " - "with _mixin_: %r" % (cls,)) - base = b1 - if mixins_before and mixins_after: - raise Exception("unsupported: class %r has mixin bases both" - " before and after the regular base" % (self,)) - self.add_mixins(mixins_after, check_not_in=base) - self.add_mixins(mixins_before) - self.add_sources_for_class(cls) + mixins_after.append(b1) + else: + assert base is object, ("multiple inheritance only supported " + "with _mixin_: %r" % (cls,)) + base = b1 + if mixins_before and mixins_after: + raise Exception("unsupported: class %r has mixin bases both" + " before and after the regular base" % (self,)) + self.add_mixins(mixins_after, check_not_in=base) + self.add_mixins(mixins_before) + self.add_sources_for_class(cls) - if base is not object: - self.basedesc = bookkeeper.getdesc(base) + if base is not object: + self.basedesc = bookkeeper.getdesc(base) - if '_settled_' in cls.__dict__: - self.settled = bool(cls.__dict__['_settled_']) + if '_settled_' in cls.__dict__: + self.settled = bool(cls.__dict__['_settled_']) - if '__slots__' in cls.__dict__ or '_attrs_' in cls.__dict__: - attrs = {} - for decl in ('__slots__', '_attrs_'): - decl = cls.__dict__.get(decl, []) - if isinstance(decl, str): - decl = (decl,) - decl = dict.fromkeys(decl) - attrs.update(decl) - if self.basedesc is not None: - if self.basedesc.all_enforced_attrs is None: - raise Exception("%r has slots or _attrs_, " - "but not its base class" - % (pyobj,)) - attrs.update(self.basedesc.all_enforced_attrs) - self.all_enforced_attrs = attrs + if '__slots__' in cls.__dict__ or '_attrs_' in cls.__dict__: + attrs = {} + for decl in ('__slots__', '_attrs_'): + decl = cls.__dict__.get(decl, []) + if isinstance(decl, str): + decl = (decl,) + decl = dict.fromkeys(decl) + attrs.update(decl) + if self.basedesc is not None: + if self.basedesc.all_enforced_attrs is None: + raise Exception("%r has slots or _attrs_, " + "but not its base class" % (cls,)) + attrs.update(self.basedesc.all_enforced_attrs) + self.all_enforced_attrs = attrs - if (self.is_builtin_exception_class() and + if (self.is_builtin_exception_class() and self.all_enforced_attrs is None): - from rpython.annotator import classdef - if self.pyobj not in classdef.FORCE_ATTRIBUTES_INTO_CLASSES: - self.all_enforced_attrs = [] # no attribute allowed + from rpython.annotator import classdef + if cls not in classdef.FORCE_ATTRIBUTES_INTO_CLASSES: + self.all_enforced_attrs = [] # no attribute allowed def add_source_attribute(self, name, value, mixin=False): if isinstance(value, types.FunctionType): # for debugging if not hasattr(value, 'class_'): - value.class_ = self.pyobj # remember that this is really a method + value.class_ = self.pyobj if self.specialize: # make a custom funcdesc that specializes on its first # argument (i.e. 'self'). @@ -507,7 +510,7 @@ if isinstance(value, staticmethod) and mixin: # make a new copy of staticmethod func = value.__get__(42) - value = staticmethod(func_with_new_name(func, func.__name__)) + value = staticmethod(func_with_new_name(func, func.__name__)) if type(value) in MemberDescriptorTypes: # skip __slots__, showing up in the class as 'member' objects @@ -539,8 +542,8 @@ add(check_not_in) # for base in reversed(mro): - assert base.__dict__.get("_mixin_", False), ("Mixin class %r has non" - "mixin base class %r" % (mixins, base)) + assert is_mixin(base), ( + "Mixin class %r has non mixin base class %r" % (mixins, base)) for name, value in base.__dict__.items(): if name in skip: continue @@ -557,18 +560,18 @@ try: return self._classdefs[key] except KeyError: - from rpython.annotator.classdef import ClassDef, FORCE_ATTRIBUTES_INTO_CLASSES + from rpython.annotator.classdef import ( + ClassDef, FORCE_ATTRIBUTES_INTO_CLASSES) classdef = ClassDef(self.bookkeeper, self) self.bookkeeper.classdefs.append(classdef) self._classdefs[key] = classdef # forced attributes - if self.pyobj is not None: - cls = self.pyobj - if cls in FORCE_ATTRIBUTES_INTO_CLASSES: - for name, s_value in FORCE_ATTRIBUTES_INTO_CLASSES[cls].items(): - classdef.generalize_attr(name, s_value) - classdef.find_attribute(name).modified(classdef) + cls = self.pyobj + if cls in FORCE_ATTRIBUTES_INTO_CLASSES: + for name, s_value in FORCE_ATTRIBUTES_INTO_CLASSES[cls].items(): + classdef.generalize_attr(name, s_value) + classdef.find_attribute(name).modified(classdef) # register all class attributes as coming from this ClassDesc # (as opposed to prebuilt instances) @@ -635,8 +638,7 @@ return s_instance def is_exception_class(self): - return self.pyobj is not None and issubclass(self.pyobj, - py.builtin.BaseException) + return issubclass(self.pyobj, BaseException) def is_builtin_exception_class(self): if self.is_exception_class(): @@ -703,14 +705,12 @@ def find_source_for(self, name): if name in self.classdict: return self - if self.pyobj is not None: - # check whether in the case the classdesc corresponds to a real class - # there is a new attribute - cls = self.pyobj - if name in cls.__dict__: - self.add_source_attribute(name, cls.__dict__[name]) - if name in self.classdict: - return self + # check whether there is a new attribute + cls = self.pyobj + if name in cls.__dict__: + self.add_source_attribute(name, cls.__dict__[name]) + if name in self.classdict: + return self return None def maybe_return_immutable_list(self, attr, s_result): @@ -718,7 +718,7 @@ # either 'lst[*]' or 'lst?[*]' # should really return an immutable list as a result. Implemented # by changing the result's annotation (but not, of course, doing an - # actual copy in the rtyper). Tested in rpython.rtyper.test.test_rlist, + # actual copy in the rtyper). Tested in rpython.rtyper.test.test_rlist, # test_immutable_list_out_of_instance. if self._detect_invalid_attrs and attr in self._detect_invalid_attrs: raise Exception("field %r was migrated to %r from a subclass in " @@ -730,7 +730,7 @@ while cdesc is not None: if '_immutable_fields_' in cdesc.classdict: if (search1 in cdesc.classdict['_immutable_fields_'].value or - search2 in cdesc.classdict['_immutable_fields_'].value): + search2 in cdesc.classdict['_immutable_fields_'].value): s_result.listdef.never_resize() s_copy = s_result.listdef.offspring() s_copy.listdef.mark_as_immutable() @@ -746,6 +746,7 @@ cdesc = cdesc.basedesc return s_result # common case + @staticmethod def consider_call_site(bookkeeper, family, descs, args, s_result, op): from rpython.annotator.model import SomeInstance, SomePBC, s_None if len(descs) == 1: @@ -792,7 +793,6 @@ initfamily = initdescs[0].getcallfamily() MethodDesc.consider_call_site(bookkeeper, initfamily, initdescs, args, s_None, op) - consider_call_site = staticmethod(consider_call_site) def getallbases(self): desc = self @@ -800,6 +800,7 @@ yield desc desc = desc.basedesc + @staticmethod def getcommonbase(descs): commondesc = descs[0] for desc in descs[1:]: @@ -809,7 +810,6 @@ desc = desc.basedesc commondesc = desc return commondesc - getcommonbase = staticmethod(getcommonbase) def rowkey(self): return self @@ -868,7 +868,7 @@ from rpython.annotator.model import SomeInstance if self.selfclassdef is None: raise Exception("calling %r" % (self,)) - s_instance = SomeInstance(self.selfclassdef, flags = self.flags) + s_instance = SomeInstance(self.selfclassdef, flags=self.flags) args = args.prepend(s_instance) return self.funcdesc.pycall(schedule, args, s_previous_result, op) @@ -896,6 +896,7 @@ # FunctionDesc to use as a key in that family. return self.funcdesc + @staticmethod def simplify_desc_set(descs): # Some hacking needed to make contains() happy on SomePBC: if the # set of MethodDescs contains some "redundant" ones, i.e. ones that @@ -942,7 +943,6 @@ if cdef1 is not cdef2 and cdef1.issubclass(cdef2): descs.remove(desc1) break - simplify_desc_set = staticmethod(simplify_desc_set) def new_or_old_class(c): @@ -960,7 +960,7 @@ self._read_attribute = read_attribute self.attrcache = {} self.knowntype = new_or_old_class(pyobj) - assert bool(pyobj), "__nonzero__ unsupported on frozen PBC %r" %(pyobj,) + assert bool(pyobj), "__nonzero__ unsupported on frozen PBC %r" % (pyobj,) def has_attribute(self, attr): if attr in self.attrcache: diff --git a/rpython/annotator/test/test_description.py b/rpython/annotator/test/test_description.py --- a/rpython/annotator/test/test_description.py +++ b/rpython/annotator/test/test_description.py @@ -1,4 +1,4 @@ -from rpython.annotator.description import ClassDesc +from rpython.annotator.description import ClassDesc, is_mixin class FakeBookkeeper: def __init__(self): @@ -20,3 +20,13 @@ dC = bk.getdesc(C) dD = bk.getdesc(D) assert ClassDesc.getcommonbase([dC, dD]) is dA + +def test_is_mixin(): + class Mixin1(object): + _mixin_ = True + + class A(Mixin1): + pass + + assert is_mixin(Mixin1) + assert not is_mixin(A) From noreply at buildbot.pypy.org Sun Nov 2 09:34:50 2014 From: noreply at buildbot.pypy.org (arigo) Date: Sun, 2 Nov 2014 09:34:50 +0100 (CET) Subject: [pypy-commit] stmgc hashtable: hashtables now shed their old tables, their old entries, and possibly Message-ID: <20141102083450.319D41C093D@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: hashtable Changeset: r1490:997d05467167 Date: 2014-11-02 09:35 +0100 http://bitbucket.org/pypy/stmgc/changeset/997d05467167/ Log: hashtables now shed their old tables, their old entries, and possibly shrink during major GCs. diff --git a/c7/stm/gcpage.c b/c7/stm/gcpage.c --- a/c7/stm/gcpage.c +++ b/c7/stm/gcpage.c @@ -344,6 +344,8 @@ LIST_APPEND(mark_objects_to_trace, obj); } +#define TRACE_FOR_MAJOR_COLLECTION (&mark_record_trace) + static void mark_trace(object_t *obj, char *segment_base) { assert(list_is_empty(mark_objects_to_trace)); @@ -352,7 +354,7 @@ /* trace into the object (the version from 'segment_base') */ struct object_s *realobj = (struct object_s *)REAL_ADDRESS(segment_base, obj); - stmcb_trace(realobj, &mark_record_trace); + stmcb_trace(realobj, TRACE_FOR_MAJOR_COLLECTION); if (list_is_empty(mark_objects_to_trace)) break; diff --git a/c7/stm/hashtable.c b/c7/stm/hashtable.c --- a/c7/stm/hashtable.c +++ b/c7/stm/hashtable.c @@ -63,6 +63,7 @@ struct stm_hashtable_s { stm_hashtable_table_t *table; stm_hashtable_table_t initial_table; + uint64_t additions; }; @@ -78,6 +79,7 @@ stm_hashtable_t *hashtable = malloc(sizeof(stm_hashtable_t)); assert(hashtable); hashtable->table = &hashtable->initial_table; + hashtable->additions = 0; init_table(&hashtable->initial_table, INITIAL_HASHTABLE_SIZE); return hashtable; } @@ -95,42 +97,17 @@ } } -#if 0 -static void stm_compact_hashtable(stm_hashtable_t *hashtable) +static bool _stm_was_read_by_anybody(object_t *obj) { - stm_hashtable_table_t *table = hashtable->table; - assert(!IS_EVEN(table->resize_counter)); - - if (table != &hashtable->initial_table) { - uintptr_t rc = hashtable->initial_table.resize_counter; - while (1) { - assert(IS_EVEN(rc)); - assert(rc != RESIZING_LOCK); - - stm_hashtable_table_t *old_table = (stm_hashtable_table_t *)rc; - if (old_table == table) - break; - rc = old_table->resize_counter; - free(old_table); - } - hashtable->initial_table.resize_counter = (uintptr_t)table; + long i; + for (i = 1; i <= NB_SEGMENTS; i++) { + char *remote_base = get_segment_base(i); + uint8_t remote_version = get_segment(i)->transaction_read_version; + if (was_read_remote(remote_base, obj, remote_version)) + return true; } - if (table->resize_counter < table->mask * 3) { - uintptr_t j, mask = table->mask; - uintptr_t rc = table->resize_counter; - for (j = 0; j <= mask; j++) { - stm_hashtable_entry_t *e = table->items[j]; - if (e != NULL && e->object == NULL) { - if (!_stm_was_read_by_anybody(e)) { - table->items[j] = NULL; - rc += 6; - } - } - } - table->resize_counter = rc; - } + return false; } -#endif #define VOLATILE_HASHTABLE(p) ((volatile stm_hashtable_t *)(p)) #define VOLATILE_TABLE(p) ((volatile stm_hashtable_table_t *)(p)) @@ -158,6 +135,45 @@ } } +static void _stm_rehash_hashtable(stm_hashtable_t *hashtable, + uintptr_t biggercount, + bool remove_unread) +{ + size_t size = (offsetof(stm_hashtable_table_t, items) + + biggercount * sizeof(stm_hashtable_entry_t *)); + stm_hashtable_table_t *biggertable = malloc(size); + assert(biggertable); // XXX + + stm_hashtable_table_t *table = hashtable->table; + table->resize_counter = (uintptr_t)biggertable; + /* ^^^ this unlocks the table by writing a non-zero value to + table->resize_counter, but the new value is a pointer to the + new bigger table, so IS_EVEN() is still true */ + + init_table(biggertable, biggercount); + + uintptr_t j, mask = table->mask; + uintptr_t rc = biggertable->resize_counter; + for (j = 0; j <= mask; j++) { + stm_hashtable_entry_t *entry = table->items[j]; + if (entry == NULL) + continue; + if (remove_unread) { + if (entry->object == NULL && + !_stm_was_read_by_anybody((object_t *)entry)) + continue; + } + _insert_clean(biggertable, entry); + rc -= 6; + } + biggertable->resize_counter = rc; + + write_fence(); /* make sure that 'biggertable' is valid here, + and make sure 'table->resize_counter' is updated + ('table' must be immutable from now on). */ + VOLATILE_HASHTABLE(hashtable)->table = biggertable; +} + static stm_hashtable_entry_t *_stm_hashtable_lookup(object_t *hashtableobj, stm_hashtable_t *hashtable, uintptr_t index) @@ -210,7 +226,12 @@ rc, RESIZING_LOCK)) { goto restart; } - /* we now have the lock. Check that 'table->items[i]' is still NULL, + /* we now have the lock. The only table with a non-even value of + 'resize_counter' should be the last one in the chain, so if we + succeeded in locking it, check this. */ + assert(table == hashtable->table); + + /* Check that 'table->items[i]' is still NULL, i.e. hasn't been populated under our feet. */ if (table->items[i] != NULL) { @@ -248,6 +269,7 @@ } write_fence(); /* make sure 'entry' is fully initialized here */ table->items[i] = entry; + hashtable->additions += 1; write_fence(); /* make sure 'table->items' is written here */ VOLATILE_TABLE(table)->resize_counter = rc - 6; /* unlock */ return entry; @@ -260,30 +282,7 @@ biggercount *= 4; else biggercount *= 2; - size_t size = (offsetof(stm_hashtable_table_t, items) - + biggercount * sizeof(stm_hashtable_entry_t *)); - stm_hashtable_table_t *biggertable = malloc(size); - assert(biggertable); // XXX - table->resize_counter = (uintptr_t)biggertable; - /* unlock, but put the new table, so IS_EVEN() is still true */ - - init_table(biggertable, biggercount); - - uintptr_t j; - rc = biggertable->resize_counter; - for (j = 0; j <= mask; j++) { - entry = table->items[j]; - if (entry != NULL) { - _insert_clean(biggertable, entry); - rc -= 6; - } - } - biggertable->resize_counter = rc; - - write_fence(); /* make sure that 'biggertable' is valid here, - and make sure 'table->resize_counter' is updated - ('table' must be immutable from now on). */ - VOLATILE_HASHTABLE(hashtable)->table = biggertable; + _stm_rehash_hashtable(hashtable, biggercount, /*remove_unread=*/false); goto restart; } } @@ -309,14 +308,56 @@ e->object = nvalue; } +static void _stm_compact_hashtable(stm_hashtable_t *hashtable) +{ + stm_hashtable_table_t *table = hashtable->table; + assert(!IS_EVEN(table->resize_counter)); + + if (hashtable->additions * 4 > table->mask) { + hashtable->additions = 0; + uintptr_t initial_rc = (table->mask + 1) * 4 + 1; + uintptr_t num_entries_times_6 = initial_rc - table->resize_counter; + uintptr_t count = INITIAL_HASHTABLE_SIZE; + while (count * 4 < num_entries_times_6) + count *= 2; + /* sanity-check: 'num_entries_times_6 < initial_rc', and so 'count' + can never grow larger than the current table size. */ + assert(count <= table->mask + 1); + + _stm_rehash_hashtable(hashtable, count, /*remove_unread=*/true); + } + + table = hashtable->table; + assert(!IS_EVEN(table->resize_counter)); + + if (table != &hashtable->initial_table) { + uintptr_t rc = hashtable->initial_table.resize_counter; + while (1) { + assert(IS_EVEN(rc)); + assert(rc != RESIZING_LOCK); + + stm_hashtable_table_t *old_table = (stm_hashtable_table_t *)rc; + if (old_table == table) + break; + rc = old_table->resize_counter; + free(old_table); + } + hashtable->initial_table.resize_counter = (uintptr_t)table; + } +} + void stm_hashtable_tracefn(stm_hashtable_t *hashtable, void trace(object_t **)) { + if (trace == TRACE_FOR_MAJOR_COLLECTION) + _stm_compact_hashtable(hashtable); + stm_hashtable_table_t *table; table = VOLATILE_HASHTABLE(hashtable)->table; uintptr_t j, mask = table->mask; for (j = 0; j <= mask; j++) { - stm_hashtable_entry_t **pentry = &table->items[j]; + stm_hashtable_entry_t *volatile *pentry; + pentry = &VOLATILE_TABLE(table)->items[j]; if (*pentry != NULL) { trace((object_t **)pentry); } From noreply at buildbot.pypy.org Sun Nov 2 16:53:14 2014 From: noreply at buildbot.pypy.org (arigo) Date: Sun, 2 Nov 2014 16:53:14 +0100 (CET) Subject: [pypy-commit] pypy.org extradoc: Try to expand a bit the "features" page, and change its title to "What Message-ID: <20141102155314.57F6F1C100F@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: extradoc Changeset: r551:0434ccc41610 Date: 2014-11-02 16:52 +0100 http://bitbucket.org/pypy/pypy.org/changeset/0434ccc41610/ Log: Try to expand a bit the "features" page, and change its title to "What is PyPy?" diff --git a/archive.html b/archive.html --- a/archive.html +++ b/archive.html @@ -38,7 +38,7 @@
- +
diff --git a/compat.html b/compat.html --- a/compat.html +++ b/compat.html @@ -38,7 +38,7 @@
- +
diff --git a/contact.html b/contact.html --- a/contact.html +++ b/contact.html @@ -38,7 +38,7 @@
- +
diff --git a/download.html b/download.html --- a/download.html +++ b/download.html @@ -38,7 +38,7 @@
- +
diff --git a/features.html b/features.html --- a/features.html +++ b/features.html @@ -1,7 +1,7 @@ - PyPy - Features + PyPy - What is PyPy? @@ -38,13 +38,16 @@
- +
-

Features

+

What is PyPy?

+

PyPy is a replacement for CPython. It is built using the RPython +language that was co-developed with it. The main reason to use it +instead of CPython is speed: it runs generally faster (see next section).

PyPy 2.4 implements Python 2.7.8 and runs on Intel x86 (IA-32) , x86_64 and ARM platforms, with PPC being stalled. It supports all of the core language, passing the Python test suite @@ -53,10 +56,54 @@ standard library modules. For known differences with CPython, see our compatibility page.

If you are interested in helping to move forward, see our howtohelp page.

+

We also have a beta release of PyPy3 which implements Python 3.2.5. +It runs on the same platforms as above.

+
+

The main features of PyPy:

+

Speed

Our main executable comes with a Just-in-Time compiler. It is -really fast in running most benchmarks. Try it out!

+really fast in running most benchmarks – including very large and +complicated Python applications, not just 10-liners.

+

There are two cases that you should be aware where PyPy will not be +able to speed up your code:

+
    +
  • Short-running processes: if it doesn't run for at least a few seconds, +then the JIT compiler won't have enough time to warm up.
    • +
  • If all the time is spent in run-time libraries (i.e. in C functions), +and not actually running Python code, the JIT compiler will not help.
    • +
+

So the case where PyPy works best is when executing long-running +programs where a significant fraction of the time is spent executing +Python code. This is the case covered by the majority of our +benchmarks, but not all of them – the goal of PyPy is to get speed +but still support (ideally) any Python program.

+
+
+

Memory usage

+

Memory-hungry Python programs (several hundreds of MBs or more) might +end up taking less space than they do in CPython. It is not always +the case, though, as it depends on a lot of details. Also note that +the baseline is higher than CPython's.

+
+
+

Stackless

+

Support for Stackless and greenlets are now integrated in the normal +PyPy. More detailed information is available here.

+
+
+

Other features

+

PyPy has many secondary features and semi-independent +projects. We will mention here:

+
    +

  • Other languages: we also implemented other languages that makes +use of our RPython toolchain: Prolog (almost complete), as +well as Smalltalk, JavaScript, Io, Scheme and Gameboy.

    +

    There is also a Ruby implementation called Topaz and a PHP implementation +called HippyVM.

    +
    • +

Sandboxing

@@ -65,6 +112,8 @@ try to limit language features considered “unsafe”. Instead we replace all calls to external libraries (C or platform) with a stub that communicates with an external process handling the policy.

+

Please be aware that it is a prototype only. It needs work to become +more complete, and you are welcome to help.

To run the sandboxed process, you need pypy-sandbox. You also need to get the full sources (step 1 only). Run:

@@ -85,24 +134,6 @@
 
To read more about its features, try pypy_interact.py --help or go to
 our documentation site.
-
-

Stackless

-

Support for Stackless and greenlets are now integrated in the normal -PyPy. More detailed information is available here.

-
-
-

Other features

-

PyPy has many secondary features and semi-independent -projects. We will mention here:

-
    -

  • Other languages: we also implemented other languages that makes -use of our RPython toolchain: Prolog (almost complete), as -well as Smalltalk, JavaScript, Io, Scheme and Gameboy.

    -

    There is also a Ruby implementation called Topaz and a PHP implementation -called HippyVM.

    -
    • -
-
diff --git a/howtohelp.html b/howtohelp.html --- a/howtohelp.html +++ b/howtohelp.html @@ -38,7 +38,7 @@
- +
diff --git a/index.html b/index.html --- a/index.html +++ b/index.html @@ -38,7 +38,7 @@
- +
diff --git a/numpydonate.html b/numpydonate.html --- a/numpydonate.html +++ b/numpydonate.html @@ -38,7 +38,7 @@
- +
diff --git a/people.html b/people.html --- a/people.html +++ b/people.html @@ -38,7 +38,7 @@
- +
diff --git a/performance.html b/performance.html --- a/performance.html +++ b/performance.html @@ -38,7 +38,7 @@
- +
diff --git a/py3donate.html b/py3donate.html --- a/py3donate.html +++ b/py3donate.html @@ -38,7 +38,7 @@
- +
diff --git a/source/_layouts/site.genshi b/source/_layouts/site.genshi --- a/source/_layouts/site.genshi +++ b/source/_layouts/site.genshi @@ -5,7 +5,7 @@ section_data = { 'code': [ ('Home', 'index.html'), - ('Features', 'features.html'), + ('What is PyPy?', 'features.html'), ('Download', 'download.html'), ('Compatibility', 'compat.html'), ('Performance', 'performance.html'), diff --git a/source/features.txt b/source/features.txt --- a/source/features.txt +++ b/source/features.txt @@ -1,11 +1,15 @@ --- layout: page -title: Features +title: What is PyPy? --- -PyPy features +What is PyPy? =========================================================== +PyPy is a replacement for CPython. It is built using the RPython +language that was co-developed with it. The main reason to use it +instead of CPython is speed: it runs generally faster (see next section). + **PyPy 2.4** implements **Python 2.7.8** and runs on Intel `x86 (IA-32)`_ , `x86_64`_ and `ARM`_ platforms, with PPC being stalled. It supports all of the core language, passing the Python test suite @@ -16,6 +20,9 @@ If you are interested in helping to move forward, see our `howtohelp`_ page. +We also have a beta release of **PyPy3** which implements **Python 3.2.5**. +It runs on the same platforms as above. + .. _`compatibility`: compat.html .. _`x86 (IA-32)`: http://en.wikipedia.org/wiki/IA-32 .. _`x86_64`: http://en.wikipedia.org/wiki/X86_64 @@ -23,15 +30,69 @@ .. _`howtohelp`: howtohelp.html + +The main features of PyPy: +-------------------------- + Speed ----- Our `main executable`_ comes with a Just-in-Time compiler. It is -`really fast`_ in running most benchmarks. `Try it out!`_ +`really fast`_ in running most benchmarks --- including very large and +complicated Python applications, not just 10-liners. + +There are two cases that you should be aware where PyPy will *not* be +able to speed up your code: + +* Short-running processes: if it doesn't run for at least a few seconds, + then the JIT compiler won't have enough time to warm up. + +* If all the time is spent in run-time libraries (i.e. in C functions), + and not actually running Python code, the JIT compiler will not help. + +So the case where PyPy works best is when executing long-running +programs where a significant fraction of the time is spent executing +Python code. This is the case covered by the majority of `our +benchmarks`_, but not all of them --- the goal of PyPy is to get speed +but still support (ideally) any Python program. .. _`main executable`: download.html#with-a-jit-compiler -.. _`Try it out!`: download.html#with-a-jit-compiler .. _`really fast`: http://speed.pypy.org/ +.. _`our benchmarks`: http://speed.pypy.org/ + + +Memory usage +-------------------------- + +Memory-hungry Python programs (several hundreds of MBs or more) might +end up taking less space than they do in CPython. It is not always +the case, though, as it depends on a lot of details. Also note that +the baseline is higher than CPython's. + + +Stackless +-------------------------- + +Support for Stackless_ and greenlets are now integrated in the normal +PyPy. More detailed information is available here__. + +.. _Stackless: http://www.stackless.com/ +.. __: http://doc.pypy.org/en/latest/stackless.html + + +Other features +--------------------------------------- + +PyPy has many secondary features and semi-independent +projects. We will mention here: + +* **Other languages:** we also implemented other languages that makes + use of our RPython toolchain: Prolog_ (almost complete), as + well as Smalltalk_, JavaScript_, Io_, Scheme_ and Gameboy_. + + There is also a Ruby implementation called Topaz_ and a PHP implementation + called HippyVM_. + Sandboxing -------------------- @@ -42,6 +103,9 @@ calls to external libraries (C or platform) with a stub that communicates with an external process handling the policy. +Please be aware that it is a **prototype** only. It needs work to become +more complete, and you are welcome to help. + To run the sandboxed process, you need `pypy-sandbox`_. You also need to get the `full sources`_ (step 1 only). Run:: @@ -68,28 +132,6 @@ .. _`our documentation site`: http://pypy.readthedocs.org/en/latest/sandbox.html -Stackless --------------------------- - -Support for Stackless_ and greenlets are now integrated in the normal -PyPy. More detailed information is available here__. - -.. _Stackless: http://www.stackless.com/ -.. __: http://doc.pypy.org/en/latest/stackless.html - - -Other features ---------------------------------------- - -PyPy has many secondary features and semi-independent -projects. We will mention here: - -* **Other languages:** we also implemented other languages that makes - use of our RPython toolchain: Prolog_ (almost complete), as - well as Smalltalk_, JavaScript_, Io_, Scheme_ and Gameboy_. - - There is also a Ruby implementation called Topaz_ and a PHP implementation - called HippyVM_. .. _`the cli-jit branch`: https://bitbucket.org/pypy/pypy/src/cli-jit diff --git a/sponsor.html b/sponsor.html --- a/sponsor.html +++ b/sponsor.html @@ -38,7 +38,7 @@
- +
diff --git a/success.html b/success.html --- a/success.html +++ b/success.html @@ -38,7 +38,7 @@
- +
diff --git a/tmdonate.html b/tmdonate.html --- a/tmdonate.html +++ b/tmdonate.html @@ -38,7 +38,7 @@
- +
diff --git a/tmdonate2.html b/tmdonate2.html --- a/tmdonate2.html +++ b/tmdonate2.html @@ -38,7 +38,7 @@
- +
From noreply at buildbot.pypy.org Sun Nov 2 17:09:50 2014 From: noreply at buildbot.pypy.org (arigo) Date: Sun, 2 Nov 2014 17:09:50 +0100 (CET) Subject: [pypy-commit] stmgc hashtable: Change the big test function into a class Message-ID: <20141102160950.50EFA1C362C@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: hashtable Changeset: r1491:7a21812ef019 Date: 2014-11-02 17:10 +0100 http://bitbucket.org/pypy/stmgc/changeset/7a21812ef019/ Log: Change the big test function into a class diff --git a/c7/test/test_hashtable.py b/c7/test/test_hashtable.py --- a/c7/test/test_hashtable.py +++ b/c7/test/test_hashtable.py @@ -17,7 +17,7 @@ raise Conflict -class TestHashtable(BaseTest): +class BaseTestHashtable(BaseTest): def setup_method(self, meth): BaseTest.setup_method(self, meth) @@ -49,6 +49,9 @@ self.seen_hashtables += 1 return h + +class TestHashtable(BaseTestHashtable): + def test_empty(self): self.start_transaction() h = self.allocate_hashtable() @@ -184,106 +187,115 @@ assert htget(h, 1) == lp1 stm_major_collect() # to get rid of the hashtable object + +class TestRandomHashtable(BaseTestHashtable): + + def setup_method(self, meth): + BaseTestHashtable.setup_method(self, meth) + self.values = [] + self.mirror = None + self.roots = [] + + def push_roots(self): + assert self.roots is None + self.roots = [] + for k, hitems in self.mirror.items(): + assert lib._get_type_id(k) == 421419 + for key, value in hitems.items(): + assert lib._get_type_id(value) < 1000 + self.push_root(value) + self.roots.append(key) + self.push_root(k) + self.roots.append(None) + for v in self.values: + self.push_root(v) + self.mirror = None + + def pop_roots(self): + assert self.mirror is None + for i in reversed(range(len(self.values))): + self.values[i] = self.pop_root() + assert stm_get_char(self.values[i]) == chr((i + 1) & 255) + self.mirror = {} + for r in reversed(self.roots): + obj = self.pop_root() + if r is None: + assert lib._get_type_id(obj) == 421419 + self.mirror[obj] = curhitems = {} + else: + assert lib._get_type_id(obj) < 1000 + curhitems[r] = obj + self.roots = None + def test_random_single_thread(self): import random - values = [] - mirror = {} - roots = [] - def push_roots(): - assert roots == [] - for k, hitems in mirror.items(): - assert lib._get_type_id(k) == 421419 - for key, value in hitems.items(): - assert lib._get_type_id(value) < 1000 - self.push_root(value) - roots.append(key) - self.push_root(k) - roots.append(None) - for v in values: - self.push_root(v) - mirror.clear() - # - def pop_roots(): - assert mirror == {} - for i in reversed(range(len(values))): - values[i] = self.pop_root() - assert stm_get_char(values[i]) == chr((i + 1) & 255) - for r in reversed(roots): - obj = self.pop_root() - if r is None: - assert lib._get_type_id(obj) == 421419 - mirror[obj] = curhitems = {} - else: - assert lib._get_type_id(obj) < 1000 - curhitems[r] = obj - del roots[:] # for i in range(100): print "start_transaction" self.start_transaction() - pop_roots() + self.pop_roots() for j in range(10): r = random.random() if r < 0.05: h = self.allocate_hashtable() print "allocate_hashtable ->", h - mirror[h] = {} + self.mirror[h] = {} elif r < 0.10: print "stm_minor_collect" - push_roots() + self.push_roots() stm_minor_collect() - pop_roots() + self.pop_roots() elif r < 0.11: print "stm_major_collect" - push_roots() + self.push_roots() stm_major_collect() - pop_roots() + self.pop_roots() elif r < 0.5: - if not mirror: continue - h = random.choice(mirror.keys()) - if not mirror[h]: continue - key = random.choice(mirror[h].keys()) - value = mirror[h][key] + if not self.mirror: continue + h = random.choice(self.mirror.keys()) + if not self.mirror[h]: continue + key = random.choice(self.mirror[h].keys()) + value = self.mirror[h][key] print "htget(%r, %r) == %r" % (h, key, value) - push_roots() + self.push_roots() self.push_root(value) result = htget(h, key) value = self.pop_root() assert result == value - pop_roots() + self.pop_roots() elif r < 0.6: - if not mirror: continue - h = random.choice(mirror.keys()) + if not self.mirror: continue + h = random.choice(self.mirror.keys()) key = random.randrange(0, 40) - if key in mirror[h]: continue + if key in self.mirror[h]: continue print "htget(%r, %r) == NULL" % (h, key) - push_roots() + self.push_roots() assert htget(h, key) == ffi.NULL - pop_roots() + self.pop_roots() elif r < 0.63: - if not mirror: continue - h, _ = mirror.popitem() + if not self.mirror: continue + h, _ = self.mirror.popitem() print "popped", h elif r < 0.75: obj = stm_allocate(32) - values.append(obj) - stm_set_char(obj, chr(len(values) & 255)) + self.values.append(obj) + stm_set_char(obj, chr(len(self.values) & 255)) else: - if not mirror or not values: continue - h = random.choice(mirror.keys()) + if not self.mirror or not self.values: continue + h = random.choice(self.mirror.keys()) key = random.randrange(0, 32) - value = random.choice(values) + value = random.choice(self.values) print "htset(%r, %r, %r)" % (h, key, value) - push_roots() + self.push_roots() tl = self.tls[self.current_thread] htset(h, key, value, tl) - pop_roots() - mirror[h][key] = value - push_roots() + self.pop_roots() + self.mirror[h][key] = value + self.push_roots() print "commit_transaction" self.commit_transaction() # self.start_transaction() self.become_inevitable() - pop_roots() + self.pop_roots() stm_major_collect() # to get rid of the hashtable objects From noreply at buildbot.pypy.org Sun Nov 2 17:53:23 2014 From: noreply at buildbot.pypy.org (arigo) Date: Sun, 2 Nov 2014 17:53:23 +0100 (CET) Subject: [pypy-commit] stmgc hashtable: A test running on multiple threads. Still missing: exchange Message-ID: <20141102165323.69FFB1C100F@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: hashtable Changeset: r1492:df03cb28ee6b Date: 2014-11-02 17:52 +0100 http://bitbucket.org/pypy/stmgc/changeset/df03cb28ee6b/ Log: A test running on multiple threads. Still missing: exchange of data across threads diff --git a/c7/test/test_hashtable.py b/c7/test/test_hashtable.py --- a/c7/test/test_hashtable.py +++ b/c7/test/test_hashtable.py @@ -195,6 +195,7 @@ self.values = [] self.mirror = None self.roots = [] + self.other_thread = ([], []) def push_roots(self): assert self.roots is None @@ -227,6 +228,13 @@ curhitems[r] = obj self.roots = None + def exchange_threads(self): + old_thread = (self.values, self.roots) + self.switch(1 - self.current_thread) + (self.values, self.roots) = self.other_thread + self.mirror = None + self.other_thread = old_thread + def test_random_single_thread(self): import random # @@ -299,3 +307,88 @@ self.become_inevitable() self.pop_roots() stm_major_collect() # to get rid of the hashtable objects + + def test_random_multiple_threads(self): + import random + self.start_transaction() + self.exchange_threads() + self.start_transaction() + self.pop_roots() + # + for j in range(1000): + r = random.random() + if r > 0.9: + if r > 0.95: + self.push_roots() + self.commit_transaction() + self.start_transaction() + self.pop_roots() + else: + self.push_roots() + self.exchange_threads() + self.pop_roots() + continue + + if r < 0.05: + h = self.allocate_hashtable() + print "allocate_hashtable ->", h + self.mirror[h] = {} + elif r < 0.10: + print "stm_minor_collect" + self.push_roots() + stm_minor_collect() + self.pop_roots() + elif r < 0.11: + print "stm_major_collect" + self.push_roots() + stm_major_collect() + self.pop_roots() + elif r < 0.5: + if not self.mirror: continue + h = random.choice(self.mirror.keys()) + if not self.mirror[h]: continue + key = random.choice(self.mirror[h].keys()) + value = self.mirror[h][key] + print "htget(%r, %r) == %r" % (h, key, value) + self.push_roots() + self.push_root(value) + result = htget(h, key) + value = self.pop_root() + assert result == value + self.pop_roots() + elif r < 0.6: + if not self.mirror: continue + h = random.choice(self.mirror.keys()) + key = random.randrange(0, 40) + if key in self.mirror[h]: continue + print "htget(%r, %r) == NULL" % (h, key) + self.push_roots() + assert htget(h, key) == ffi.NULL + self.pop_roots() + elif r < 0.63: + if not self.mirror: continue + h, _ = self.mirror.popitem() + print "popped", h + elif r < 0.75: + obj = stm_allocate(32) + self.values.append(obj) + stm_set_char(obj, chr(len(self.values) & 255)) + else: + if not self.mirror or not self.values: continue + h = random.choice(self.mirror.keys()) + key = random.randrange(0, 32) + value = random.choice(self.values) + print "htset(%r, %r, %r)" % (h, key, value) + self.push_roots() + tl = self.tls[self.current_thread] + htset(h, key, value, tl) + self.pop_roots() + self.mirror[h][key] = value + # + print 'closing down...' + self.become_inevitable() + self.commit_transaction() + self.exchange_threads() + self.pop_roots() + self.become_inevitable() + stm_major_collect() # to get rid of the hashtable objects From noreply at buildbot.pypy.org Sun Nov 2 23:41:03 2014 From: noreply at buildbot.pypy.org (rlamy) Date: Sun, 2 Nov 2014 23:41:03 +0100 (CET) Subject: [pypy-commit] pypy online-transforms: fix for _mixin_ methods Message-ID: <20141102224103.031591C100F@cobra.cs.uni-duesseldorf.de> Author: Ronan Lamy Branch: online-transforms Changeset: r74326:aa409f72647b Date: 2014-10-30 17:04 +0000 http://bitbucket.org/pypy/pypy/changeset/aa409f72647b/ Log: fix for _mixin_ methods diff --git a/rpython/annotator/bookkeeper.py b/rpython/annotator/bookkeeper.py --- a/rpython/annotator/bookkeeper.py +++ b/rpython/annotator/bookkeeper.py @@ -12,7 +12,7 @@ SomeBuiltin, SomePBC, SomeInteger, TLS, SomeUnicodeCodePoint, s_None, s_ImpossibleValue, SomeBool, SomeTuple, SomeImpossibleValue, SomeUnicodeString, SomeList, HarmlesslyBlocked, - SomeWeakRef, SomeByteArray, SomeConstantType) + SomeWeakRef, SomeByteArray, SomeConstantType, AnnotatorError) from rpython.annotator.classdef import InstanceSource, ClassDef from rpython.annotator.listdef import ListDef, ListItem from rpython.annotator.dictdef import DictDef @@ -330,7 +330,10 @@ if hasattr(x.im_class, '_freeze_'): return self.immutablevalue(x.im_func) cls_s = self.annotationclass(x.im_class) - result = cls_s.find_unboundmethod(x.im_func.__name__) + if cls_s is None: + result = None + else: + result = cls_s.find_unboundmethod(x.im_func.__name__) else: result = None if result is None: @@ -458,7 +461,10 @@ try: return _cls2Some[cls] except KeyError: - return type(self.valueoftype(cls)) + try: + return type(self.valueoftype(cls)) + except AnnotatorError: + return None def get_classpbc_attr_families(self, attrname): """Return the UnionFind for the ClassAttrFamilies corresponding to From noreply at buildbot.pypy.org Sun Nov 2 23:41:06 2014 From: noreply at buildbot.pypy.org (rlamy) Date: Sun, 2 Nov 2014 23:41:06 +0100 (CET) Subject: [pypy-commit] pypy online-transforms: hg merge default Message-ID: <20141102224106.5E9601C100F@cobra.cs.uni-duesseldorf.de> Author: Ronan Lamy Branch: online-transforms Changeset: r74327:d935b9812077 Date: 2014-11-01 05:16 +0000 http://bitbucket.org/pypy/pypy/changeset/d935b9812077/ Log: hg merge default diff too long, truncating to 2000 out of 4803 lines diff --git a/pypy/config/pypyoption.py b/pypy/config/pypyoption.py --- a/pypy/config/pypyoption.py +++ b/pypy/config/pypyoption.py @@ -255,10 +255,6 @@ BoolOption("optimized_list_getitem", "special case the 'list[integer]' expressions", default=False), - BoolOption("builtinshortcut", - "a shortcut for operations between built-in types. XXX: " - "deprecated, not really a shortcut any more.", - default=False), BoolOption("getattributeshortcut", "track types that override __getattribute__", default=False, @@ -270,9 +266,6 @@ # weakrefs needed, because of get_subclasses() requires=[("translation.rweakref", True)]), - ChoiceOption("multimethods", "the multimethod implementation to use", - ["doubledispatch", "mrd"], - default="mrd"), BoolOption("withidentitydict", "track types that override __hash__, __eq__ or __cmp__ and use a special dict strategy for those which do not", default=False, diff --git a/pypy/doc/config/objspace.std.builtinshortcut.txt b/pypy/doc/config/objspace.std.builtinshortcut.txt deleted file mode 100644 --- a/pypy/doc/config/objspace.std.builtinshortcut.txt +++ /dev/null @@ -1,5 +0,0 @@ -A shortcut speeding up primitive operations between built-in types. - -This is a space-time trade-off: at the moment, this option makes a -translated pypy-c executable bigger by about 1.7 MB. (This can probably -be improved with careful analysis.) diff --git a/pypy/doc/config/objspace.std.multimethods.txt b/pypy/doc/config/objspace.std.multimethods.txt deleted file mode 100644 --- a/pypy/doc/config/objspace.std.multimethods.txt +++ /dev/null @@ -1,8 +0,0 @@ -Choose the multimethod implementation. - -* ``doubledispatch`` turns - a multimethod call into a sequence of normal method calls. - -* ``mrd`` uses a technique known as Multiple Row Displacement - which precomputes a few compact tables of numbers and - function pointers. diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -35,3 +35,7 @@ Split RPython documentation from PyPy documentation and clean up. There now is a clearer separation between documentation for users, developers and people interested in background information. + +.. branch: kill-multimethod + +Kill multimethod machinery, all multimethods were removed earlier. diff --git a/pypy/goal/targetpypystandalone.py b/pypy/goal/targetpypystandalone.py --- a/pypy/goal/targetpypystandalone.py +++ b/pypy/goal/targetpypystandalone.py @@ -208,23 +208,6 @@ from pypy.config.pypyoption import set_pypy_opt_level set_pypy_opt_level(config, translateconfig.opt) - # as of revision 27081, multimethod.py uses the InstallerVersion1 by default - # because it is much faster both to initialize and run on top of CPython. - # The InstallerVersion2 is optimized for making a translator-friendly - # structure for low level backends. However, InstallerVersion1 is still - # preferable for high level backends, so we patch here. - - from pypy.objspace.std import multimethod - if config.objspace.std.multimethods == 'mrd': - assert multimethod.InstallerVersion1.instance_counter == 0,\ - 'The wrong Installer version has already been instatiated' - multimethod.Installer = multimethod.InstallerVersion2 - elif config.objspace.std.multimethods == 'doubledispatch': - # don't rely on the default, set again here - assert multimethod.InstallerVersion2.instance_counter == 0,\ - 'The wrong Installer version has already been instatiated' - multimethod.Installer = multimethod.InstallerVersion1 - def print_help(self, config): self.opt_parser(config).print_help() diff --git a/pypy/interpreter/typedef.py b/pypy/interpreter/typedef.py --- a/pypy/interpreter/typedef.py +++ b/pypy/interpreter/typedef.py @@ -618,6 +618,7 @@ from pypy.interpreter.nestedscope import Cell from pypy.interpreter.special import NotImplemented, Ellipsis + def descr_get_dict(space, w_obj): w_dict = w_obj.getdict(space) if w_dict is None: @@ -638,6 +639,11 @@ return space.w_None return lifeline.get_any_weakref(space) +dict_descr = GetSetProperty(descr_get_dict, descr_set_dict, descr_del_dict, + doc="dictionary for instance variables (if defined)") +dict_descr.name = '__dict__' + + def generic_ne(space, w_obj1, w_obj2): if space.eq_w(w_obj1, w_obj2): return space.w_False diff --git a/pypy/module/_file/test/test_file.py b/pypy/module/_file/test/test_file.py --- a/pypy/module/_file/test/test_file.py +++ b/pypy/module/_file/test/test_file.py @@ -275,6 +275,24 @@ finally: f.close() + def test_ignore_ioerror_in_readall_if_nonempty_result(self): + # this is the behavior of regular files in CPython 2.7, as + # well as of _io.FileIO at least in CPython 3.3. This is + # *not* the behavior of _io.FileIO in CPython 3.4 or 3.5; + # see CPython's issue #21090. + try: + from posix import openpty, fdopen, write, close + except ImportError: + skip('no openpty on this platform') + read_fd, write_fd = openpty() + write(write_fd, 'Abc\n') + close(write_fd) + f = fdopen(read_fd) + s = f.read() + assert s == 'Abc\r\n' + raises(IOError, f.read) + f.close() + class AppTestNonblocking(object): def setup_class(cls): diff --git a/pypy/module/_io/interp_bufferedio.py b/pypy/module/_io/interp_bufferedio.py --- a/pypy/module/_io/interp_bufferedio.py +++ b/pypy/module/_io/interp_bufferedio.py @@ -812,11 +812,6 @@ self._check_closed(space, "flush of closed file") with self.lock: self._flush_and_rewind_unlocked(space) - if self.readable: - # Rewind the raw stream so that its position corresponds to - # the current logical position. - self._raw_seek(space, -self._raw_offset(), 1) - self._reader_reset_buf() def _flush_and_rewind_unlocked(self, space): self._writer_flush_unlocked(space) diff --git a/pypy/module/_io/interp_iobase.py b/pypy/module/_io/interp_iobase.py --- a/pypy/module/_io/interp_iobase.py +++ b/pypy/module/_io/interp_iobase.py @@ -24,8 +24,7 @@ try: w_value = error.get_w_value(space) w_errno = space.getattr(w_value, space.wrap("errno")) - return space.is_true( - space.eq(w_errno, space.wrap(EINTR))) + return space.eq_w(w_errno, space.wrap(EINTR)) except OperationError: return False diff --git a/pypy/module/_io/test/test_io.py b/pypy/module/_io/test/test_io.py --- a/pypy/module/_io/test/test_io.py +++ b/pypy/module/_io/test/test_io.py @@ -352,3 +352,13 @@ assert mod == 'io' else: assert mod == '_io' + + def test_issue1902(self): + import _io + with _io.open(self.tmpfile, 'w+b', 4096) as f: + f.write(b'\xff' * 13569) + f.flush() + f.seek(0, 0) + f.read(1) + f.seek(-1, 1) + f.write(b'') diff --git a/pypy/module/array/interp_array.py b/pypy/module/array/interp_array.py --- a/pypy/module/array/interp_array.py +++ b/pypy/module/array/interp_array.py @@ -1,5 +1,3 @@ -from __future__ import with_statement - from rpython.rlib import jit from rpython.rlib.buffer import Buffer from rpython.rlib.objectmodel import keepalive_until_here @@ -15,9 +13,7 @@ interp2app, interpindirect2app, unwrap_spec) from pypy.interpreter.typedef import ( GetSetProperty, TypeDef, make_weakref_descr) -from pypy.interpreter.generator import GeneratorIterator from pypy.module._file.interp_file import W_File -from pypy.objspace.std.floatobject import W_FloatObject @unwrap_spec(typecode=str) @@ -654,7 +650,7 @@ try: item = unwrap(w_item) except OperationError, e: - if isinstance(w_item, W_FloatObject): + if space.isinstance_w(w_item, space.w_float): # Odd special case from cpython raise if mytype.method != '' and e.match(space, space.w_TypeError): diff --git a/pypy/module/array/test/test_array.py b/pypy/module/array/test/test_array.py --- a/pypy/module/array/test/test_array.py +++ b/pypy/module/array/test/test_array.py @@ -1035,8 +1035,3 @@ def test_fresh_array_buffer_str(self): assert str(buffer(self.array('i'))) == '' - - -class AppTestArrayBuiltinShortcut(AppTestArray): - spaceconfig = AppTestArray.spaceconfig.copy() - spaceconfig['objspace.std.builtinshortcut'] = True diff --git a/pypy/module/cpyext/typeobject.py b/pypy/module/cpyext/typeobject.py --- a/pypy/module/cpyext/typeobject.py +++ b/pypy/module/cpyext/typeobject.py @@ -303,11 +303,6 @@ @bootstrap_function def init_typeobject(space): - # Probably a hack - space.model.typeorder[W_PyCTypeObject] = [(W_PyCTypeObject, None), - (W_TypeObject, None), - (W_Root, None)] - make_typedescr(space.w_type.instancetypedef, basestruct=PyTypeObject, alloc=type_alloc, diff --git a/pypy/objspace/fake/objspace.py b/pypy/objspace/fake/objspace.py --- a/pypy/objspace/fake/objspace.py +++ b/pypy/objspace/fake/objspace.py @@ -2,7 +2,6 @@ from pypy.interpreter import argument, gateway from pypy.interpreter.baseobjspace import W_Root, ObjSpace, SpaceCache from pypy.interpreter.typedef import TypeDef, GetSetProperty -from pypy.objspace.std.stdtypedef import StdTypeDef from pypy.objspace.std.sliceobject import W_SliceObject from rpython.rlib.buffer import StringBuffer from rpython.rlib.objectmodel import instantiate, we_are_translated, specialize @@ -112,6 +111,10 @@ # ____________________________________________________________ +BUILTIN_TYPES = ['int', 'str', 'float', 'long', 'tuple', 'list', 'dict', + 'unicode', 'complex', 'slice', 'bool', 'basestring', 'object', + 'bytearray', 'buffer'] + class FakeObjSpace(ObjSpace): def __init__(self, config=None): self._seen_extras = [] @@ -342,9 +345,7 @@ def setup(space): for name in (ObjSpace.ConstantTable + ObjSpace.ExceptionTable + - ['int', 'str', 'float', 'long', 'tuple', 'list', - 'dict', 'unicode', 'complex', 'slice', 'bool', - 'basestring', 'object', 'bytearray', 'buffer']): + BUILTIN_TYPES): setattr(space, 'w_' + name, w_some_obj()) space.w_type = w_some_type() # @@ -375,7 +376,7 @@ @specialize.memo() def see_typedef(space, typedef): assert isinstance(typedef, TypeDef) - if not isinstance(typedef, StdTypeDef): + if typedef.name not in BUILTIN_TYPES: for name, value in typedef.rawdict.items(): space.wrap(value) diff --git a/pypy/objspace/std/basestringtype.py b/pypy/objspace/std/basestringtype.py --- a/pypy/objspace/std/basestringtype.py +++ b/pypy/objspace/std/basestringtype.py @@ -1,7 +1,7 @@ -from pypy.objspace.std.stdtypedef import StdTypeDef +from pypy.interpreter.typedef import TypeDef -basestring_typedef = StdTypeDef("basestring", +basestring_typedef = TypeDef("basestring", __doc__ = ("basestring cannot be instantiated; " "it is the base for str and unicode.") ) diff --git a/pypy/objspace/std/boolobject.py b/pypy/objspace/std/boolobject.py --- a/pypy/objspace/std/boolobject.py +++ b/pypy/objspace/std/boolobject.py @@ -6,8 +6,8 @@ from rpython.tool.sourcetools import func_renamer, func_with_new_name from pypy.interpreter.gateway import WrappedDefault, interp2app, unwrap_spec +from pypy.interpreter.typedef import TypeDef from pypy.objspace.std.intobject import W_AbstractIntObject, W_IntObject -from pypy.objspace.std.stdtypedef import StdTypeDef class W_BoolObject(W_IntObject): @@ -80,7 +80,7 @@ W_BoolObject.w_True = W_BoolObject(True) -W_BoolObject.typedef = StdTypeDef("bool", W_IntObject.typedef, +W_BoolObject.typedef = TypeDef("bool", W_IntObject.typedef, __doc__ = """bool(x) -> bool Returns True when the argument x is true, False otherwise. diff --git a/pypy/objspace/std/builtinshortcut.py b/pypy/objspace/std/builtinshortcut.py deleted file mode 100644 --- a/pypy/objspace/std/builtinshortcut.py +++ /dev/null @@ -1,137 +0,0 @@ -from pypy.interpreter.baseobjspace import ObjSpace -from pypy.interpreter.error import OperationError -from pypy.objspace.descroperation import DescrOperation -from pypy.objspace.std.multimethod import FailedToImplement -from pypy.objspace.std.boolobject import W_BoolObject -from rpython.tool.sourcetools import func_with_new_name - -# ____________________________________________________________ -# -# The sole purpose of this file is performance. -# It speeds up the dispatch of operations between -# built-in objects. -# - -# this is a selection... a few operations are missing because they are -# thought to be very rare or most commonly used with non-builtin types -METHODS_WITH_SHORTCUT = dict.fromkeys( - ['add', 'sub', 'mul', 'truediv', 'floordiv', 'div', - 'mod', 'lshift', 'rshift', 'and_', 'or_', 'xor', 'pow', - 'lt', 'le', 'eq', 'ne', 'gt', 'ge', 'contains', - # unary - 'len', 'nonzero', 'repr', 'str', 'hash', - 'neg', 'invert', 'index', 'iter', 'next', 'buffer', - 'getitem', 'setitem', 'int', - # in-place - 'inplace_add', 'inplace_sub', 'inplace_mul', 'inplace_truediv', - 'inplace_floordiv', 'inplace_div', 'inplace_mod', 'inplace_pow', - 'inplace_lshift', 'inplace_rshift', 'inplace_and', 'inplace_or', - 'inplace_xor', - # other - 'format', - ]) - -KNOWN_MISSING = ['getattr', # mostly non-builtins or optimized by CALL_METHOD - 'setattr', 'delattr', 'userdel', # mostly for non-builtins - 'get', 'set', 'delete', # uncommon (except on functions) - 'getslice', 'setslice', 'delslice', # see below - 'delitem', 'trunc', # rare stuff? - 'abs', 'hex', 'oct', # rare stuff? - 'pos', 'divmod', 'cmp', # rare stuff? - 'float', 'long', 'coerce', # rare stuff? - 'isinstance', 'issubtype', - ] -# We cannot support {get,set,del}slice right now because -# DescrOperation.{get,set,del}slice do a bit more work than just call -# the special methods: they call old_slice_range(). See e.g. -# test_builtinshortcut.AppTestString. - -for _name, _, _, _specialmethods in ObjSpace.MethodTable: - if _specialmethods: - assert _name in METHODS_WITH_SHORTCUT or _name in KNOWN_MISSING, ( - "operation %r should be in METHODS_WITH_SHORTCUT or KNOWN_MISSING" - % (_name,)) - - -def filter_out_conversions(typeorder): - res = {} - for cls, order in typeorder.iteritems(): - res[cls] = [(target_type, converter) for (target_type, converter) in - order if converter is None] - return res - - -def install(space, mm, fallback_mm=None): - """Install a function () on the space instance which invokes - a shortcut for built-in types. Returns the shortcutting multimethod - object or None. - """ - name = mm.name - if name not in METHODS_WITH_SHORTCUT: - return None - - # can be called multiple times without re-installing - if name in space.__dict__: - mm1, shortcut_method = space.__dict__[name].builtinshortcut - assert mm1 is mm - return shortcut_method - - #print 'installing shortcut for:', name - assert hasattr(DescrOperation, name) - - base_method = getattr(space.__class__, name) - - # Basic idea: we first try to dispatch the operation using purely - # the multimethod. If this is done naively, subclassing a built-in - # type like 'int' and overriding a special method like '__add__' - # doesn't work any more, because the multimethod will accept the int - # subclass and compute the result in the built-in way. To avoid - # this issue, we tweak the shortcut multimethods so that these ones - # (and only these ones) never match the interp-level subclasses - # built in pypy.interpreter.typedef.get_unique_interplevel_subclass. - expanded_order = space.model.get_typeorder_with_empty_usersubcls() - if fallback_mm: - mm = mm.merge_with(fallback_mm) - shortcut_method = mm.install_not_sliced(filter_out_conversions(expanded_order)) - - def operate(*args_w): - try: - return shortcut_method(space, *args_w) - except FailedToImplement: - pass - return base_method(space, *args_w) - - operate = func_with_new_name(operate, name) - operate.builtinshortcut = (mm, shortcut_method) - setattr(space, name, operate) - return shortcut_method - - -def install_is_true(space, mm_nonzero, mm_len): - shortcut = install(space, mm_nonzero, fallback_mm = mm_len) - assert 'is_true' not in space.__dict__ - - def is_true(w_obj): - # a bit of duplication of the logic from DescrOperation.is_true... - try: - w_res = shortcut(space, w_obj) - except FailedToImplement: - pass - else: - # the __nonzero__ method of built-in objects should - # always directly return a Bool; however, the __len__ method - # of built-in objects typically returns an unwrappable integer - if isinstance(w_res, W_BoolObject): - return bool(w_res.intval) - try: - return space.int_w(w_res) != 0 - except OperationError: - # I think no OperationError other than w_OverflowError - # could occur here - w_obj = w_res - - # general case fallback - return _DescrOperation_is_true(space, w_obj) - - _DescrOperation_is_true = DescrOperation.is_true.im_func - space.is_true = is_true diff --git a/pypy/objspace/std/bytearrayobject.py b/pypy/objspace/std/bytearrayobject.py --- a/pypy/objspace/std/bytearrayobject.py +++ b/pypy/objspace/std/bytearrayobject.py @@ -10,8 +10,8 @@ from pypy.interpreter.error import OperationError, oefmt from pypy.interpreter.gateway import WrappedDefault, interp2app, unwrap_spec from pypy.interpreter.signature import Signature +from pypy.interpreter.typedef import TypeDef from pypy.objspace.std.sliceobject import W_SliceObject -from pypy.objspace.std.stdtypedef import StdTypeDef from pypy.objspace.std.stringmethods import StringMethods, _get_buffer from pypy.objspace.std.bytesobject import W_BytesObject from pypy.objspace.std.util import get_positive_index @@ -990,7 +990,7 @@ """ -W_BytearrayObject.typedef = StdTypeDef( +W_BytearrayObject.typedef = TypeDef( "bytearray", __doc__ = BytearrayDocstrings.__doc__, __new__ = interp2app(W_BytearrayObject.descr_new), diff --git a/pypy/objspace/std/bytesobject.py b/pypy/objspace/std/bytesobject.py --- a/pypy/objspace/std/bytesobject.py +++ b/pypy/objspace/std/bytesobject.py @@ -10,10 +10,10 @@ from pypy.interpreter.error import OperationError, oefmt from pypy.interpreter.gateway import ( WrappedDefault, interp2app, interpindirect2app, unwrap_spec) +from pypy.interpreter.typedef import TypeDef from pypy.objspace.std import newformat from pypy.objspace.std.basestringtype import basestring_typedef from pypy.objspace.std.formatting import mod_format -from pypy.objspace.std.stdtypedef import StdTypeDef from pypy.objspace.std.stringmethods import StringMethods from pypy.objspace.std.unicodeobject import ( _get_encoding_and_errors, decode_object, unicode_from_encoded_object, @@ -874,7 +874,7 @@ return W_BytesObject(c) -W_BytesObject.typedef = StdTypeDef( +W_BytesObject.typedef = TypeDef( "str", basestring_typedef, __new__ = interp2app(W_BytesObject.descr_new), __doc__ = """str(object='') -> string diff --git a/pypy/objspace/std/complexobject.py b/pypy/objspace/std/complexobject.py --- a/pypy/objspace/std/complexobject.py +++ b/pypy/objspace/std/complexobject.py @@ -1,18 +1,19 @@ import math -from pypy.interpreter.baseobjspace import W_Root -from pypy.interpreter.error import OperationError, oefmt -from pypy.interpreter.gateway import interp2app, unwrap_spec, WrappedDefault -from pypy.objspace.std import newformat -from pypy.objspace.std.floatobject import _hash_float -from pypy.objspace.std.stdtypedef import GetSetProperty, StdTypeDef from rpython.rlib import jit, rcomplex from rpython.rlib.rarithmetic import intmask, r_ulonglong from rpython.rlib.rbigint import rbigint from rpython.rlib.rfloat import ( - formatd, DTSF_STR_PRECISION, isinf, isnan, copysign, string_to_float) + DTSF_STR_PRECISION, copysign, formatd, isinf, isnan, string_to_float) from rpython.rlib.rstring import ParseStringError +from pypy.interpreter.baseobjspace import W_Root +from pypy.interpreter.error import OperationError, oefmt +from pypy.interpreter.gateway import WrappedDefault, interp2app, unwrap_spec +from pypy.interpreter.typedef import GetSetProperty, TypeDef +from pypy.objspace.std import newformat +from pypy.objspace.std.floatobject import _hash_float + def _split_complex(s): slen = len(s) @@ -264,7 +265,7 @@ if self.user_overridden_class: return None from rpython.rlib.longlong2float import float2longlong - from pypy.objspace.std.model import IDTAG_COMPLEX as tag + from pypy.objspace.std.util import IDTAG_COMPLEX as tag real = space.float_w(space.getattr(self, space.wrap("real"))) imag = space.float_w(space.getattr(self, space.wrap("imag"))) real_b = rbigint.fromrarith_int(float2longlong(real)) @@ -587,7 +588,7 @@ return space.newfloat(getattr(w_obj, name)) return GetSetProperty(fget) -W_ComplexObject.typedef = StdTypeDef("complex", +W_ComplexObject.typedef = TypeDef("complex", __doc__ = """complex(real[, imag]) -> complex number Create a complex number from a real part and an optional imaginary part. diff --git a/pypy/objspace/std/dictmultiobject.py b/pypy/objspace/std/dictmultiobject.py --- a/pypy/objspace/std/dictmultiobject.py +++ b/pypy/objspace/std/dictmultiobject.py @@ -11,7 +11,7 @@ WrappedDefault, applevel, interp2app, unwrap_spec) from pypy.interpreter.mixedmodule import MixedModule from pypy.interpreter.signature import Signature -from pypy.objspace.std.stdtypedef import StdTypeDef +from pypy.interpreter.typedef import TypeDef from pypy.objspace.std.util import negate @@ -372,7 +372,7 @@ dictrepr = app.interphook("dictrepr") -W_DictMultiObject.typedef = StdTypeDef("dict", +W_DictMultiObject.typedef = TypeDef("dict", __doc__ = '''dict() -> new empty dictionary. dict(mapping) -> new dictionary initialized from a mapping object\'s (key, value) pairs. @@ -1217,8 +1217,6 @@ class W_BaseDictMultiIterObject(W_Root): _immutable_fields_ = ["iteratorimplementation"] - ignore_for_isinstance_cache = True - def __init__(self, space, iteratorimplementation): self.space = space self.iteratorimplementation = iteratorimplementation @@ -1309,7 +1307,7 @@ return space.newtuple([w_key, w_value]) raise OperationError(space.w_StopIteration, space.w_None) -W_DictMultiIterItemsObject.typedef = StdTypeDef( +W_DictMultiIterItemsObject.typedef = TypeDef( "dict_iteritems", __iter__ = interp2app(W_DictMultiIterItemsObject.descr_iter), next = interp2app(W_DictMultiIterItemsObject.descr_next), @@ -1317,7 +1315,7 @@ __reduce__ = interp2app(W_BaseDictMultiIterObject.descr_reduce), ) -W_DictMultiIterKeysObject.typedef = StdTypeDef( +W_DictMultiIterKeysObject.typedef = TypeDef( "dict_iterkeys", __iter__ = interp2app(W_DictMultiIterKeysObject.descr_iter), next = interp2app(W_DictMultiIterKeysObject.descr_next), @@ -1325,7 +1323,7 @@ __reduce__ = interp2app(W_BaseDictMultiIterObject.descr_reduce), ) -W_DictMultiIterValuesObject.typedef = StdTypeDef( +W_DictMultiIterValuesObject.typedef = TypeDef( "dict_itervalues", __iter__ = interp2app(W_DictMultiIterValuesObject.descr_iter), next = interp2app(W_DictMultiIterValuesObject.descr_next), @@ -1433,7 +1431,7 @@ def descr_iter(self, space): return W_DictMultiIterValuesObject(space, self.w_dict.itervalues()) -W_DictViewItemsObject.typedef = StdTypeDef( +W_DictViewItemsObject.typedef = TypeDef( "dict_items", __repr__ = interp2app(W_DictViewItemsObject.descr_repr), __len__ = interp2app(W_DictViewItemsObject.descr_len), @@ -1456,7 +1454,7 @@ __rxor__ = interp2app(W_DictViewItemsObject.descr_rxor), ) -W_DictViewKeysObject.typedef = StdTypeDef( +W_DictViewKeysObject.typedef = TypeDef( "dict_keys", __repr__ = interp2app(W_DictViewKeysObject.descr_repr), __len__ = interp2app(W_DictViewKeysObject.descr_len), @@ -1479,7 +1477,7 @@ __rxor__ = interp2app(W_DictViewKeysObject.descr_rxor), ) -W_DictViewValuesObject.typedef = StdTypeDef( +W_DictViewValuesObject.typedef = TypeDef( "dict_values", __repr__ = interp2app(W_DictViewValuesObject.descr_repr), __len__ = interp2app(W_DictViewValuesObject.descr_len), diff --git a/pypy/objspace/std/dictproxyobject.py b/pypy/objspace/std/dictproxyobject.py --- a/pypy/objspace/std/dictproxyobject.py +++ b/pypy/objspace/std/dictproxyobject.py @@ -1,8 +1,9 @@ -from pypy.objspace.std.dictmultiobject import DictStrategy, create_iterator_classes +from rpython.rlib import rerased + +from pypy.interpreter.error import OperationError, oefmt +from pypy.objspace.std.dictmultiobject import ( + DictStrategy, create_iterator_classes) from pypy.objspace.std.typeobject import unwrap_cell -from pypy.interpreter.error import OperationError, oefmt - -from rpython.rlib import rerased class DictProxyStrategy(DictStrategy): diff --git a/pypy/objspace/std/floatobject.py b/pypy/objspace/std/floatobject.py --- a/pypy/objspace/std/floatobject.py +++ b/pypy/objspace/std/floatobject.py @@ -2,24 +2,24 @@ import operator import sys +from rpython.rlib import rarithmetic, rfloat +from rpython.rlib.rarithmetic import LONG_BIT, intmask, ovfcheck_float_to_int +from rpython.rlib.rbigint import rbigint +from rpython.rlib.rfloat import ( + DTSF_ADD_DOT_0, DTSF_STR_PRECISION, INFINITY, NAN, copysign, + float_as_rbigint_ratio, formatd, isfinite, isinf, isnan) +from rpython.rlib.rstring import ParseStringError +from rpython.rlib.unroll import unrolling_iterable +from rpython.rtyper.lltypesystem.module.ll_math import math_fmod +from rpython.tool.sourcetools import func_with_new_name + from pypy.interpreter.baseobjspace import W_Root from pypy.interpreter.error import OperationError, oefmt -from pypy.interpreter.gateway import interp2app, unwrap_spec, WrappedDefault -from pypy.interpreter.typedef import GetSetProperty +from pypy.interpreter.gateway import WrappedDefault, interp2app, unwrap_spec +from pypy.interpreter.typedef import GetSetProperty, TypeDef from pypy.objspace.std import newformat from pypy.objspace.std.longobject import W_LongObject -from pypy.objspace.std.stdtypedef import StdTypeDef from pypy.objspace.std.util import wrap_parsestringerror -from rpython.rlib import rarithmetic, rfloat -from rpython.rlib.rarithmetic import ovfcheck_float_to_int, intmask, LONG_BIT -from rpython.rlib.rbigint import rbigint -from rpython.rlib.rfloat import ( - isinf, isnan, isfinite, INFINITY, NAN, copysign, formatd, - DTSF_ADD_DOT_0, DTSF_STR_PRECISION, float_as_rbigint_ratio) -from rpython.rlib.rstring import ParseStringError -from rpython.tool.sourcetools import func_with_new_name -from rpython.rlib.unroll import unrolling_iterable -from rpython.rtyper.lltypesystem.module.ll_math import math_fmod def float2string(x, code, precision): @@ -182,7 +182,7 @@ if self.user_overridden_class: return None from rpython.rlib.longlong2float import float2longlong - from pypy.objspace.std.model import IDTAG_FLOAT as tag + from pypy.objspace.std.util import IDTAG_FLOAT as tag val = float2longlong(space.float_w(self)) b = rbigint.fromrarith_int(val) b = b.lshift(3).or_(rbigint.fromint(tag)) @@ -646,7 +646,7 @@ return space.wrap("0x%sp%s%d" % (s, sign, exp)) -W_FloatObject.typedef = StdTypeDef("float", +W_FloatObject.typedef = TypeDef("float", __doc__ = '''float(x) -> floating point number Convert a string or number to a floating point number, if possible.''', diff --git a/pypy/objspace/std/formatting.py b/pypy/objspace/std/formatting.py --- a/pypy/objspace/std/formatting.py +++ b/pypy/objspace/std/formatting.py @@ -1,15 +1,15 @@ -""" -String formatting routines. -""" +"""String formatting routines""" import sys -from pypy.interpreter.error import OperationError, oefmt + from rpython.rlib import jit -from rpython.rlib.rfloat import formatd, DTSF_ALT, isnan, isinf +from rpython.rlib.rarithmetic import INT_MAX +from rpython.rlib.rfloat import DTSF_ALT, formatd, isnan, isinf from rpython.rlib.rstring import StringBuilder, UnicodeBuilder from rpython.rlib.unroll import unrolling_iterable -from rpython.rlib.rarithmetic import INT_MAX from rpython.tool.sourcetools import func_with_new_name +from pypy.interpreter.error import OperationError, oefmt + class BaseStringFormatter(object): def __init__(self, space, values_w, w_valuedict): diff --git a/pypy/objspace/std/intobject.py b/pypy/objspace/std/intobject.py --- a/pypy/objspace/std/intobject.py +++ b/pypy/objspace/std/intobject.py @@ -21,11 +21,10 @@ from pypy.interpreter.baseobjspace import W_Root from pypy.interpreter.error import OperationError, oefmt from pypy.interpreter.gateway import WrappedDefault, interp2app, unwrap_spec +from pypy.interpreter.typedef import TypeDef from pypy.objspace.std import newformat -from pypy.objspace.std.model import ( - BINARY_OPS, CMP_OPS, COMMUTATIVE_OPS, IDTAG_INT) -from pypy.objspace.std.stdtypedef import StdTypeDef -from pypy.objspace.std.util import wrap_parsestringerror +from pypy.objspace.std.util import ( + BINARY_OPS, CMP_OPS, COMMUTATIVE_OPS, IDTAG_INT, wrap_parsestringerror) SENTINEL = object() @@ -601,6 +600,16 @@ _divmod, ovf2small=_divmod_ovf2small) +def setup_prebuilt(space): + if space.config.objspace.std.withprebuiltint: + W_IntObject.PREBUILT = [] + for i in range(space.config.objspace.std.prebuiltintfrom, + space.config.objspace.std.prebuiltintto): + W_IntObject.PREBUILT.append(W_IntObject(i)) + else: + W_IntObject.PREBUILT = None + + def wrapint(space, x): if not space.config.objspace.std.withprebuiltint: return W_IntObject(x) @@ -723,7 +732,7 @@ return w_obj -W_IntObject.typedef = StdTypeDef("int", +W_IntObject.typedef = TypeDef("int", __doc__ = """int(x=0) -> int or long int(x, base=10) -> int or long diff --git a/pypy/objspace/std/iterobject.py b/pypy/objspace/std/iterobject.py --- a/pypy/objspace/std/iterobject.py +++ b/pypy/objspace/std/iterobject.py @@ -3,7 +3,7 @@ from pypy.interpreter.baseobjspace import W_Root from pypy.interpreter.gateway import interp2app, interpindirect2app from pypy.interpreter.error import OperationError -from pypy.objspace.std.stdtypedef import StdTypeDef +from pypy.interpreter.typedef import TypeDef class W_AbstractSeqIterObject(W_Root): @@ -40,7 +40,7 @@ def descr_length_hint(self, space): return self.getlength(space) -W_AbstractSeqIterObject.typedef = StdTypeDef( +W_AbstractSeqIterObject.typedef = TypeDef( "sequenceiterator", __doc__ = '''iter(collection) -> iterator iter(callable, sentinel) -> iterator @@ -159,7 +159,7 @@ raise OperationError(space.w_StopIteration, space.w_None) return w_item -W_ReverseSeqIterObject.typedef = StdTypeDef( +W_ReverseSeqIterObject.typedef = TypeDef( "reversesequenceiterator", __iter__ = interp2app(W_ReverseSeqIterObject.descr_iter), next = interp2app(W_ReverseSeqIterObject.descr_next), diff --git a/pypy/objspace/std/listobject.py b/pypy/objspace/std/listobject.py --- a/pypy/objspace/std/listobject.py +++ b/pypy/objspace/std/listobject.py @@ -10,28 +10,29 @@ import operator import sys +from rpython.rlib import debug, jit, rerased +from rpython.rlib.listsort import make_timsort_class +from rpython.rlib.objectmodel import ( + import_from_mixin, instantiate, newlist_hint, resizelist_hint, specialize) +from rpython.tool.sourcetools import func_with_new_name + from pypy.interpreter.baseobjspace import W_Root from pypy.interpreter.error import OperationError, oefmt -from pypy.interpreter.gateway import (WrappedDefault, unwrap_spec, applevel, - interp2app) +from pypy.interpreter.gateway import ( + WrappedDefault, applevel, interp2app, unwrap_spec) from pypy.interpreter.generator import GeneratorIterator from pypy.interpreter.signature import Signature +from pypy.interpreter.typedef import TypeDef from pypy.objspace.std.bytesobject import W_BytesObject from pypy.objspace.std.floatobject import W_FloatObject from pypy.objspace.std.intobject import W_IntObject -from pypy.objspace.std.iterobject import (W_FastListIterObject, - W_ReverseSeqIterObject) -from pypy.objspace.std.sliceobject import (W_SliceObject, unwrap_start_stop, - normalize_simple_slice) -from pypy.objspace.std.stdtypedef import StdTypeDef +from pypy.objspace.std.iterobject import ( + W_FastListIterObject, W_ReverseSeqIterObject) +from pypy.objspace.std.sliceobject import ( + W_SliceObject, normalize_simple_slice, unwrap_start_stop) from pypy.objspace.std.tupleobject import W_AbstractTupleObject from pypy.objspace.std.unicodeobject import W_UnicodeObject from pypy.objspace.std.util import get_positive_index, negate -from rpython.rlib import debug, jit, rerased -from rpython.rlib.listsort import make_timsort_class -from rpython.rlib.objectmodel import ( - instantiate, newlist_hint, resizelist_hint, specialize, import_from_mixin) -from rpython.tool.sourcetools import func_with_new_name __all__ = ['W_ListObject', 'make_range_list', 'make_empty_list_with_size'] @@ -1830,7 +1831,7 @@ return CustomCompareSort.lt(self, a.w_key, b.w_key) -W_ListObject.typedef = StdTypeDef("list", +W_ListObject.typedef = TypeDef("list", __doc__ = """list() -> new empty list list(iterable) -> new list initialized from iterable's items""", __new__ = interp2app(W_ListObject.descr_new), diff --git a/pypy/objspace/std/longobject.py b/pypy/objspace/std/longobject.py --- a/pypy/objspace/std/longobject.py +++ b/pypy/objspace/std/longobject.py @@ -12,12 +12,11 @@ from pypy.interpreter.error import OperationError, oefmt from pypy.interpreter.gateway import ( WrappedDefault, interp2app, interpindirect2app, unwrap_spec) +from pypy.interpreter.typedef import TypeDef from pypy.objspace.std import newformat from pypy.objspace.std.intobject import W_AbstractIntObject -from pypy.objspace.std.model import ( - BINARY_OPS, CMP_OPS, COMMUTATIVE_OPS, IDTAG_LONG) -from pypy.objspace.std.stdtypedef import StdTypeDef -from pypy.objspace.std.util import wrap_parsestringerror +from pypy.objspace.std.util import ( + BINARY_OPS, CMP_OPS, COMMUTATIVE_OPS, IDTAG_LONG, wrap_parsestringerror) def delegate_other(func): @@ -567,7 +566,7 @@ return w_obj -W_AbstractLongObject.typedef = StdTypeDef("long", +W_AbstractLongObject.typedef = TypeDef("long", __doc__ = """long(x=0) -> long long(x, base=10) -> long diff --git a/pypy/objspace/std/model.py b/pypy/objspace/std/model.py deleted file mode 100644 --- a/pypy/objspace/std/model.py +++ /dev/null @@ -1,357 +0,0 @@ -""" -The full list of which Python types and which implementation we want -to provide in this version of PyPy, along with conversion rules. -""" - -from pypy.objspace.std.multimethod import MultiMethodTable, FailedToImplement -from pypy.interpreter.baseobjspace import W_Root, ObjSpace -import pypy.interpreter.pycode -import pypy.interpreter.special - -_registered_implementations = set() -def registerimplementation(implcls): - """Hint to objspace.std.model to register the implementation class.""" - assert issubclass(implcls, W_Object) - _registered_implementations.add(implcls) - -option_to_typename = { - "withsmalllong" : ["smalllongobject.W_SmallLongObject"], - "withstrbuf" : ["strbufobject.W_StringBufferObject"], -} - -IDTAG_INT = 1 -IDTAG_LONG = 3 -IDTAG_FLOAT = 5 -IDTAG_COMPLEX = 7 - -class StdTypeModel: - - def __init__(self, config): - """NOT_RPYTHON: inititialization only""" - self.config = config - # All the Python types that we want to provide in this StdObjSpace - - # The object implementations that we want to 'link' into PyPy must be - # imported here. This registers them into the multimethod tables, - # *before* the type objects are built from these multimethod tables. - from pypy.objspace.std import objectobject - from pypy.objspace.std import boolobject - from pypy.objspace.std import intobject - from pypy.objspace.std import floatobject - from pypy.objspace.std import complexobject - from pypy.objspace.std import tupleobject - from pypy.objspace.std import listobject - from pypy.objspace.std import dictmultiobject - from pypy.objspace.std import setobject - from pypy.objspace.std import basestringtype - from pypy.objspace.std import bytesobject - from pypy.objspace.std import bytearrayobject - from pypy.objspace.std import typeobject - from pypy.objspace.std import sliceobject - from pypy.objspace.std import longobject - from pypy.objspace.std import noneobject - from pypy.objspace.std import iterobject - from pypy.objspace.std import unicodeobject - from pypy.objspace.std import dictproxyobject - from pypy.objspace.std import proxyobject - from pypy.objspace.std import bufferobject - from pypy.objspace.std import memoryobject - - - self.pythontypes = [] - self.pythontypes.append(objectobject.W_ObjectObject.typedef) - self.pythontypes.append(typeobject.W_TypeObject.typedef) - self.pythontypes.append(noneobject.W_NoneObject.typedef) - self.pythontypes.append(tupleobject.W_TupleObject.typedef) - self.pythontypes.append(listobject.W_ListObject.typedef) - self.pythontypes.append(dictmultiobject.W_DictMultiObject.typedef) - self.pythontypes.append(setobject.W_SetObject.typedef) - self.pythontypes.append(setobject.W_FrozensetObject.typedef) - self.pythontypes.append(iterobject.W_AbstractSeqIterObject.typedef) - self.pythontypes.append(basestringtype.basestring_typedef) - self.pythontypes.append(bytesobject.W_BytesObject.typedef) - self.pythontypes.append(bytearrayobject.W_BytearrayObject.typedef) - self.pythontypes.append(unicodeobject.W_UnicodeObject.typedef) - self.pythontypes.append(intobject.W_IntObject.typedef) - self.pythontypes.append(boolobject.W_BoolObject.typedef) - self.pythontypes.append(longobject.W_LongObject.typedef) - self.pythontypes.append(floatobject.W_FloatObject.typedef) - self.pythontypes.append(complexobject.W_ComplexObject.typedef) - self.pythontypes.append(sliceobject.W_SliceObject.typedef) - self.pythontypes.append(bufferobject.W_Buffer.typedef) - self.pythontypes.append(memoryobject.W_MemoryView.typedef) - - # the set of implementation types - self.typeorder = { - objectobject.W_ObjectObject: [], - # XXX: Bool/Int/Long are pythontypes but still included here - # for delegation to Float/Complex - boolobject.W_BoolObject: [], - intobject.W_IntObject: [], - floatobject.W_FloatObject: [], - typeobject.W_TypeObject: [], - sliceobject.W_SliceObject: [], - longobject.W_LongObject: [], - noneobject.W_NoneObject: [], - complexobject.W_ComplexObject: [], - pypy.interpreter.pycode.PyCode: [], - pypy.interpreter.special.Ellipsis: [], - } - - self.imported_but_not_registered = { - bytesobject.W_BytesObject: True, - } - for option, value in config.objspace.std: - if option.startswith("with") and option in option_to_typename: - for classname in option_to_typename[option]: - modname = classname[:classname.index('.')] - classname = classname[classname.index('.')+1:] - d = {} - exec "from pypy.objspace.std.%s import %s" % ( - modname, classname) in d - implcls = d[classname] - if value: - self.typeorder[implcls] = [] - else: - self.imported_but_not_registered[implcls] = True - - # check if we missed implementations - for implcls in _registered_implementations: - if hasattr(implcls, 'register'): - implcls.register(self.typeorder) - assert (implcls in self.typeorder or - implcls in self.imported_but_not_registered), ( - "please add %r in StdTypeModel.typeorder" % (implcls,)) - - - for type in self.typeorder: - self.typeorder[type].append((type, None)) - - # register the order in which types are converted into each others - # when trying to dispatch multimethods. - # XXX build these lists a bit more automatically later - - if config.objspace.std.withsmalllong: - from pypy.objspace.std import smalllongobject - self.typeorder[smalllongobject.W_SmallLongObject] += [ - (floatobject.W_FloatObject, smalllongobject.delegate_SmallLong2Float), - (complexobject.W_ComplexObject, smalllongobject.delegate_SmallLong2Complex), - ] - - if config.objspace.std.withstrbuf: - from pypy.objspace.std import strbufobject - - # put W_Root everywhere - self.typeorder[W_Root] = [] - for type in self.typeorder: - from pypy.objspace.std import stdtypedef - if type is not W_Root and isinstance(type.typedef, stdtypedef.StdTypeDef): - self.typeorder[type].append((type.typedef.any, None)) - self.typeorder[type].append((W_Root, None)) - - self._typeorder_with_empty_usersubcls = None - - # ____________________________________________________________ - # Prebuilt common integer values - - if config.objspace.std.withprebuiltint: - intobject.W_IntObject.PREBUILT = [] - for i in range(config.objspace.std.prebuiltintfrom, - config.objspace.std.prebuiltintto): - intobject.W_IntObject.PREBUILT.append(intobject.W_IntObject(i)) - del i - else: - intobject.W_IntObject.PREBUILT = None - - # ____________________________________________________________ - - def get_typeorder_with_empty_usersubcls(self): - if self._typeorder_with_empty_usersubcls is None: - from pypy.interpreter.typedef import enum_interplevel_subclasses - from pypy.objspace.std import stdtypedef - result = self.typeorder.copy() - for cls in self.typeorder: - if (hasattr(cls, 'typedef') and cls.typedef is not None and - cls.typedef.acceptable_as_base_class): - subclslist = enum_interplevel_subclasses(self.config, cls) - for subcls in subclslist: - if cls in subcls.__bases__: # only direct subclasses - # for user subclasses we only accept "generic" - # matches: "typedef.any" is the applevel-type-based - # matching, and "W_Root" is ANY. - matches = [] - if isinstance(cls.typedef, stdtypedef.StdTypeDef): - matches.append((cls.typedef.any, None)) - matches.append((W_Root, None)) - result[subcls] = matches - self._typeorder_with_empty_usersubcls = result - return self._typeorder_with_empty_usersubcls - -def _op_negated(function): - def op(space, w_1, w_2): - return space.not_(function(space, w_1, w_2)) - return op - -def _op_swapped(function): - def op(space, w_1, w_2): - return function(space, w_2, w_1) - return op - -def _op_swapped_negated(function): - def op(space, w_1, w_2): - return space.not_(function(space, w_2, w_1)) - return op - - -CMP_OPS = dict(lt='<', le='<=', eq='==', ne='!=', gt='>', ge='>=') -CMP_CORRESPONDANCES = [ - ('eq', 'ne', _op_negated), - ('lt', 'gt', _op_swapped), - ('le', 'ge', _op_swapped), - ('lt', 'ge', _op_negated), - ('le', 'gt', _op_negated), - ('lt', 'le', _op_swapped_negated), - ('gt', 'ge', _op_swapped_negated), - ] -for op1, op2, value in CMP_CORRESPONDANCES[:]: - i = CMP_CORRESPONDANCES.index((op1, op2, value)) - CMP_CORRESPONDANCES.insert(i+1, (op2, op1, value)) -BINARY_BITWISE_OPS = {'and': '&', 'lshift': '<<', 'or': '|', 'rshift': '>>', - 'xor': '^'} -BINARY_OPS = dict(add='+', div='/', floordiv='//', mod='%', mul='*', sub='-', - truediv='/', **BINARY_BITWISE_OPS) -COMMUTATIVE_OPS = ('add', 'mul', 'and', 'or', 'xor') - -def add_extra_comparisons(): - """ - Add the missing comparison operators if they were not explicitly - defined: eq <-> ne and lt <-> le <-> gt <-> ge. - We try to add them in the order defined by the CMP_CORRESPONDANCES - table, thus favouring swapping the arguments over negating the result. - """ - originalentries = {} - for op in CMP_OPS.iterkeys(): - originalentries[op] = getattr(MM, op).signatures() - - for op1, op2, correspondance in CMP_CORRESPONDANCES: - mirrorfunc = getattr(MM, op2) - for types in originalentries[op1]: - t1, t2 = types - if t1 is t2: - if not mirrorfunc.has_signature(types): - functions = getattr(MM, op1).getfunctions(types) - assert len(functions) == 1, ('Automatic' - ' registration of comparison functions' - ' only work when there is a single method for' - ' the operation.') - mirrorfunc.register(correspondance(functions[0]), *types) - - -# ____________________________________________________________ - -W_ANY = W_Root - -class W_Object(W_Root): - "Parent base class for wrapped objects provided by the StdObjSpace." - # Note that not all wrapped objects in the interpreter inherit from - # W_Object. (They inherit from W_Root.) - __slots__ = () - - def __repr__(self): - name = getattr(self, 'name', '') - if not isinstance(name, str): - name = '' - s = '%s(%s)' % (self.__class__.__name__, name) - w_cls = getattr(self, 'w__class__', None) - if w_cls is not None and w_cls is not self: - s += ' instance of %s' % self.w__class__ - return '<%s>' % s - - -class UnwrapError(Exception): - pass - - -class StdObjSpaceMultiMethod(MultiMethodTable): - - def __init__(self, operatorsymbol, arity, specialnames=None, **extras): - """NOT_RPYTHON: cannot create new multimethods dynamically. - """ - MultiMethodTable.__init__(self, arity, W_ANY, - argnames_before = ['space']) - self.operatorsymbol = operatorsymbol - if specialnames is None: - specialnames = [operatorsymbol] - assert isinstance(specialnames, list) - self.specialnames = specialnames # e.g. ['__xxx__', '__rxxx__'] - self.extras = extras - # transform '+' => 'add' etc. - for line in ObjSpace.MethodTable: - realname, symbolname = line[:2] - if symbolname == operatorsymbol: - self.name = realname - break - else: - self.name = operatorsymbol - - if extras.get('general__args__', False): - self.argnames_after = ['__args__'] - if extras.get('varargs_w', False): - self.argnames_after = ['args_w'] - self.argnames_after += extras.get('extra_args', []) - - def install_not_sliced(self, typeorder, baked_perform_call=True): - return self.install(prefix = '__mm_' + self.name, - list_of_typeorders = [typeorder]*self.arity, - baked_perform_call=baked_perform_call) - - def merge_with(self, other): - # Make a new 'merged' multimethod including the union of the two - # tables. In case of conflict, pick the entry from 'self'. - if self.arity != other.arity: - return self # XXX that's the case of '**' - operatorsymbol = '%s_merge_%s' % (self.name, other.name) - assert self.extras == other.extras - mm = StdObjSpaceMultiMethod(operatorsymbol, self.arity, **self.extras) - # - def merge(node1, node2): - assert type(node1) is type(node2) - if isinstance(node1, dict): - d = node1.copy() - d.update(node2) - for key in node1: - if key in node2: - d[key] = merge(node1[key], node2[key]) - return d - else: - assert isinstance(node1, list) - assert node1 - return node1 # pick the entry from 'self' - # - mm.dispatch_tree = merge(self.dispatch_tree, other.dispatch_tree) - return mm - -NOT_MULTIMETHODS = set( - ['delattr', 'delete', 'get', 'id', 'inplace_div', 'inplace_floordiv', - 'inplace_lshift', 'inplace_mod', 'inplace_pow', 'inplace_rshift', - 'inplace_truediv', 'is_', 'set', 'setattr', 'type', 'userdel', - 'isinstance', 'issubtype', 'int', 'ord']) -# XXX should we just remove those from the method table or we're happy -# with just not having multimethods? - -class MM: - """StdObjSpace multimethods""" - - call = StdObjSpaceMultiMethod('call', 1, ['__call__'], - general__args__=True) - init = StdObjSpaceMultiMethod('__init__', 1, general__args__=True) - getnewargs = StdObjSpaceMultiMethod('__getnewargs__', 1) - - # add all regular multimethods here - for _name, _symbol, _arity, _specialnames in ObjSpace.MethodTable: - if _name not in locals() and _name not in NOT_MULTIMETHODS: - mm = StdObjSpaceMultiMethod(_symbol, _arity, _specialnames) - locals()[_name] = mm - del mm - - pow.extras['defaults'] = (None,) diff --git a/pypy/objspace/std/multimethod.py b/pypy/objspace/std/multimethod.py deleted file mode 100644 --- a/pypy/objspace/std/multimethod.py +++ /dev/null @@ -1,972 +0,0 @@ - -from rpython.tool.sourcetools import compile2 - -# This provide two compatible implementations of "multimethods". A -# multimethod is a callable object which chooses and calls a real -# function from a table of pre-registered functions. The choice depends -# on the '__class__' of all arguments. For example usages see -# test_multimethod. - -# These multimethods support delegation: for each class A we must -# provide a "typeorder", which is list of pairs (B, converter) where B -# is a class and 'converter' is a function that can convert from an -# instance of A to an instance of B. If 'converter' is None it is -# assumed that the instance needs no conversion. The first entry in the -# typeorder of a class A must almost always be (A, None). - -# A slightly non-standard feature of PyPy's multimethods is the way in -# which they interact with normal subclassing. Basically, they don't. -# Suppose that A is a parent class of B. Then a function registered for -# an argument class A only accepts an instance whose __class__ is A, not -# B. To make it accept an instance of B, the typeorder for B must -# contain (A, None). An exception to this strict rule is if C is -# another subclass of A which is not mentioned at all in the typeorder; -# in this case C is considered to be equivalent to A. - - -class FailedToImplement(Exception): - def __new__(cls, *args): - if cls is FailedToImplement: - assert not args, "use FailedToImplementArgs!" - return Exception.__new__(cls, *args) - - def get_w_value(self, space): - return None - - def get_w_type(self, space): - return None - - def __str__(self): - return '' - -class FailedToImplementArgs(FailedToImplement): - def __init__(self, w_type=None, w_value=None): - self.w_type = w_type - self.w_value = w_value - - def get_w_value(self, space): - # convenience: same semantics as with OperationError - return self.w_value - - def get_w_type(self, space): - return self.w_type - - def __str__(self): - return '' % (self.w_type, self.w_value) - - - -def raiseFailedToImplement(): - raise FailedToImplement - - -class MultiMethodTable: - - def __init__(self, arity, root_class, argnames_before=[], argnames_after=[]): - """NOT_RPYTHON: cannot create new multimethods dynamically. - MultiMethod-maker dispatching on exactly 'arity' arguments. - """ - if arity < 1: - raise ValueError, "multimethods cannot dispatch on nothing" - self.arity = arity - self.root_class = root_class - self.dispatch_tree = {} - self.argnames_before = list(argnames_before) - self.argnames_after = list(argnames_after) - - def register(self, function, *types, **kwds): - assert len(types) == self.arity - assert kwds.keys() == [] or kwds.keys() == ['order'] - order = kwds.get('order', 0) - node = self.dispatch_tree - for type in types[:-1]: - node = node.setdefault(type, {}) - lst = node.setdefault(types[-1], []) - if order >= len(lst): - lst += [None] * (order+1 - len(lst)) - assert lst[order] is None, "duplicate function for %r@%d" % ( - types, order) - lst[order] = function - - def install(self, prefix, list_of_typeorders, baked_perform_call=True, - base_typeorder=None, installercls=None): - "NOT_RPYTHON: initialization-time only" - assert len(list_of_typeorders) == self.arity - installercls = installercls or Installer - installer = installercls(self, prefix, list_of_typeorders, - baked_perform_call=baked_perform_call, - base_typeorder=base_typeorder) - return installer.install() - - def install_if_not_empty(self, prefix, list_of_typeorders, - base_typeorder=None, installercls=None): - "NOT_RPYTHON: initialization-time only" - assert len(list_of_typeorders) == self.arity - installercls = installercls or Installer - installer = installercls(self, prefix, list_of_typeorders, - base_typeorder=base_typeorder) - if installer.is_empty(): - return None - else: - return installer.install() - - - - # ____________________________________________________________ - # limited dict-like interface to the dispatch table - - def getfunctions(self, types): - assert len(types) == self.arity - node = self.dispatch_tree - for type in types: - node = node[type] - return [fn for fn in node if fn is not None] - - def has_signature(self, types): - try: - self.getfunctions(types) - except KeyError: - return False - else: - return True - - def signatures(self): - """NOT_RPYTHON""" - result = [] - def enum_keys(types_so_far, node): - for type, subnode in node.items(): - next_types = types_so_far+(type,) - if isinstance(subnode, dict): - enum_keys(next_types, subnode) - else: - assert len(next_types) == self.arity - result.append(next_types) - enum_keys((), self.dispatch_tree) - return result - -# ____________________________________________________________ -# Installer version 1 - -class InstallerVersion1: - """NOT_RPYTHON""" - - instance_counter = 0 - - mmfunccache = {} - - prefix_memo = {} - - def __init__(self, multimethod, prefix, list_of_typeorders, - baked_perform_call=True, base_typeorder=None): - self.__class__.instance_counter += 1 - self.multimethod = multimethod - # avoid prefix clashes, user code should supply different prefixes - # itself for nice names in tracebacks - base_prefix = prefix - n = 1 - while prefix in self.prefix_memo: - n += 1 - prefix = "%s%d" % (base_prefix, n) - self.prefix = prefix - self.prefix_memo[prefix] = 1 - self.list_of_typeorders = list_of_typeorders - self.check_typeorders() - self.subtree_cache = {} - self.to_install = [] - self.non_empty = self.build_tree([], multimethod.dispatch_tree) - - self.baked_perform_call = baked_perform_call - - if self.non_empty: - perform = [(None, prefix, 0)] - else: - perform = [] - - self.perform_call = self.build_function(None, prefix+'_perform_call', - None, perform) - - def check_typeorders(self): - # xxx we use a '__'-separated list of the '__name__' of the types - # in build_single_method(), so types with the same __name__ or - # with '__' in them would obscurely break this logic - for typeorder in self.list_of_typeorders: - for type in typeorder: - assert '__' not in type.__name__, ( - "avoid '__' in the name of %r" % (type,)) - names = dict.fromkeys([type.__name__ for type in typeorder]) - assert len(names) == len(typeorder), ( - "duplicate type.__name__ in %r" % (typeorder,)) - - def is_empty(self): - return not self.non_empty - - def install(self): - #f = open('LOGFILE', 'a') - #print >> f, '_'*60 - #import pprint - #pprint.pprint(self.list_of_typeorders, f) - - def class_key(cls): - "Returns an object such that class_key(subcls) > class_key(cls)." - return len(cls.__mro__) - - # Sort 'to_install' so that base classes come first, which is - # necessary for the 'parentfunc' logic in the loop below to work. - # Moreover, 'to_install' can contain two functions with the same - # name for the root class: the default fallback one and the real - # one. So we have to sort the real one just after the default one - # so that the default one gets overridden. - def key(target, funcname, func, source, fallback): - if target is None: - return () - return (class_key(target), not fallback) - self.to_install.sort(lambda a, b: cmp(key(*a), key(*b))) - - for target, funcname, func, source, fallback in self.to_install: - if target is not None: - # If the parent class provides a method of the same - # name which is actually the same 'func', we don't need - # to install it again. Useful with fallback functions. - parentfunc = getattr(target, funcname, None) - parentfunc = getattr(parentfunc, 'im_func', None) - if parentfunc is func: - continue - #print >> f, target.__name__, funcname - #if source: - # print >> f, source - #else: - # print >> f, '*\n' - setattr(target, funcname, func) - #f.close() - return self.perform_call - - def build_tree(self, types_so_far, dispatch_node): - key = tuple(types_so_far) - if key in self.subtree_cache: - return self.subtree_cache[key] - non_empty = False - typeorder = self.list_of_typeorders[len(types_so_far)] - for next_type in typeorder: - if self.build_single_method(typeorder, types_so_far, next_type, - dispatch_node): - non_empty = True - self.subtree_cache[key] = non_empty - return non_empty - - def build_single_method(self, typeorder, types_so_far, next_type, - dispatch_node): - funcname = '__'.join([self.prefix] + [t.__name__ for t in types_so_far]) - - order = typeorder[next_type] - #order = [(next_type, None)] + order - - things_to_call = [] - for type, conversion in order: - if type not in dispatch_node: - # there is no possible completion of types_so_far+[type] - # that could lead to a registered function. - continue - match = dispatch_node[type] - if isinstance(match, dict): - if self.build_tree(types_so_far+[type], match): - call = funcname + '__' + type.__name__ - call_selfarg_index = len(types_so_far) + 1 - things_to_call.append((conversion, call, - call_selfarg_index)) - else: - for func in match: # list of functions - if func is not None: - things_to_call.append((conversion, func, None)) - - funcname = intern(funcname) - self.build_function(next_type, funcname, len(types_so_far), - things_to_call) - return bool(things_to_call) - - def build_function(self, target, funcname, func_selfarg_index, - things_to_call): - # support for inventing names for the entries in things_to_call - # which are real function objects instead of strings - miniglobals = {'FailedToImplement': FailedToImplement, '__name__': __name__} - def invent_name(obj): - if isinstance(obj, str): - return obj - name = obj.__name__ - n = 1 - while name in miniglobals: - n += 1 - name = '%s%d' % (obj.__name__, n) - miniglobals[name] = obj - return name - - funcargs = ['arg%d' % i for i in range(self.multimethod.arity)] - - bodylines = [] - for conversion, call, call_selfarg_index in things_to_call: - callargs = funcargs[:] - if conversion is not None: - to_convert = func_selfarg_index - convert_callargs = (self.multimethod.argnames_before + - [callargs[to_convert]]) - callargs[to_convert] = '%s(%s)' % ( - invent_name(conversion), ', '.join(convert_callargs)) - callname = invent_name(call) - if call_selfarg_index is not None: - # fallback on root_class - self.build_function(self.multimethod.root_class, - callname, call_selfarg_index, []) - callname = '%s.%s' % (callargs.pop(call_selfarg_index), callname) - callargs = (self.multimethod.argnames_before + - callargs + self.multimethod.argnames_after) - bodylines.append('return %s(%s)' % (callname, ', '.join(callargs))) - - fallback = False - if not bodylines: - miniglobals['raiseFailedToImplement'] = raiseFailedToImplement - bodylines = ['return raiseFailedToImplement()'] - fallback = True - # NB. make sure that there is only one fallback function object, - # i.e. the key used in the mmfunccache below is always the same - # for all functions with the same name and an empty bodylines. - - # protect all lines apart from the last one by a try:except: - for i in range(len(bodylines)-2, -1, -1): - bodylines[i:i+1] = ['try:', - ' ' + bodylines[i], - 'except FailedToImplement:', - ' pass'] - - if func_selfarg_index is not None: - selfargs = [funcargs.pop(func_selfarg_index)] - else: - selfargs = [] - funcargs = (selfargs + self.multimethod.argnames_before + - funcargs + self.multimethod.argnames_after) - - if target is None and not self.baked_perform_call: - return funcargs, bodylines[0][len('return '):], miniglobals, fallback - - # indent mode - bodylines = [' ' + line for line in bodylines] - - bodylines.insert(0, 'def %s(%s):' % (funcname, ', '.join(funcargs))) - bodylines.append('') - source = '\n'.join(bodylines) - - # XXX find a better place (or way) to avoid duplicate functions - l = miniglobals.items() - l.sort() - l = tuple(l) - key = (source, l) - try: - func = self.mmfunccache[key] - except KeyError: - exec compile2(source) in miniglobals - func = miniglobals[funcname] - self.mmfunccache[key] = func - #else: - # print "avoided duplicate function", func - self.to_install.append((target, funcname, func, source, fallback)) - return func - -# ____________________________________________________________ -# Installer version 2 - -class MMDispatcher(object): - """NOT_RPYTHON - Explicit dispatcher class. The __call__ and dispatch() methods - are only present for documentation purposes. The InstallerVersion2 - uses the expressions() method to precompute fast RPython-friendly - dispatch tables. - """ - _revcache = None - - def __init__(self, multimethod, list_of_typeorders): - self.multimethod = multimethod - self.list_of_typeorders = list_of_typeorders - - def __call__(self, *args): - # for testing only: this is slow - i = len(self.multimethod.argnames_before) - j = i + self.multimethod.arity - k = j + len(self.multimethod.argnames_after) - assert len(args) == k - prefixargs = args[:i] - dispatchargs = args[i:j] - suffixargs = args[j:] - return self.dispatch([x.__class__ for x in dispatchargs], - prefixargs, - dispatchargs, - suffixargs) - - def dispatch(self, argtypes, prefixargs, args, suffixargs): - # for testing only: this is slow - def expr(v): - if isinstance(v, Call): - return v.function(*[expr(w) for w in v.arguments]) - else: - return v - # XXX this is incomplete: for each type in argtypes but not - # in the typeorder, we should look for the first base class - # that is in the typeorder. - e = None - for v in self.expressions(argtypes, prefixargs, args, suffixargs): - try: - return expr(v) - except FailedToImplement, e: - pass - else: - raise e or FailedToImplement() - - def expressions(self, argtypes, prefixargs, args, suffixargs): - """Lists the possible expressions that call the appropriate - function for the given argument types. Each expression is a Call - object. The intent is that at run-time the first Call that doesn't - cause FailedToImplement to be raised is the good one. - """ - prefixargs = tuple(prefixargs) - suffixargs = tuple(suffixargs) - - def walktree(node, args_so_far): - if isinstance(node, list): - for func in node: - if func is not None: - result.append(Call(func, prefixargs + - args_so_far + - suffixargs)) - else: - index = len(args_so_far) - typeorder = self.list_of_typeorders[index] - next_type = argtypes[index] - for target_type, converter in typeorder[next_type]: - if target_type not in node: - continue - next_arg = args[index] - if converter: - next_arg = Call(converter, prefixargs + (next_arg,)) - walktree(node[target_type], args_so_far + (next_arg,)) - - result = [] - walktree(self.multimethod.dispatch_tree, ()) - return result - - def anychance(self, typesprefix): - # is there any chance that a list of types starting with typesprefix - # could lead to a successful dispatch? - # (START-UP TIME OPTIMIZATION ONLY) - if self._revcache is None: - - def build_tree(types_so_far, dispatch_node): - non_empty = False - typeorder = self.list_of_typeorders[len(types_so_far)] - for next_type in typeorder: - if build_single_method(typeorder, types_so_far, next_type, - dispatch_node): - non_empty = True - if non_empty: - self._revcache[types_so_far] = True - return non_empty - - def build_single_method(typeorder, types_so_far, next_type, - dispatch_node): - order = typeorder[next_type] - things_to_call = False - for type, conversion in order: - if type not in dispatch_node: - # there is no possible completion of - # types_so_far+[type] that could lead to a - # registered function. - continue - match = dispatch_node[type] - if isinstance(match, dict): - if build_tree(types_so_far+(next_type,), match): - things_to_call = True - elif match: - things_to_call = True - return things_to_call - - self._revcache = {} - build_tree((), self.multimethod.dispatch_tree) - return tuple(typesprefix) in self._revcache - - -class Call(object): - """ Represents a call expression. - The arguments may themselves be Call objects. - """ - def __init__(self, function, arguments): - self.function = function - self.arguments = arguments - - -class CompressedArray(object): - def __init__(self, null_value): - self.null_value = null_value - self.items = [null_value] - - def ensure_length(self, newlen): - if newlen > len(self.items): - self.items.extend([self.null_value] * (newlen - len(self.items))) - - def insert_subarray(self, array): - # insert the given array of numbers into the indexlist, - # allowing null values to become non-null - if array.count(self.null_value) == len(array): - return 0 - test = 1 - while True: - self.ensure_length(test+len(array)) - for i in xrange(len(array)): - if not (array[i] == self.items[test+i] or - array[i] == self.null_value or - self.items[test+i] == self.null_value): - break - else: - # success - for i in range(len(array)): - if array[i] != self.null_value: - self.items[test+i] = array[i] - return test - test += 1 - - def _freeze_(self): - return True - - -class MRDTable(object): - # Multi-Method Dispatch Using Multiple Row Displacement, - # Candy Pang, Wade Holst, Yuri Leontiev, and Duane Szafron - # University of Alberta, Edmonton AB T6G 2H1 Canada - # can be found on http://web.cs.ualberta.ca/~yuri/publ.htm - - Counter = 0 - - def __init__(self, list_of_types): - self.id = MRDTable.Counter - MRDTable.Counter += 1 - self.list_of_types = list_of_types - self.typenum = dict(zip(list_of_types, range(len(list_of_types)))) - self.attrname = '__mrd%d_typenum' % self.id - for t1, num in self.typenum.items(): - setattr(t1, self.attrname, num) - self.indexarray = CompressedArray(0) - - def get_typenum(self, cls): - return self.typenum[cls] - - def is_anti_range(self, typenums): - # NB. typenums should be sorted. Returns (a, b) if typenums contains - # at least half of all typenums and its complement is range(a, b). - # Returns (None, None) otherwise. Returns (0, 0) if typenums contains - # everything. - n = len(self.list_of_types) - if len(typenums) <= n // 2: - return (None, None) - typenums = dict.fromkeys(typenums) - complement = [typenum for typenum in range(n) - if typenum not in typenums] - if not complement: - return (0, 0) - a = min(complement) - b = max(complement) + 1 - if complement == range(a, b): - return (a, b) - else: - return (None, None) - - def normalize_length(self, next_array): - # make sure that the indexarray is not smaller than any funcarray - self.indexarray.ensure_length(len(next_array.items)) - - -def invent_name(miniglobals, obj): - if isinstance(obj, str): - return obj - name = obj.__name__ - n = 1 - while name in miniglobals: - n += 1 - name = '%s%d' % (obj.__name__, n) - miniglobals[name] = obj - return name - - -class FuncEntry(object): - - def __init__(self, bodylines, miniglobals, fallback): - self.body = '\n '.join(bodylines) - self.miniglobals = miniglobals - self.fallback = fallback - self.possiblenames = [] - self.typetree = {} - self._function = None - - def key(self): - lst = self.miniglobals.items() - lst.sort() - return self.body, tuple(lst) - - def get_function_name(self): - # pick a name consistently based on self.possiblenames - length = min([len(parts) for parts in self.possiblenames]) - result = [] - for i in range(length): - choices = {} - for parts in self.possiblenames: - choices[parts[i]] = True - parts = choices.keys() - res = str(len(parts)) - for part in parts: - if type(part) is str: # there is a string at this pos - if '0_fail' in choices: - res = '0_fail' - elif len(parts) == 1: - res = part - break - else: - # only types at this location, try to find a common base - basecls = parts[0] - for cls in parts[1:]: - if issubclass(basecls, cls): - basecls = cls - for cls in parts[1:]: - if not issubclass(cls, basecls): - break # no common base - else: - res = basecls.__name__ - result.append(res) - return '_'.join(result) - - def make_function(self, fnargs, nbargs_before, mrdtable): - if self._function is not None: - return self._function - name = self.get_function_name() - self.compress_typechecks(mrdtable) - checklines = self.generate_typechecks(mrdtable, fnargs[nbargs_before:]) - if not checklines: - body = self.body - else: - checklines.append(self.body) - body = '\n '.join(checklines) - source = 'def %s(%s):\n %s\n' % (name, ', '.join(fnargs), body) - self.debug_dump(source) - exec compile2(source) in self.miniglobals - self._function = self.miniglobals[name] - return self._function - - def debug_dump(self, source): - if 0: # for debugging the generated mm sources - name = self.get_function_name() - f = open('/tmp/mm-source/%s' % name, 'a') - for possiblename in self.possiblenames: - print >> f, '#', - for part in possiblename: - print >> f, getattr(part, '__name__', part), - print >> f - print >> f - print >> f, source - f.close() - - def register_valid_types(self, types): - node = self.typetree - for t1 in types[:-1]: - if node is True: - return - node = node.setdefault(t1, {}) - if node is True: - return - node[types[-1]] = True - - def no_typecheck(self): - self.typetree = True - - def compress_typechecks(self, mrdtable): - def full(node): - if node is True: - return 1 - fulls = 0 - for key, subnode in node.items(): - if full(subnode): - node[key] = True - fulls += 1 - if fulls == types_total: - return 1 - return 0 - - types_total = len(mrdtable.list_of_types) - if full(self.typetree): - self.typetree = True - - def generate_typechecks(self, mrdtable, args): - attrname = mrdtable.attrname - possibletypes = [{} for _ in args] - any_type_is_ok = [False for _ in args] - - def generate(node, level=0): - # this generates type-checking code like the following: - # - # _argtypenum = arg1.__typenum - # if _argtypenum == 5: - # ... - # elif _argtypenum == 6 or _argtypenum == 8: - # ... - # else: - # _failedtoimplement = True - # - # or, in the common particular case of an "anti-range", we optimize it to: - # - # _argtypenum = arg1.__typenum - # if _argtypenum < 5 or _argtypenum >= 10: - # ... - # else: - # _failedtoimplement = True - # - result = [] - indent = ' '*level - if node is True: - for i in range(level, len(args)): - any_type_is_ok[i] = True - result.append('%s_failedtoimplement = False' % (indent,)) - return result - if not node: - result.append('%s_failedtoimplement = True' % (indent,)) - return result - result.append('%s_argtypenum = %s.%s' % (indent, args[level], - attrname)) - cases = {} - for key, subnode in node.items(): - possibletypes[level][key] = True - casebody = tuple(generate(subnode, level+1)) - typenum = mrdtable.get_typenum(key) - cases.setdefault(casebody, []).append(typenum) - for casebody, typenums in cases.items(): - typenums.sort() From noreply at buildbot.pypy.org Sun Nov 2 23:41:07 2014 From: noreply at buildbot.pypy.org (rlamy) Date: Sun, 2 Nov 2014 23:41:07 +0100 (CET) Subject: [pypy-commit] pypy online-transforms: change interface of ClassDesc.add_source_attribute() Message-ID: <20141102224107.B1B4F1C100F@cobra.cs.uni-duesseldorf.de> Author: Ronan Lamy Branch: online-transforms Changeset: r74328:4539281f65b9 Date: 2014-11-01 15:29 +0000 http://bitbucket.org/pypy/pypy/changeset/4539281f65b9/ Log: change interface of ClassDesc.add_source_attribute() diff --git a/rpython/annotator/description.py b/rpython/annotator/description.py --- a/rpython/annotator/description.py +++ b/rpython/annotator/description.py @@ -482,7 +482,8 @@ if cls not in classdef.FORCE_ATTRIBUTES_INTO_CLASSES: self.all_enforced_attrs = [] # no attribute allowed - def add_source_attribute(self, name, value, mixin=False): + def add_source_attribute(self, cls, name, mixin=False): + value = cls.__dict__[name] if isinstance(value, types.FunctionType): # for debugging if not hasattr(value, 'class_'): @@ -548,14 +549,14 @@ for base in reversed(mro): assert is_mixin(base), ( "Mixin class %r has non mixin base class %r" % (mixins, base)) - for name, value in base.__dict__.items(): + for name in base.__dict__: if name in skip: continue - self.add_source_attribute(name, value, mixin=True) + self.add_source_attribute(base, name, mixin=True) def add_sources_for_class(self, cls): - for name, value in cls.__dict__.items(): - self.add_source_attribute(name, value) + for name in cls.__dict__: + self.add_source_attribute(cls, name) def getallclassdefs(self): return self._classdefs.values() @@ -712,7 +713,7 @@ # check whether there is a new attribute cls = self.pyobj if name in cls.__dict__: - self.add_source_attribute(name, cls.__dict__[name]) + self.add_source_attribute(cls, name) if name in self.classdict: return self return None From noreply at buildbot.pypy.org Sun Nov 2 23:41:08 2014 From: noreply at buildbot.pypy.org (rlamy) Date: Sun, 2 Nov 2014 23:41:08 +0100 (CET) Subject: [pypy-commit] pypy online-transforms: Use normalised class attributes in add_source_attribute(), instead of raw values from the class dict Message-ID: <20141102224108.DC5461C100F@cobra.cs.uni-duesseldorf.de> Author: Ronan Lamy Branch: online-transforms Changeset: r74329:9c33a580d088 Date: 2014-11-02 22:40 +0000 http://bitbucket.org/pypy/pypy/changeset/9c33a580d088/ Log: Use normalised class attributes in add_source_attribute(), instead of raw values from the class dict diff --git a/rpython/annotator/description.py b/rpython/annotator/description.py --- a/rpython/annotator/description.py +++ b/rpython/annotator/description.py @@ -8,7 +8,7 @@ from rpython.tool.sourcetools import valid_identifier, func_with_new_name from rpython.tool.pairtype import extendabletype from rpython.annotator.model import AnnotatorError -from rpython.tool.descriptor import normalize_method +from rpython.tool.descriptor import normalize_method, InstanceMethod class CallFamily(object): """A family of Desc objects that could be called from common call sites. @@ -483,25 +483,35 @@ self.all_enforced_attrs = [] # no attribute allowed def add_source_attribute(self, cls, name, mixin=False): - value = cls.__dict__[name] - if isinstance(value, types.FunctionType): - # for debugging - if not hasattr(value, 'class_'): - value.class_ = self.pyobj + try: + value = getattr(cls, name) + except: + if name in cls.__dict__: + return # ignore misbehaving descriptors and the like + raise + try: + value = normalize_method(value) + except ValueError: + pass + if isinstance(value, InstanceMethod): + func = value.im_func + if isinstance(func, types.FunctionType): + if not hasattr(func, 'class_'): + func.class_ = cls if self.specialize: # make a custom funcdesc that specializes on its first # argument (i.e. 'self'). from rpython.annotator.specialize import specialize_argtype def argtype0(funcdesc, args_s): return specialize_argtype(funcdesc, args_s, 0) - funcdesc = FunctionDesc(self.bookkeeper, value, + funcdesc = FunctionDesc(self.bookkeeper, func, specializer=argtype0) self.classdict[name] = funcdesc return if mixin: # make a new copy of the FunctionDesc for this class, # but don't specialize further for all subclasses - funcdesc = FunctionDesc(self.bookkeeper, value) + funcdesc = FunctionDesc(self.bookkeeper, func) self.classdict[name] = funcdesc return # NB. if value is, say, AssertionError.__init__, then we @@ -509,10 +519,9 @@ # that the py lib has its own AssertionError.__init__ which # is of type FunctionType. But bookkeeper.immutablevalue() # will do the right thing in s_get_value(). - if isinstance(value, staticmethod) and mixin: + if isinstance(value, types.FunctionType) and mixin: # make a new copy of staticmethod - func = value.__get__(42) - value = staticmethod(func_with_new_name(func, func.__name__)) + value = func_with_new_name(value, value.__name__) if type(value) in MemberDescriptorTypes: # skip __slots__, showing up in the class as 'member' objects @@ -521,10 +530,6 @@ # pretend that built-in exceptions have no __init__, # unless explicitly specified in builtin.py from rpython.annotator.builtin import BUILTIN_ANALYZERS - try: - value = normalize_method(value) - except ValueError: - pass if value not in BUILTIN_ANALYZERS: return self.classdict[name] = Constant(value) @@ -686,10 +691,11 @@ obj = self.classdict[name] if isinstance(obj, Constant): value = obj.value - if isinstance(value, staticmethod): # special case - value = value.__get__(42) + if isinstance(value, types.FunctionType): # staticmethod classdef = None # don't bind - elif isinstance(value, classmethod): + elif (isinstance(value, InstanceMethod) and + value.im_self is not None): + # This is a method bound to the class, i.e. a classmethod raise AnnotatorError("classmethods are not supported") s_value = self.bookkeeper.immutablevalue(value) if classdef is not None: From noreply at buildbot.pypy.org Mon Nov 3 11:48:45 2014 From: noreply at buildbot.pypy.org (arigo) Date: Mon, 3 Nov 2014 11:48:45 +0100 (CET) Subject: [pypy-commit] pypy default: Issue #1920: test and fix Message-ID: <20141103104845.D24FD1D23E0@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r74330:85a12a7d5a34 Date: 2014-11-03 11:47 +0100 http://bitbucket.org/pypy/pypy/changeset/85a12a7d5a34/ Log: Issue #1920: test and fix diff --git a/lib_pypy/grp.py b/lib_pypy/grp.py --- a/lib_pypy/grp.py +++ b/lib_pypy/grp.py @@ -66,11 +66,12 @@ @builtinify def getgrnam(name): - if not isinstance(name, str): + if not isinstance(name, basestring): raise TypeError("expected string") + name = str(name) res = libc.getgrnam(name) if not res: - raise KeyError(name) + raise KeyError("'getgrnam(): name not found: %s'" % name) return _group_from_gstruct(res) @builtinify diff --git a/pypy/module/test_lib_pypy/test_grp_extra.py b/pypy/module/test_lib_pypy/test_grp_extra.py --- a/pypy/module/test_lib_pypy/test_grp_extra.py +++ b/pypy/module/test_lib_pypy/test_grp_extra.py @@ -9,7 +9,8 @@ "No grp module on this platform") def test_basic(self): - raises(KeyError, self.grp.getgrnam, "dEkLofcG") + e = raises(KeyError, self.grp.getgrnam, "dEkLofcG") + assert e.value.args[0] == "'getgrnam(): name not found: dEkLofcG'" for name in ["root", "wheel"]: try: g = self.grp.getgrnam(name) @@ -19,6 +20,8 @@ assert 'root' in g.gr_mem or g.gr_mem == [] assert g.gr_name == name assert isinstance(g.gr_passwd, str) # usually just 'x', don't hope :-) + g2 = self.grp.getgrnam(unicode(name)) + assert g2 == g break else: raise From noreply at buildbot.pypy.org Mon Nov 3 15:29:57 2014 From: noreply at buildbot.pypy.org (arigo) Date: Mon, 3 Nov 2014 15:29:57 +0100 (CET) Subject: [pypy-commit] pypy default: Issue 1895: test and fix for the file's locks on multithreaded apps with fork() Message-ID: <20141103142957.923E71D2304@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r74331:99a70ef5eaaf Date: 2014-11-03 15:29 +0100 http://bitbucket.org/pypy/pypy/changeset/99a70ef5eaaf/ Log: Issue 1895: test and fix for the file's locks on multithreaded apps with fork() diff --git a/pypy/interpreter/executioncontext.py b/pypy/interpreter/executioncontext.py --- a/pypy/interpreter/executioncontext.py +++ b/pypy/interpreter/executioncontext.py @@ -32,6 +32,17 @@ self.compiler = space.createcompiler() self.profilefunc = None self.w_profilefuncarg = None + self.thread_disappeared = False # might be set to True after os.fork() + + @staticmethod + def _mark_thread_disappeared(space): + # Called in the child process after os.fork() by interp_posix.py. + # Marks all ExecutionContexts except the current one + # with 'thread_disappeared = True'. + me = space.getexecutioncontext() + for ec in space.threadlocals.getallvalues().values(): + if ec is not me: + ec.thread_disappeared = True def gettopframe(self): return self.topframeref() diff --git a/pypy/module/_file/interp_stream.py b/pypy/module/_file/interp_stream.py --- a/pypy/module/_file/interp_stream.py +++ b/pypy/module/_file/interp_stream.py @@ -34,8 +34,12 @@ # this function runs with the GIL acquired so there is no race # condition in the creation of the lock me = self.space.getexecutioncontext() # used as thread ident - if self.slockowner is me: - return False # already acquired by the current thread + if self.slockowner is not None: + if self.slockowner is me: + return False # already acquired by the current thread + if self.slockowner.thread_disappeared: + self.slockowner = None + self.slock = None try: if self.slock is None: self.slock = self.space.allocate_lock() diff --git a/pypy/module/posix/interp_posix.py b/pypy/module/posix/interp_posix.py --- a/pypy/module/posix/interp_posix.py +++ b/pypy/module/posix/interp_posix.py @@ -10,6 +10,7 @@ from pypy.interpreter.gateway import unwrap_spec from pypy.interpreter.error import OperationError, wrap_oserror, wrap_oserror2 +from pypy.interpreter.executioncontext import ExecutionContext from pypy.module.sys.interp_encoding import getfilesystemencoding @@ -721,6 +722,8 @@ "NOT_RPYTHON" get_fork_hooks(where).append(hook) +add_fork_hook('child', ExecutionContext._mark_thread_disappeared) + @specialize.arg(0) def run_fork_hooks(where, space): for hook in get_fork_hooks(where): diff --git a/pypy/module/test_lib_pypy/test_posix_extra.py b/pypy/module/test_lib_pypy/test_posix_extra.py new file mode 100644 --- /dev/null +++ b/pypy/module/test_lib_pypy/test_posix_extra.py @@ -0,0 +1,40 @@ +import py +import sys, os, subprocess + + +CODE = """ +import sys, os, thread, time + +fd1, fd2 = os.pipe() +f1 = os.fdopen(fd1, 'r', 0) +f2 = os.fdopen(fd2, 'w', 0) + +def f(): + print "thread started" + x = f1.read(1) + assert x == "X" + print "thread exit" + +thread.start_new_thread(f, ()) +time.sleep(0.5) +if os.fork() == 0: # in the child + time.sleep(0.5) + x = f1.read(1) + assert x == "Y" + print "ok!" + sys.exit() + +f2.write("X") # in the parent +f2.write("Y") # in the parent +time.sleep(1.0) +""" + + +def test_thread_fork_file_lock(): + if not hasattr(os, 'fork'): + py.test.skip("requires 'fork'") + output = subprocess.check_output([sys.executable, '-u', '-c', CODE]) + assert output.splitlines() == [ + 'thread started', + 'thread exit', + 'ok!'] From noreply at buildbot.pypy.org Mon Nov 3 17:49:35 2014 From: noreply at buildbot.pypy.org (rlamy) Date: Mon, 3 Nov 2014 17:49:35 +0100 (CET) Subject: [pypy-commit] pypy online-transforms: Do perform_normalizations() at the end of annotation Message-ID: <20141103164935.E55911D2304@cobra.cs.uni-duesseldorf.de> Author: Ronan Lamy Branch: online-transforms Changeset: r74332:50956c5661c5 Date: 2014-11-03 16:00 +0000 http://bitbucket.org/pypy/pypy/changeset/50956c5661c5/ Log: Do perform_normalizations() at the end of annotation diff --git a/rpython/annotator/annrpython.py b/rpython/annotator/annrpython.py --- a/rpython/annotator/annrpython.py +++ b/rpython/annotator/annrpython.py @@ -12,6 +12,7 @@ from rpython.annotator import model as annmodel, signature from rpython.annotator.argument import simple_args from rpython.annotator.bookkeeper import Bookkeeper +from rpython.rtyper.normalizecalls import perform_normalizations import py log = py.log.Producer("annrpython") @@ -313,6 +314,8 @@ graphs[graph] = True for graph in graphs: simplify.eliminate_empty_blocks(graph) + if block_subset is None: + perform_normalizations(self) #___ flowing annotations in blocks _____________________ diff --git a/rpython/rtyper/normalizecalls.py b/rpython/rtyper/normalizecalls.py --- a/rpython/rtyper/normalizecalls.py +++ b/rpython/rtyper/normalizecalls.py @@ -378,6 +378,9 @@ # ____________________________________________________________ def perform_normalizations(annotator): + from rpython.rtyper.exceptiondata import standardexceptions + for cls in standardexceptions: + annotator.bookkeeper.getuniqueclassdef(cls) create_class_constructors(annotator) annotator.frozen += 1 try: diff --git a/rpython/rtyper/rtyper.py b/rpython/rtyper/rtyper.py --- a/rpython/rtyper/rtyper.py +++ b/rpython/rtyper/rtyper.py @@ -26,7 +26,6 @@ attachRuntimeTypeInfo, Primitive) from rpython.rtyper.rmodel import Repr, inputconst, BrokenReprTyperError from rpython.rtyper.typesystem import LowLevelTypeSystem, getfunctionptr -from rpython.rtyper.normalizecalls import perform_normalizations from rpython.rtyper import rclass from rpython.rtyper.rclass import RootClassRepr from rpython.tool.pairtype import pair @@ -167,22 +166,16 @@ def specialize(self, dont_simplify_again=False): """Main entry point: specialize all annotated blocks of the program.""" # specialize depends on annotator simplifications - assert dont_simplify_again in (False, True) # safety check if not dont_simplify_again: self.annotator.simplify() - - # first make sure that all functions called in a group have exactly - # the same signature, by hacking their flow graphs if needed - perform_normalizations(self.annotator) self.exceptiondata.finish(self) # new blocks can be created as a result of specialize_block(), so # we need to be careful about the loop here. self.already_seen = {} self.specialize_more_blocks() - if self.exceptiondata is not None: - self.exceptiondata.make_helpers(self) - self.specialize_more_blocks() # for the helpers just made + self.exceptiondata.make_helpers(self) + self.specialize_more_blocks() # for the helpers just made def getannmixlevel(self): if self.annmixlevel is not None: From noreply at buildbot.pypy.org Mon Nov 3 17:49:37 2014 From: noreply at buildbot.pypy.org (rlamy) Date: Mon, 3 Nov 2014 17:49:37 +0100 (CET) Subject: [pypy-commit] pypy online-transforms: register builtin exceptions with the annotator at the same time as the other builtins Message-ID: <20141103164937.26D201D2304@cobra.cs.uni-duesseldorf.de> Author: Ronan Lamy Branch: online-transforms Changeset: r74333:a5326145a459 Date: 2014-11-03 16:48 +0000 http://bitbucket.org/pypy/pypy/changeset/a5326145a459/ Log: register builtin exceptions with the annotator at the same time as the other builtins diff --git a/rpython/annotator/bookkeeper.py b/rpython/annotator/bookkeeper.py --- a/rpython/annotator/bookkeeper.py +++ b/rpython/annotator/bookkeeper.py @@ -54,7 +54,7 @@ def __setstate__(self, dic): self.__dict__.update(dic) # normal action - delayed_imports() + self.register_builtins() def __init__(self, annotator): self.annotator = annotator @@ -78,7 +78,13 @@ self.needs_generic_instantiate = {} - delayed_imports() + self.register_builtins() + + def register_builtins(self): + import rpython.annotator.builtin # for side-effects + from rpython.annotator.exception import standardexceptions + for cls in standardexceptions: + self.getuniqueclassdef(cls) def enter(self, position_key): """Start of an operation. @@ -629,6 +635,3 @@ def immutablevalue(x): return getbookkeeper().immutablevalue(x) - -def delayed_imports(): - import rpython.annotator.builtin diff --git a/rpython/annotator/exception.py b/rpython/annotator/exception.py new file mode 100644 --- /dev/null +++ b/rpython/annotator/exception.py @@ -0,0 +1,7 @@ +from rpython.rlib import rstackovf + +# the exceptions that can be implicitely raised by some operations +standardexceptions = set([TypeError, OverflowError, ValueError, + ZeroDivisionError, MemoryError, IOError, OSError, StopIteration, KeyError, + IndexError, AssertionError, RuntimeError, UnicodeDecodeError, + UnicodeEncodeError, NotImplementedError, rstackovf._StackOverflow]) diff --git a/rpython/rtyper/exceptiondata.py b/rpython/rtyper/exceptiondata.py --- a/rpython/rtyper/exceptiondata.py +++ b/rpython/rtyper/exceptiondata.py @@ -1,15 +1,9 @@ from rpython.annotator import model as annmodel +from rpython.annotator.exception import standardexceptions from rpython.rtyper.llannotation import SomePtr -from rpython.rlib import rstackovf from rpython.rtyper.rclass import ( ll_issubclass, ll_type, ll_cast_to_object, getclassrepr, getinstancerepr) -# the exceptions that can be implicitely raised by some operations -standardexceptions = set([TypeError, OverflowError, ValueError, - ZeroDivisionError, MemoryError, IOError, OSError, StopIteration, KeyError, - IndexError, AssertionError, RuntimeError, UnicodeDecodeError, - UnicodeEncodeError, NotImplementedError, rstackovf._StackOverflow]) - class UnknownException(Exception): pass @@ -20,7 +14,6 @@ standardexceptions = standardexceptions def __init__(self, rtyper): - self.make_standard_exceptions(rtyper) # (NB. rclass identifies 'Exception' and 'object') r_type = rtyper.rootclass_repr r_instance = getinstancerepr(rtyper, None) @@ -32,11 +25,6 @@ self.lltype_of_exception_value = r_instance.lowleveltype self.rtyper = rtyper - def make_standard_exceptions(self, rtyper): - bk = rtyper.annotator.bookkeeper - for cls in self.standardexceptions: - bk.getuniqueclassdef(cls) - def finish(self, rtyper): bk = rtyper.annotator.bookkeeper for cls in self.standardexceptions: diff --git a/rpython/rtyper/normalizecalls.py b/rpython/rtyper/normalizecalls.py --- a/rpython/rtyper/normalizecalls.py +++ b/rpython/rtyper/normalizecalls.py @@ -378,9 +378,6 @@ # ____________________________________________________________ def perform_normalizations(annotator): - from rpython.rtyper.exceptiondata import standardexceptions - for cls in standardexceptions: - annotator.bookkeeper.getuniqueclassdef(cls) create_class_constructors(annotator) annotator.frozen += 1 try: From noreply at buildbot.pypy.org Mon Nov 3 20:39:19 2014 From: noreply at buildbot.pypy.org (mattip) Date: Mon, 3 Nov 2014 20:39:19 +0100 (CET) Subject: [pypy-commit] pypy.org extradoc: fix broken link (LaarstiQ, Cheery) Message-ID: <20141103193919.DD55B1C003C@cobra.cs.uni-duesseldorf.de> Author: mattip Branch: extradoc Changeset: r552:837da9c0884c Date: 2014-11-03 21:39 +0200 http://bitbucket.org/pypy/pypy.org/changeset/837da9c0884c/ Log: fix broken link (LaarstiQ, Cheery) diff --git a/download.html b/download.html --- a/download.html +++ b/download.html @@ -200,7 +200,7 @@ hg clone https://bitbucket.org/pypy/pypy -

  • Make sure you installed the dependencies. See the list here.

    +

  • Make sure you installed the dependencies. See the list here.

  • Enter the goal directory:

    diff --git a/source/download.txt b/source/download.txt
--- a/source/download.txt
+++ b/source/download.txt
@@ -232,7 +232,7 @@
 
 2. Make sure you **installed the dependencies.**  See the list here__.
 
-   .. __: http://pypy.readthedocs.org/en/latest/getting-started-python.html#translating-the-pypy-python-interpreter
+   .. __: http://pypy.readthedocs.org/en/latest/build.html#install-build-time-dependencies
 
 3. Enter the ``goal`` directory::
 

From noreply at buildbot.pypy.org  Tue Nov  4 17:59:41 2014
From: noreply at buildbot.pypy.org (fijal)
Date: Tue,  4 Nov 2014 17:59:41 +0100 (CET)
Subject: [pypy-commit] extradoc extradoc: check in the graph
Message-ID: <20141104165941.760DF1D38D0@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: extradoc
Changeset: r5452:defba3934c83
Date: 2014-11-04 18:59 +0200
http://bitbucket.org/pypy/extradoc/changeset/defba3934c83/

Log:	check in the graph

diff --git a/blog/draft/iobase.png b/blog/draft/iobase.png
new file mode 100644
index 0000000000000000000000000000000000000000..0b8bedc12421ce39c24931d68f8993da74bb2808
GIT binary patch

[cut]


From noreply at buildbot.pypy.org  Tue Nov  4 19:13:06 2014
From: noreply at buildbot.pypy.org (pjenvey)
Date: Tue,  4 Nov 2014 19:13:06 +0100 (CET)
Subject: [pypy-commit] pypy default: kill dead code
Message-ID: <20141104181306.365291C073F@cobra.cs.uni-duesseldorf.de>

Author: Philip Jenvey 
Branch: 
Changeset: r74334:c940cf052d44
Date: 2014-11-04 10:12 -0800
http://bitbucket.org/pypy/pypy/changeset/c940cf052d44/

Log:	kill dead code

diff --git a/rpython/rtyper/module/ll_os.py b/rpython/rtyper/module/ll_os.py
--- a/rpython/rtyper/module/ll_os.py
+++ b/rpython/rtyper/module/ll_os.py
@@ -22,8 +22,6 @@
 from rpython.rtyper.tool import rffi_platform as platform
 from rpython.rlib import rposix
 from rpython.translator.tool.cbuild import ExternalCompilationInfo
-from rpython.rtyper.lltypesystem.llmemory import itemoffsetof, offsetof
-from rpython.rtyper.lltypesystem.rstr import STR
 from rpython.rlib.objectmodel import specialize
 from rpython.translator import cdir
 
@@ -1001,8 +999,6 @@
                                   [rffi.INT, rffi.VOIDP, rffi.SIZE_T],
                                   rffi.SIZE_T)
 
-        offset = offsetof(STR, 'chars') + itemoffsetof(STR.chars, 0)
-
         def os_read_llimpl(fd, count):
             if count < 0:
                 raise OSError(errno.EINVAL, None)

From noreply at buildbot.pypy.org  Tue Nov  4 20:20:34 2014
From: noreply at buildbot.pypy.org (berkerpeksag)
Date: Tue,  4 Nov 2014 20:20:34 +0100 (CET)
Subject: [pypy-commit] pypy
 berkerpeksag/fix-broken-link-in-readmerst-1415127402066: Fix broken link in
 README.rst.
Message-ID: <20141104192034.66D221D38DE@cobra.cs.uni-duesseldorf.de>

Author: Berker Peksag 
Branch: berkerpeksag/fix-broken-link-in-readmerst-1415127402066
Changeset: r74335:822ced144305
Date: 2014-11-04 18:56 +0000
http://bitbucket.org/pypy/pypy/changeset/822ced144305/

Log:	Fix broken link in README.rst.

diff --git a/README.rst b/README.rst
--- a/README.rst
+++ b/README.rst
@@ -37,4 +37,4 @@
 to use virtualenv with the resulting pypy-c as the interpreter; you can
 find more details about various installation schemes here:
 
-http://doc.pypy.org/en/latest/getting-started.html#installing-pypy
+    http://doc.pypy.org/en/latest/install.html

From noreply at buildbot.pypy.org  Tue Nov  4 20:20:35 2014
From: noreply at buildbot.pypy.org (rlamy)
Date: Tue,  4 Nov 2014 20:20:35 +0100 (CET)
Subject: [pypy-commit] pypy default: Merged in
 berkerpeksag/pypy/berkerpeksag/fix-broken-link-in-readmerst-1415127402066
 (pull request #289)
Message-ID: <20141104192035.DA5DE1D38DE@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy 
Branch: 
Changeset: r74336:04f76bbfd718
Date: 2014-11-04 19:20 +0000
http://bitbucket.org/pypy/pypy/changeset/04f76bbfd718/

Log:	Merged in berkerpeksag/pypy/berkerpeksag/fix-broken-link-in-
	readmerst-1415127402066 (pull request #289)

	Fix broken link in README.rst.

diff --git a/README.rst b/README.rst
--- a/README.rst
+++ b/README.rst
@@ -37,4 +37,4 @@
 to use virtualenv with the resulting pypy-c as the interpreter; you can
 find more details about various installation schemes here:
 
-http://doc.pypy.org/en/latest/getting-started.html#installing-pypy
+    http://doc.pypy.org/en/latest/install.html

From noreply at buildbot.pypy.org  Tue Nov  4 21:01:11 2014
From: noreply at buildbot.pypy.org (amauryfa)
Date: Tue,  4 Nov 2014 21:01:11 +0100 (CET)
Subject: [pypy-commit] pypy more-rposix: A branch to move C bindings from
 ll_os.py to rlib/rposix.py
Message-ID: <20141104200111.07B0F1C08E7@cobra.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc 
Branch: more-rposix
Changeset: r74337:c05643a703fd
Date: 2014-11-04 15:43 +0100
http://bitbucket.org/pypy/pypy/changeset/c05643a703fd/

Log:	A branch to move C bindings from ll_os.py to rlib/rposix.py


From noreply at buildbot.pypy.org  Tue Nov  4 21:01:12 2014
From: noreply at buildbot.pypy.org (amauryfa)
Date: Tue,  4 Nov 2014 21:01:12 +0100 (CET)
Subject: [pypy-commit] pypy more-rposix: Start moving functions: os.dup,
	os.dup2
Message-ID: <20141104200112.4063F1C08E7@cobra.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc 
Branch: more-rposix
Changeset: r74338:e2c7497831fa
Date: 2014-11-04 20:59 +0100
http://bitbucket.org/pypy/pypy/changeset/e2c7497831fa/

Log:	Start moving functions: os.dup, os.dup2

diff --git a/rpython/rlib/objectmodel.py b/rpython/rlib/objectmodel.py
--- a/rpython/rlib/objectmodel.py
+++ b/rpython/rlib/objectmodel.py
@@ -286,6 +286,17 @@
     return Constant(True)
 
 
+def register_replacement_for(replaced_function):
+    """Decorator that causes RPython to replace the function passed as parameter
+    with the function being defined."""
+    def wrap(func):
+        @register_flow_sc(replaced_function)
+        def sc_redirected_function(ctx, *args_w):
+            return ctx.appcall(func, *args_w)
+        return func
+    return wrap
+
+
 def keepalive_until_here(*values):
     pass
 
diff --git a/rpython/rlib/rposix.py b/rpython/rlib/rposix.py
--- a/rpython/rlib/rposix.py
+++ b/rpython/rlib/rposix.py
@@ -1,12 +1,15 @@
 import os
+import sys
 from rpython.rtyper.lltypesystem.rffi import CConstant, CExternVariable, INT
 from rpython.rtyper.lltypesystem import ll2ctypes, rffi
 from rpython.translator.tool.cbuild import ExternalCompilationInfo
 from rpython.rlib.rarithmetic import intmask
-from rpython.rlib.objectmodel import specialize
+from rpython.rlib.objectmodel import specialize, register_replacement_for
 from rpython.rlib import jit
 from rpython.translator.platform import platform
 
+_WIN32 = sys.platform.startswith('win')
+UNDERSCORE_ON_WIN32 = '_' if _WIN32 else ''
 
 class CConstantErrno(CConstant):
     # these accessors are used when calling get_errno() or set_errno()
@@ -127,6 +130,20 @@
         except OSError:
             pass
 
+if _WIN32:
+    includes = ['io.h']
+else:
+    includes = ['unistd.h']
+eci = ExternalCompilationInfo(
+    includes=includes,
+)
+
+def external(name, args, result, **kwds):
+    return rffi.llexternal(name, args, result, compilation_info=eci, **kwds)
+
+c_dup = external(UNDERSCORE_ON_WIN32 + 'dup', [rffi.INT], rffi.INT)
+c_dup2 = external(UNDERSCORE_ON_WIN32 + 'dup2', [rffi.INT, rffi.INT], rffi.INT)
+
 #___________________________________________________________________
 # Wrappers around posix functions, that accept either strings, or
 # instances with a "as_bytes()" method.
@@ -227,3 +244,23 @@
     os_kill = rwin32.os_kill
 else:
     os_kill = os.kill
+
+#___________________________________________________________________
+# Implementation of many posix functions.
+# They usually check the return value and raise an (RPython) OSError
+# with errno.
+
+ at register_replacement_for(os.dup)
+def dup(fd):
+    validate_fd(fd)
+    newfd = c_dup(fd)
+    if newfd < 0:
+        raise OSError(get_errno(), "dup failed")
+    return newfd
+
+ at register_replacement_for(os.dup2)
+def dup2(fd, newfd):
+    validate_fd(fd)
+    error = c_dup2(fd, newfd)
+    if error < 0:
+        raise OSError(get_errno(), "dup2 failed")
diff --git a/rpython/rlib/test/test_rposix.py b/rpython/rlib/test/test_rposix.py
--- a/rpython/rlib/test/test_rposix.py
+++ b/rpython/rlib/test/test_rposix.py
@@ -169,3 +169,10 @@
     def _get_filename(self):
         return (unicode(udir.join('test_open')) +
                 u'\u65e5\u672c.txt') # "Japan"
+
+class TestRegisteredFunctions:
+    def test_dup(self):
+        def f():
+            os.dup(4)
+            os.dup2(5, 6)
+        compile(f, ())
diff --git a/rpython/rtyper/module/ll_os.py b/rpython/rtyper/module/ll_os.py
--- a/rpython/rtyper/module/ll_os.py
+++ b/rpython/rtyper/module/ll_os.py
@@ -346,35 +346,6 @@
                       llimpl=spawnve_llimpl,
                       export_name="ll_os.ll_os_spawnve")
 
-    @registering(os.dup)
-    def register_os_dup(self):
-        os_dup = self.llexternal(UNDERSCORE_ON_WIN32 + 'dup',
-                                 [rffi.INT], rffi.INT)
-
-        def dup_llimpl(fd):
-            rposix.validate_fd(fd)
-            newfd = rffi.cast(lltype.Signed, os_dup(rffi.cast(rffi.INT, fd)))
-            if newfd == -1:
-                raise OSError(rposix.get_errno(), "dup failed")
-            return newfd
-
-        return extdef([int], int, llimpl=dup_llimpl, export_name="ll_os.ll_os_dup")
-
-    @registering(os.dup2)
-    def register_os_dup2(self):
-        os_dup2 = self.llexternal(UNDERSCORE_ON_WIN32 + 'dup2',
-                                  [rffi.INT, rffi.INT], rffi.INT)
-
-        def dup2_llimpl(fd, newfd):
-            rposix.validate_fd(fd)
-            error = rffi.cast(lltype.Signed, os_dup2(rffi.cast(rffi.INT, fd),
-                                             rffi.cast(rffi.INT, newfd)))
-            if error == -1:
-                raise OSError(rposix.get_errno(), "dup2 failed")
-
-        return extdef([int, int], s_None, llimpl=dup2_llimpl,
-                      export_name="ll_os.ll_os_dup2")
-
     @registering_if(os, "getlogin", condition=not _WIN32)
     def register_os_getlogin(self):
         os_getlogin = self.llexternal('getlogin', [], rffi.CCHARP, releasegil=False)

From noreply at buildbot.pypy.org  Wed Nov  5 01:11:48 2014
From: noreply at buildbot.pypy.org (pjenvey)
Date: Wed,  5 Nov 2014 01:11:48 +0100 (CET)
Subject: [pypy-commit] pypy py3k: issue1915: fix peercert(True) to return
	bytes
Message-ID: <20141105001148.E5D291C08E7@cobra.cs.uni-duesseldorf.de>

Author: Philip Jenvey 
Branch: py3k
Changeset: r74339:a1f50860377d
Date: 2014-11-04 16:11 -0800
http://bitbucket.org/pypy/pypy/changeset/a1f50860377d/

Log:	issue1915: fix peercert(True) to return bytes

diff --git a/pypy/module/_ssl/interp_ssl.py b/pypy/module/_ssl/interp_ssl.py
--- a/pypy/module/_ssl/interp_ssl.py
+++ b/pypy/module/_ssl/interp_ssl.py
@@ -622,7 +622,8 @@
                     raise _ssl_seterror(space, self, length)
                 try:
                     # this is actually an immutable bytes sequence
-                    return space.wrap(rffi.charpsize2str(buf_ptr[0], length))
+                    return space.wrapbytes(
+                        rffi.charpsize2str(buf_ptr[0], length))
                 finally:
                     libssl_OPENSSL_free(buf_ptr[0])
         else:
diff --git a/pypy/module/_ssl/test/test_ssl.py b/pypy/module/_ssl/test/test_ssl.py
--- a/pypy/module/_ssl/test/test_ssl.py
+++ b/pypy/module/_ssl/test/test_ssl.py
@@ -181,6 +181,13 @@
         self.s.close()
         del ss; gc.collect()
         
+    def test_peer_certificate(self):
+        import gc, ssl
+        ss = ssl.wrap_socket(self.s)
+        assert ss.getpeercert() == {}
+        assert type(ss.getpeercert(True)) is bytes
+        self.s.close()
+        del ss; gc.collect()
 
 class AppTestConnectedSSL_Timeout(AppTestConnectedSSL):
     # Same tests, with a socket timeout

From noreply at buildbot.pypy.org  Wed Nov  5 10:02:26 2014
From: noreply at buildbot.pypy.org (cfbolz)
Date: Wed,  5 Nov 2014 10:02:26 +0100 (CET)
Subject: [pypy-commit] pypy default: a decorator for inlining functions on
	the rpython level
Message-ID: <20141105090226.DEA451C003C@cobra.cs.uni-duesseldorf.de>

Author: Carl Friedrich Bolz 
Branch: 
Changeset: r74340:a772923edb32
Date: 2014-11-05 10:01 +0100
http://bitbucket.org/pypy/pypy/changeset/a772923edb32/

Log:	a decorator for inlining functions on the rpython level

diff --git a/rpython/rlib/objectmodel.py b/rpython/rlib/objectmodel.py
--- a/rpython/rlib/objectmodel.py
+++ b/rpython/rlib/objectmodel.py
@@ -201,6 +201,11 @@
         return result
     return decorator
 
+def always_inline(func):
+    """ mark the function as to-be-inlined by the RPython optimizations (not
+    the JIT!), no matter its size."""
+    func._always_inline_ = True
+    return func
 
 
 # ____________________________________________________________
diff --git a/rpython/rlib/test/test_objectmodel.py b/rpython/rlib/test/test_objectmodel.py
--- a/rpython/rlib/test/test_objectmodel.py
+++ b/rpython/rlib/test/test_objectmodel.py
@@ -438,6 +438,11 @@
     assert exc.value.message == "f argument 'b' must be of type "
     py.test.raises(TypeError, "f('hello', 'world', 3)")
 
+def test_always_inline():
+    @always_inline
+    def f(a, b, c):
+        return a, b, c
+    assert f._always_inline_ == True
 
 def test_enforceargs_defaults():
     @enforceargs(int, int)

From noreply at buildbot.pypy.org  Wed Nov  5 10:07:11 2014
From: noreply at buildbot.pypy.org (cfbolz)
Date: Wed,  5 Nov 2014 10:07:11 +0100 (CET)
Subject: [pypy-commit] extradoc extradoc: some tweaks. looks good,
	but some more work needed
Message-ID: <20141105090711.265A21C003C@cobra.cs.uni-duesseldorf.de>

Author: Carl Friedrich Bolz 
Branch: extradoc
Changeset: r5453:5ef5d23c25e3
Date: 2014-11-05 10:07 +0100
http://bitbucket.org/pypy/extradoc/changeset/5ef5d23c25e3/

Log:	some tweaks. looks good, but some more work needed

diff --git a/blog/draft/io-improvements.rst b/blog/draft/io-improvements.rst
--- a/blog/draft/io-improvements.rst
+++ b/blog/draft/io-improvements.rst
@@ -1,20 +1,20 @@
 
 Hello everyone!
 
-We're about to wrap up the Warsaw sprint, so I would like to describe some
+We've wrapped up the Warsaw sprint, so I would like to describe some
 branches which have been recently merged and which improved the I/O and the
 GC: `gc_no_cleanup_nursery`_ and `gc-incminimark-pinning`_.
 
 The first branch was started by Wenzhu Man for her Google Summer of Code
-and finished by Maciej Fijalkowski and Armin Rigo.
+and finished by Maciej Fijałkowski and Armin Rigo.
 The PyPy GC works by allocating new objects in the young object
 area (the nursery), simply by incrementing a pointer. After each minor
 collection, the nursery has to be cleaned up. For simplicity, the GC used 
 to do it by zeroing the whole nursery.
 
-This approach has bad effects on cache, since you zero a large
-memory at once and does unnecessary work for things that don't require zeroing
-like large strings. We somehow mitigated the first problem with incremental
+This approach has bad effects on the cache, since you zero a large piece of
+memory at once and do unnecessary work for things that don't require zeroing
+like large strings. We mitigated the first problem somewhat with incremental
 nursery zeroing, but this branch removes the zeroing completely, thus
 improving the string handling and recursive code (since jitframes don't
 requires zeroed memory either). I measured the effect on two examples: 
@@ -27,7 +27,7 @@
 XXXX
 
 The second branch was done by Gregor Wegberg for his master thesis and finished
-by Maciej Fijalkowski and Armin Rigo. Because of the way it works, the PyPy GC from
+by Maciej Fijałkowski and Armin Rigo. Because of the way it works, the PyPy GC from
 time to time moves the objects in memory, meaning that their address can change.
 Therefore, if you want to pass pointers to some external C function (for
 example, write(2) or read(2)), you need to ensure that the objects they are
@@ -37,7 +37,7 @@
 The branch introduce the concept of "pinning", which allows us to inform the
 GC that it is not allowed to move a certain object for a short period of time.
 This introduces a bit of extra complexity
-in the garbage collector, but improves the the I/O performance quite drastically,
+in the garbage collector, but improves the I/O performance quite drastically,
 because we no longer need the extra copy to and from the non-movable buffers.
 
 In `this benchmark`_, which does I/O in a loop,
@@ -51,6 +51,8 @@
 non-zero-nursery and ``3d8fe96dc4d9`` for non-zero-nursery and pinning.
 The benchmarks were run once, since the standard deviation was small.
 
+XXX explain why files are still bad and what we plan to do about it
+
 XXXX
 
 XXX summary

From noreply at buildbot.pypy.org  Wed Nov  5 10:21:03 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Wed,  5 Nov 2014 10:21:03 +0100 (CET)
Subject: [pypy-commit] extradoc extradoc: tweaks
Message-ID: <20141105092103.79C0E1C3570@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: extradoc
Changeset: r5454:44f5129e47c6
Date: 2014-11-05 10:21 +0100
http://bitbucket.org/pypy/extradoc/changeset/44f5129e47c6/

Log:	tweaks

diff --git a/blog/draft/io-improvements.rst b/blog/draft/io-improvements.rst
--- a/blog/draft/io-improvements.rst
+++ b/blog/draft/io-improvements.rst
@@ -31,8 +31,8 @@
 time to time moves the objects in memory, meaning that their address can change.
 Therefore, if you want to pass pointers to some external C function (for
 example, write(2) or read(2)), you need to ensure that the objects they are
-pointing to will not be moved by the GC.
-PyPy 2.4 solves the problem by copying the data into a non-movable buffer, which
+pointing to will not be moved by the GC (e.g. when running a different thread).
+PyPy up to 2.4 solves the problem by copying the data into or from a non-movable buffer, which
 is obviously inefficient.
 The branch introduce the concept of "pinning", which allows us to inform the
 GC that it is not allowed to move a certain object for a short period of time.

From noreply at buildbot.pypy.org  Wed Nov  5 10:52:15 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Wed,  5 Nov 2014 10:52:15 +0100 (CET)
Subject: [pypy-commit] stmgc hashtable: Add a real multithreaded hashtable
 test as a demo for now.
Message-ID: <20141105095215.65BAD1C34AB@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: hashtable
Changeset: r1493:848771cf918f
Date: 2014-11-04 17:49 +0100
http://bitbucket.org/pypy/stmgc/changeset/848771cf918f/

Log:	Add a real multithreaded hashtable test as a demo for now.
	Crashes...

diff --git a/c7/demo/demo_hashtable1.c b/c7/demo/demo_hashtable1.c
new file mode 100644
--- /dev/null
+++ b/c7/demo/demo_hashtable1.c
@@ -0,0 +1,217 @@
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+
+#include "stmgc.h"
+
+#define NUMTHREADS  4
+
+
+typedef TLPREFIX struct node_s node_t;
+typedef TLPREFIX struct dict_s dict_t;
+
+
+struct node_s {
+    struct object_s header;
+    int typeid;
+    intptr_t freevalue;
+};
+
+struct dict_s {
+    struct node_s hdr;
+    stm_hashtable_t *hashtable;
+};
+
+#define TID_NODE       0x01234567
+#define TID_DICT       0x56789ABC
+#define TID_DICTENTRY  0x6789ABCD
+
+
+static sem_t done;
+__thread stm_thread_local_t stm_thread_local;
+
+// global and per-thread-data
+time_t default_seed;
+dict_t *global_dict;
+
+struct thread_data {
+    unsigned int thread_seed;
+};
+__thread struct thread_data td;
+
+
+ssize_t stmcb_size_rounded_up(struct object_s *ob)
+{
+    if (((struct node_s*)ob)->typeid == TID_NODE)
+        return sizeof(struct node_s);
+    if (((struct node_s*)ob)->typeid == TID_DICT)
+        return sizeof(struct dict_s);
+    if (((struct node_s*)ob)->typeid == TID_DICTENTRY)
+        return sizeof(struct stm_hashtable_entry_s);
+    abort();
+}
+
+void stmcb_trace(struct object_s *obj, void visit(object_t **))
+{
+    struct node_s *n;
+    n = (struct node_s*)obj;
+    if (n->typeid == TID_NODE) {
+        return;
+    }
+    if (n->typeid == TID_DICT) {
+        stm_hashtable_tracefn(((struct dict_s *)n)->hashtable, visit);
+        return;
+    }
+    if (n->typeid == TID_DICTENTRY) {
+        object_t **ref = &((struct stm_hashtable_entry_s *)obj)->object;
+        visit(ref);
+        return;
+    }
+    abort();
+}
+
+void stmcb_commit_soon() {}
+long stmcb_obj_supports_cards(struct object_s *obj)
+{
+    return 0;
+}
+void stmcb_trace_cards(struct object_s *obj, void cb(object_t **),
+                       uintptr_t start, uintptr_t stop) {
+    abort();
+}
+void stmcb_get_card_base_itemsize(struct object_s *obj,
+                                  uintptr_t offset_itemsize[2]) {
+    abort();
+}
+
+int get_rand(int max)
+{
+    if (max == 0)
+        return 0;
+    return (int)(rand_r(&td.thread_seed) % (unsigned int)max);
+}
+
+
+void populate_hashtable(int keymin, int keymax)
+{
+    int i;
+    int diff = get_rand(keymax - keymin);
+    for (i = 0; i < keymax - keymin; i++) {
+        int key = keymin + i + diff;
+        if (key >= keymax)
+            key -= (keymax - keymin);
+        object_t *o = stm_allocate(sizeof(struct node_s));
+        ((node_t *)o)->typeid = TID_NODE;
+        ((node_t *)o)->freevalue = key;
+        assert(global_dict->hdr.freevalue == 42);
+        stm_hashtable_write((object_t *)global_dict, global_dict->hashtable,
+                            key, o, &stm_thread_local);
+    }
+}
+
+void setup_thread(void)
+{
+    memset(&td, 0, sizeof(struct thread_data));
+    td.thread_seed = default_seed++;
+}
+
+void *demo_random(void *arg)
+{
+    int threadnum = (uintptr_t)arg;
+    int status;
+    rewind_jmp_buf rjbuf;
+    stm_register_thread_local(&stm_thread_local);
+    stm_rewind_jmp_enterframe(&stm_thread_local, &rjbuf);
+
+    setup_thread();
+
+    volatile int start_count = 0;
+
+    stm_start_transaction(&stm_thread_local);
+    ++start_count;
+    assert(start_count == 1);  // all the writes that follow must not conflict
+    populate_hashtable(1291 * threadnum, 1291 * (threadnum + 1));
+    stm_commit_transaction();
+
+    stm_rewind_jmp_leaveframe(&stm_thread_local, &rjbuf);
+    stm_unregister_thread_local(&stm_thread_local);
+
+    status = sem_post(&done); assert(status == 0);
+    return NULL;
+}
+
+void newthread(void*(*func)(void*), void *arg)
+{
+    pthread_t th;
+    int status = pthread_create(&th, NULL, func, arg);
+    if (status != 0)
+        abort();
+    pthread_detach(th);
+    printf("started new thread\n");
+}
+
+void setup_globals(void)
+{
+    stm_hashtable_t *my_hashtable = stm_hashtable_create();
+    struct dict_s new_templ = {
+        .hdr = {
+            .typeid = TID_DICT,
+            .freevalue = 42,
+        },
+        .hashtable = my_hashtable,
+    };
+
+    stm_start_inevitable_transaction(&stm_thread_local);
+    global_dict = (dict_t *)stm_setup_prebuilt(
+                      (object_t* )(uintptr_t)&new_templ);
+    assert(global_dict->hashtable);
+    stm_commit_transaction();
+}
+
+
+int main(void)
+{
+    int i, status;
+    rewind_jmp_buf rjbuf;
+
+    stm_hashtable_entry_userdata = TID_DICTENTRY;
+
+    /* pick a random seed from the time in seconds.
+       A bit pointless for now... because the interleaving of the
+       threads is really random. */
+    default_seed = time(NULL);
+    printf("running with seed=%lld\n", (long long)default_seed);
+
+    status = sem_init(&done, 0, 0);
+    assert(status == 0);
+
+
+    stm_setup();
+    stm_register_thread_local(&stm_thread_local);
+    stm_rewind_jmp_enterframe(&stm_thread_local, &rjbuf);
+
+    setup_globals();
+
+    for (i = 0; i < NUMTHREADS; i++) {
+        newthread(demo_random, (void *)(uintptr_t)i);
+    }
+
+    for (i=0; i < NUMTHREADS; i++) {
+        status = sem_wait(&done);
+        assert(status == 0);
+        printf("thread finished\n");
+    }
+
+    printf("Test OK!\n");
+
+    stm_rewind_jmp_leaveframe(&stm_thread_local, &rjbuf);
+    stm_unregister_thread_local(&stm_thread_local);
+    stm_teardown();
+
+    return 0;
+}
diff --git a/c7/stm/hashtable.c b/c7/stm/hashtable.c
--- a/c7/stm/hashtable.c
+++ b/c7/stm/hashtable.c
@@ -27,12 +27,7 @@
 */
 
 
-typedef TLPREFIX struct stm_hashtable_entry_s {
-    struct object_s header;
-    uint32_t userdata;
-    uintptr_t index;
-    object_t *object;
-} stm_hashtable_entry_t;
+typedef TLPREFIX struct stm_hashtable_entry_s stm_hashtable_entry_t;
 
 uint32_t stm_hashtable_entry_userdata;
 
diff --git a/c7/stmgc.h b/c7/stmgc.h
--- a/c7/stmgc.h
+++ b/c7/stmgc.h
@@ -542,6 +542,13 @@
 extern uint32_t stm_hashtable_entry_userdata;
 void stm_hashtable_tracefn(stm_hashtable_t *, void (object_t **));
 
+struct stm_hashtable_entry_s {
+    struct object_s header;
+    uint32_t userdata;
+    uintptr_t index;
+    object_t *object;
+};
+
 /* ==================== END ==================== */
 
 #endif
diff --git a/c7/test/support.py b/c7/test/support.py
--- a/c7/test/support.py
+++ b/c7/test/support.py
@@ -342,7 +342,7 @@
             return sizeof(struct myobj_s) + 1 * sizeof(void*);
         }
         if (myobj->type_id == 421418) {    /* hashtable entry */
-            return 8 * 3;
+            return sizeof(struct stm_hashtable_entry_s);
         }
         /* basic case: tid equals 42 plus the size of the object */
         assert(myobj->type_id >= 42 + sizeof(struct myobj_s));
@@ -371,7 +371,7 @@
     }
     if (myobj->type_id == 421418) {
         /* hashtable entry */
-        object_t **ref = ((object_t **)myobj) + 2;
+        object_t **ref = &((struct stm_hashtable_entry_s *)myobj)->object;
         visit(ref);
     }
     if (myobj->type_id < 421420) {

From noreply at buildbot.pypy.org  Wed Nov  5 10:52:16 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Wed,  5 Nov 2014 10:52:16 +0100 (CET)
Subject: [pypy-commit] stmgc hashtable: Fixes
Message-ID: <20141105095216.671CA1C34AB@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: hashtable
Changeset: r1494:d24e8a6c34f3
Date: 2014-11-05 08:26 +0100
http://bitbucket.org/pypy/stmgc/changeset/d24e8a6c34f3/

Log:	Fixes

diff --git a/c7/stm/hashtable.c b/c7/stm/hashtable.c
--- a/c7/stm/hashtable.c
+++ b/c7/stm/hashtable.c
@@ -134,6 +134,9 @@
                                   uintptr_t biggercount,
                                   bool remove_unread)
 {
+    dprintf(("rehash %p to %ld, remove_unread=%d\n",
+             hashtable, biggercount, (int)remove_unread));
+
     size_t size = (offsetof(stm_hashtable_table_t, items)
                    + biggercount * sizeof(stm_hashtable_entry_t *));
     stm_hashtable_table_t *biggertable = malloc(size);
@@ -239,7 +242,7 @@
     if (rc > 6) {
         /* we can only enter here once!  If we allocate stuff, we may
            run the GC, and so 'hashtableobj' might move afterwards. */
-        if (_is_young(hashtableobj)) {
+        if (_is_from_same_transaction(hashtableobj)) {
             entry = (stm_hashtable_entry_t *)
                 stm_allocate(sizeof(stm_hashtable_entry_t));
             entry->userdata = stm_hashtable_entry_userdata;
@@ -247,6 +250,7 @@
             entry->object = NULL;
         }
         else {
+            acquire_privatization_lock();
             char *p = allocate_outside_nursery_large(
                           sizeof(stm_hashtable_entry_t));
             entry = (stm_hashtable_entry_t *)(p - stm_object_pages);
@@ -261,6 +265,7 @@
                 e->index = index;
                 e->object = NULL;
             }
+            release_privatization_lock();
         }
         write_fence();     /* make sure 'entry' is fully initialized here */
         table->items[i] = entry;
@@ -341,9 +346,34 @@
     }
 }
 
+static void _hashtable_minor_trace(object_t **pobj)
+{
+    abort();
+    object_t *obj = *pobj;
+    if (!_is_in_nursery(obj))
+        return;
+
+    TRACE_FOR_MINOR_COLLECTION(pobj);
+
+    obj = *pobj;
+    char *real_obj = REAL_ADDRESS(STM_SEGMENT->segment_base, obj);
+    assert(((struct stm_hashtable_entry_s *)real_obj)->userdata ==
+           stm_hashtable_entry_userdata);
+
+    long j, num = STM_SEGMENT->segment_num;
+    for (j = 0; j <= NB_SEGMENTS; j++) {
+        if (j == num)
+            continue;
+        memcpy(REAL_ADDRESS(get_segment_base(j), obj), real_obj,
+               sizeof(struct stm_hashtable_entry_s));
+    }
+}
+
 void stm_hashtable_tracefn(stm_hashtable_t *hashtable, void trace(object_t **))
 {
-    if (trace == TRACE_FOR_MAJOR_COLLECTION)
+    if (trace == TRACE_FOR_MINOR_COLLECTION)
+        trace = &_hashtable_minor_trace;
+    else if (trace == TRACE_FOR_MAJOR_COLLECTION)
         _stm_compact_hashtable(hashtable);
 
     stm_hashtable_table_t *table;
diff --git a/c7/stm/nursery.c b/c7/stm/nursery.c
--- a/c7/stm/nursery.c
+++ b/c7/stm/nursery.c
@@ -333,6 +333,7 @@
 }
 
 
+#define TRACE_FOR_MINOR_COLLECTION  (&minor_trace_if_young)
 
 static inline void _collect_now(object_t *obj)
 {
@@ -346,7 +347,7 @@
            outside the nursery, possibly forcing nursery objects out and
            adding them to 'objects_pointing_to_nursery' as well. */
         char *realobj = REAL_ADDRESS(STM_SEGMENT->segment_base, obj);
-        stmcb_trace((struct object_s *)realobj, &minor_trace_if_young);
+        stmcb_trace((struct object_s *)realobj, TRACE_FOR_MINOR_COLLECTION);
 
         obj->stm_flags |= GCFLAG_WRITE_BARRIER;
     }

From noreply at buildbot.pypy.org  Wed Nov  5 15:45:24 2014
From: noreply at buildbot.pypy.org (fijal)
Date: Wed,  5 Nov 2014 15:45:24 +0100 (CET)
Subject: [pypy-commit] benchmarks default: fine, add the fibonacci here
Message-ID: <20141105144524.EE8CD1C3570@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: 
Changeset: r281:69152c2aee77
Date: 2014-11-05 16:45 +0200
http://bitbucket.org/pypy/benchmarks/changeset/69152c2aee77/

Log:	fine, add the fibonacci here

diff --git a/own/fib.py b/own/fib.py
new file mode 100644
--- /dev/null
+++ b/own/fib.py
@@ -0,0 +1,31 @@
+
+import sys, time
+
+def fib(n):
+    if n == 0 or n == 1:
+        return 1
+    return fib(n - 1) + fib(n - 2)
+
+def f(n):
+    times = []
+    for k in range(n):
+        t0 = time.time()
+        for i in range(2000):
+            fib(20)
+        times.append(time.time() - t0)
+    return times
+
+def entry_point(argv):
+    import optparse
+    import util
+
+    parser = optparse.OptionParser(
+        usage="%prog [options]",
+        description="Test the performance of the fibonacci benchmark")
+    util.add_standard_options_to(parser)
+    options, args = parser.parse_args(argv)
+    util.run_benchmark(options, options.num_runs, f)
+
+if __name__ == '__main__':
+    entry_point(sys.argv[1:])
+

From noreply at buildbot.pypy.org  Wed Nov  5 15:54:15 2014
From: noreply at buildbot.pypy.org (fijal)
Date: Wed,  5 Nov 2014 15:54:15 +0100 (CET)
Subject: [pypy-commit] extradoc extradoc: work on the blog post
Message-ID: <20141105145415.620451C3570@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: extradoc
Changeset: r5455:63fbacecbb24
Date: 2014-11-05 16:54 +0200
http://bitbucket.org/pypy/extradoc/changeset/63fbacecbb24/

Log:	work on the blog post

diff --git a/blog/draft/io-improvements.rst b/blog/draft/io-improvements.rst
--- a/blog/draft/io-improvements.rst
+++ b/blog/draft/io-improvements.rst
@@ -5,6 +5,9 @@
 branches which have been recently merged and which improved the I/O and the
 GC: `gc_no_cleanup_nursery`_ and `gc-incminimark-pinning`_.
 
+.. _`gc_no_cleanup_nursery`: https://bitbucket.org/pypy/pypy/commits/9e2f7a37c1e2
+.. _`gc-incminimark-pinning`: https://bitbucket.org/pypy/pypy/commits/64017d818038
+
 The first branch was started by Wenzhu Man for her Google Summer of Code
 and finished by Maciej Fijałkowski and Armin Rigo.
 The PyPy GC works by allocating new objects in the young object
@@ -21,10 +24,21 @@
 a recursive implementation of  `fibonacci`_ and `gcbench`_,
 to measure GC performance.
 
+.. _`fibonacci`: https://bitbucket.org/pypy/benchmarks/src/69152c2aee7766051aab15735b0b82a46b82b802/own/fib.py?at=default
+.. _`gcbench`: https://bitbucket.org/pypy/benchmarks/src/69152c2aee7766051aab15735b0b82a46b82b802/own/gcbench.py?at=default
+
 The results for fibonacci and gcbench are below (normalized to cpython
-2.7). Benchmarks were run 50 times each:
+2.7). Benchmarks were run 50 times each (note that the big standard
+deviation comes mostly from the warmup at the beginning, true figures
+are smaller):
 
-XXXX
++----------------+----- ------------+-------------------------+--------------------+
+| benchmark      | CPython          | PyPy 2.4                | PyPy non-zero      |
++----------------+------------------+-------------------------+--------------------+
+| fibonacci      | 4.8+-0.15 (1.0x) | 0.59+-0.07 (8.1x)       | 0.45+-0.07 (10.6x) |
++----------------+------------------+-------------------------+--------------------+
+| gcbench        | 22+-0.36 (1.0x)  | 1.34+-0.28 (16.4x)      | 1.02+-0.15 (21.6x) |
++----------------+------------------+-------------------------+--------------------+
 
 The second branch was done by Gregor Wegberg for his master thesis and finished
 by Maciej Fijałkowski and Armin Rigo. Because of the way it works, the PyPy GC from
@@ -51,8 +65,28 @@
 non-zero-nursery and ``3d8fe96dc4d9`` for non-zero-nursery and pinning.
 The benchmarks were run once, since the standard deviation was small.
 
-XXX explain why files are still bad and what we plan to do about it
-
 XXXX
 
-XXX summary
+`The benchmark`_ consists of looped calls to ``os.read``, ``os.write``
+or ``file.read`` and ``file.write`` using different size strings. For writing
+the strings are freshly constructed. Consult the benchmark for details.
+
+What we can see is that ``os.read`` and ``os.write`` both improved greatly
+and outperforms CPython now for each combination. ``file`` operations are
+a little more tricky, and while those branches improved the situation a bit,
+the improvement is not as drastic as in ``os`` versions.  It really should not
+be the case and it showcases how our ``file`` buffering is inferior to CPython.
+We plan on removing our own buffering and using ``FILE*`` in C in the near future,
+so we should outperform CPython on those too (since our allocations are cheaper).
+If you look carefully in the benchmark, the write function is copied three times.
+This hack is intended to avoid JIT overspecializing the assembler code, which happens
+because the buffering code was written way before the JIT was done. In fact, our buffering
+is hilariously bad, but if stars align correctly it can be JIT-compiled to something
+that's not half bad. Try removing the hack and seeing how the performance of the last
+benchmark drops :-) Again, this hack should be absolutely unnecessary once we remove
+our own buffering, stay tuned for more.
+
+Cheers,
+fijal
+
+.. _`The benchmark`: https://bitbucket.org/pypy/benchmarks/src/69152c2aee7766051aab15735b0b82a46b82b802/io/iobasic.py?at=default

From noreply at buildbot.pypy.org  Wed Nov  5 16:01:45 2014
From: noreply at buildbot.pypy.org (fijal)
Date: Wed,  5 Nov 2014 16:01:45 +0100 (CET)
Subject: [pypy-commit] extradoc extradoc: ups kill a redundant section and
	fix links
Message-ID: <20141105150145.9AD2F1C35F7@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: extradoc
Changeset: r5456:a7a1d8353177
Date: 2014-11-05 17:01 +0200
http://bitbucket.org/pypy/extradoc/changeset/a7a1d8353177/

Log:	ups kill a redundant section and fix links

diff --git a/blog/draft/io-improvements.rst b/blog/draft/io-improvements.rst
--- a/blog/draft/io-improvements.rst
+++ b/blog/draft/io-improvements.rst
@@ -67,10 +67,6 @@
 
 XXXX
 
-`The benchmark`_ consists of looped calls to ``os.read``, ``os.write``
-or ``file.read`` and ``file.write`` using different size strings. For writing
-the strings are freshly constructed. Consult the benchmark for details.
-
 What we can see is that ``os.read`` and ``os.write`` both improved greatly
 and outperforms CPython now for each combination. ``file`` operations are
 a little more tricky, and while those branches improved the situation a bit,
@@ -89,4 +85,4 @@
 Cheers,
 fijal
 
-.. _`The benchmark`: https://bitbucket.org/pypy/benchmarks/src/69152c2aee7766051aab15735b0b82a46b82b802/io/iobasic.py?at=default
+.. _`this benchmark`: https://bitbucket.org/pypy/benchmarks/src/69152c2aee7766051aab15735b0b82a46b82b802/io/iobasic.py?at=default

From noreply at buildbot.pypy.org  Wed Nov  5 16:59:54 2014
From: noreply at buildbot.pypy.org (amauryfa)
Date: Wed,  5 Nov 2014 16:59:54 +0100 (CET)
Subject: [pypy-commit] pypy more-rposix: Fix integration with sandbox.
Message-ID: <20141105155954.181301C3570@cobra.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc 
Branch: more-rposix
Changeset: r74341:d5481860c952
Date: 2014-11-05 14:37 +0100
http://bitbucket.org/pypy/pypy/changeset/d5481860c952/

Log:	Fix integration with sandbox. Sandbox now accepts functions no
	created by ExtFunc.

	The "sandbox_name" looks less magic, more explicit, and easier to
	extend.

diff --git a/rpython/flowspace/operation.py b/rpython/flowspace/operation.py
--- a/rpython/flowspace/operation.py
+++ b/rpython/flowspace/operation.py
@@ -12,11 +12,10 @@
 from rpython.tool.sourcetools import compile2
 from rpython.flowspace.model import (Constant, WrapException, const, Variable,
                                      SpaceOperation)
-from rpython.flowspace.specialcase import register_flow_sc
+from rpython.flowspace.specialcase import register_flow_sc, get_specialcase
 from rpython.annotator.model import (
     SomeTuple, AnnotatorError, read_can_only_throw)
 from rpython.annotator.argument import ArgumentsForTranslation
-from rpython.flowspace.specialcase import SPECIAL_CASES
 
 
 NOT_REALLY_CONST = {
@@ -570,11 +569,8 @@
         w_callable, args_w = self.args[0], self.args[1:]
         if isinstance(w_callable, Constant):
             fn = w_callable.value
-            try:
-                sc = SPECIAL_CASES[fn]   # TypeError if 'fn' not hashable
-            except (KeyError, TypeError):
-                pass
-            else:
+            sc = get_specialcase(fn)
+            if sc:
                 return sc(ctx, *args_w)
         return ctx.do_op(self)
 
@@ -589,11 +585,8 @@
         w_callable = self.args[0]
         if isinstance(w_callable, Constant):
             fn = w_callable.value
-            try:
-                sc = SPECIAL_CASES[fn]   # TypeError if 'fn' not hashable
-            except (KeyError, TypeError):
-                pass
-            else:
+            sc = get_specialcase(fn)
+            if sc:
                 from rpython.flowspace.flowcontext import FlowingError
                 raise FlowingError(
                     "should not call %r with keyword arguments" % (fn,))
diff --git a/rpython/flowspace/specialcase.py b/rpython/flowspace/specialcase.py
--- a/rpython/flowspace/specialcase.py
+++ b/rpython/flowspace/specialcase.py
@@ -54,6 +54,26 @@
         from rpython.flowspace.operation import op
         return op.getattr(w_obj, w_index).eval(ctx)
 
+def get_specialcase(fn):
+    try:
+        return SPECIAL_CASES[fn]   # TypeError if 'fn' not hashable
+    except (KeyError, TypeError):
+        # Try to import modules containing special cases
+        for modname in SPECIAL_MODULES.get(fn.__module__, []):
+            __import__(modname)
+        try:
+            return SPECIAL_CASES[fn]
+        except (KeyError, TypeError):
+            pass
+    return None
+
+SPECIAL_MODULES = {
+    # Modules with functions registered with @register_flow_sc, and
+    # which cannot be imported when before the flow object space
+    # (because of import loops).
+    'posix': ['rpython.rlib.rposix'],
+}
+
 # _________________________________________________________________________
 
 redirect_function(open,       'rpython.rlib.rfile.create_file')
diff --git a/rpython/rlib/objectmodel.py b/rpython/rlib/objectmodel.py
--- a/rpython/rlib/objectmodel.py
+++ b/rpython/rlib/objectmodel.py
@@ -286,13 +286,15 @@
     return Constant(True)
 
 
-def register_replacement_for(replaced_function):
+def register_replacement_for(replaced_function, sandboxed_name=None):
     """Decorator that causes RPython to replace the function passed as parameter
     with the function being defined."""
     def wrap(func):
         @register_flow_sc(replaced_function)
         def sc_redirected_function(ctx, *args_w):
             return ctx.appcall(func, *args_w)
+        if sandboxed_name:
+            func._sandbox_external_name = sandboxed_name
         return func
     return wrap
 
diff --git a/rpython/rlib/rposix.py b/rpython/rlib/rposix.py
--- a/rpython/rlib/rposix.py
+++ b/rpython/rlib/rposix.py
@@ -4,7 +4,8 @@
 from rpython.rtyper.lltypesystem import ll2ctypes, rffi
 from rpython.translator.tool.cbuild import ExternalCompilationInfo
 from rpython.rlib.rarithmetic import intmask
-from rpython.rlib.objectmodel import specialize, register_replacement_for
+from rpython.rlib.objectmodel import (
+    specialize, enforceargs, register_replacement_for)
 from rpython.rlib import jit
 from rpython.translator.platform import platform
 
@@ -111,6 +112,7 @@
         "_PyVerify_fd", [rffi.INT], rffi.INT,
         compilation_info=errno_eci,
         ))
+    @enforceargs(int)
     def validate_fd(fd):
         if not is_valid_fd(fd):
             raise OSError(get_errno(), 'Bad file descriptor')
@@ -118,6 +120,7 @@
     def is_valid_fd(fd):
         return 1
 
+    @enforceargs(int)
     def validate_fd(fd):
         pass
 
@@ -250,15 +253,15 @@
 # They usually check the return value and raise an (RPython) OSError
 # with errno.
 
- at register_replacement_for(os.dup)
+ at register_replacement_for(os.dup, sandboxed_name='ll_os.ll_os_dup')
 def dup(fd):
     validate_fd(fd)
     newfd = c_dup(fd)
     if newfd < 0:
         raise OSError(get_errno(), "dup failed")
-    return newfd
+    return intmask(newfd)
 
- at register_replacement_for(os.dup2)
+ at register_replacement_for(os.dup2, sandboxed_name='ll_os.ll_os_dup2')
 def dup2(fd, newfd):
     validate_fd(fd)
     error = c_dup2(fd, newfd)
diff --git a/rpython/rlib/test/test_rposix.py b/rpython/rlib/test/test_rposix.py
--- a/rpython/rlib/test/test_rposix.py
+++ b/rpython/rlib/test/test_rposix.py
@@ -1,4 +1,5 @@
 from rpython.rtyper.test.test_llinterp import interpret
+from rpython.translator.c.test.test_genc import compile
 from rpython.tool.udir import udir
 from rpython.rlib import rposix
 import os, sys
diff --git a/rpython/translator/c/database.py b/rpython/translator/c/database.py
--- a/rpython/translator/c/database.py
+++ b/rpython/translator/c/database.py
@@ -385,6 +385,9 @@
             return False
         if hasattr(fnobj, '_safe_not_sandboxed'):
             return not fnobj._safe_not_sandboxed
+        elif getattr(getattr(fnobj, '_callable', None),
+                     '_sandbox_external_name', None):
+            return True
         else:
             return "if_external"
 
diff --git a/rpython/translator/sandbox/rsandbox.py b/rpython/translator/sandbox/rsandbox.py
--- a/rpython/translator/sandbox/rsandbox.py
+++ b/rpython/translator/sandbox/rsandbox.py
@@ -115,7 +115,11 @@
     trampoline marshals its input arguments, dumps them to STDOUT,
     and waits for an answer on STDIN.
     """
-    fnname = fnobj._name
+    if getattr(getattr(fnobj, '_callable', None),
+               '_sandbox_external_name', None):
+        fnname = fnobj._callable._sandbox_external_name
+    else:
+        fnname = fnobj._name
     if hasattr(fnobj, 'graph'):
         # get the annotation of the input arguments and the result
         graph = fnobj.graph

From noreply at buildbot.pypy.org  Wed Nov  5 16:59:55 2014
From: noreply at buildbot.pypy.org (amauryfa)
Date: Wed,  5 Nov 2014 16:59:55 +0100 (CET)
Subject: [pypy-commit] pypy more-rposix: Move os.open() to rposix.py.
Message-ID: <20141105155955.4D4A91C3570@cobra.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc 
Branch: more-rposix
Changeset: r74342:356ea110dea9
Date: 2014-11-05 16:59 +0100
http://bitbucket.org/pypy/pypy/changeset/356ea110dea9/

Log:	Move os.open() to rposix.py. Much simpler, less magic.

diff --git a/pypy/module/posix/interp_posix.py b/pypy/module/posix/interp_posix.py
--- a/pypy/module/posix/interp_posix.py
+++ b/pypy/module/posix/interp_posix.py
@@ -43,6 +43,8 @@
     return space.str0_w(w_obj)
 
 class FileEncoder(object):
+    is_unicode = True
+
     def __init__(self, space, w_obj):
         self.space = space
         self.w_obj = w_obj
@@ -54,6 +56,8 @@
         return self.space.unicode0_w(self.w_obj)
 
 class FileDecoder(object):
+    is_unicode = False
+
     def __init__(self, space, w_obj):
         self.space = space
         self.w_obj = w_obj
diff --git a/rpython/rlib/rposix.py b/rpython/rlib/rposix.py
--- a/rpython/rlib/rposix.py
+++ b/rpython/rlib/rposix.py
@@ -8,6 +8,7 @@
     specialize, enforceargs, register_replacement_for)
 from rpython.rlib import jit
 from rpython.translator.platform import platform
+from rpython.rlib import rstring
 
 _WIN32 = sys.platform.startswith('win')
 UNDERSCORE_ON_WIN32 = '_' if _WIN32 else ''
@@ -146,6 +147,11 @@
 
 c_dup = external(UNDERSCORE_ON_WIN32 + 'dup', [rffi.INT], rffi.INT)
 c_dup2 = external(UNDERSCORE_ON_WIN32 + 'dup2', [rffi.INT, rffi.INT], rffi.INT)
+c_open = external(UNDERSCORE_ON_WIN32 + 'open',
+                  [rffi.CCHARP, rffi.INT, rffi.MODE_T], rffi.INT)
+# Win32 specific functions
+c_wopen = external(UNDERSCORE_ON_WIN32 + 'wopen',
+                   [rffi.CWCHARP, rffi.INT, rffi.MODE_T], rffi.INT)
 
 #___________________________________________________________________
 # Wrappers around posix functions, that accept either strings, or
@@ -159,12 +165,50 @@
     assert path is not None
     if isinstance(path, str):
         return path
+    elif isinstance(path, unicode):
+        # This never happens in PyPy's Python interpreter!
+        # Only in raw RPython code that uses unicode strings.
+        # We implement python2 behavior: silently convert to ascii.
+        return path.encode('ascii')
     else:
         return path.as_bytes()
 
 @specialize.argtype(0)
-def open(path, flags, mode):
-    return os.open(_as_bytes(path), flags, mode)
+def _as_bytes0(path):
+    res = _as_bytes(path)
+    rstring.check_str0(path)
+    return res
+
+ at specialize.argtype(0)
+def _as_unicode(path):
+    assert path is not None
+    if isinstance(path, unicode):
+        return path
+    else:
+        return path.as_unicode()
+
+ at specialize.argtype(0)
+def _as_unicode0(path):
+    res = _as_unicode(path)
+    rstring.check_str0(path)
+    return res
+
+# Returns True when the unicode function should be called:
+# - on Windows
+# - if the path is Unicode.
+if _WIN32:
+    @specialize.argtype(0)
+    def _prefer_unicode(path):
+        if isinstance(path, str):
+            return False
+        elif isinstance(path, unicode):
+            return True
+        else:
+            return path.is_unicode
+else:
+    @specialize.argtype(0)
+    def _prefer_unicode(path):
+        return False
 
 @specialize.argtype(0)
 def stat(path):
@@ -267,3 +311,15 @@
     error = c_dup2(fd, newfd)
     if error < 0:
         raise OSError(get_errno(), "dup2 failed")
+
+ at register_replacement_for(os.open, sandboxed_name='ll_os.ll_os_open')
+ at specialize.argtype(0)
+def open(path, flags, mode):
+    if _prefer_unicode(path):
+        fd = c_wopen(_as_unicode0(path), flags, mode)
+    else:
+        fd = c_open(_as_bytes0(path), flags, mode)
+    if fd < 0:
+        raise OSError(get_errno(), "open failed")
+    return intmask(fd)
+        
diff --git a/rpython/rlib/test/test_rposix.py b/rpython/rlib/test/test_rposix.py
--- a/rpython/rlib/test/test_rposix.py
+++ b/rpython/rlib/test/test_rposix.py
@@ -45,7 +45,7 @@
     def test_open(self):
         def f():
             try:
-                fd = rposix.open(self.path, os.O_RDONLY, 0777)
+                fd = os.open(self.path, os.O_RDONLY, 0777)
                 try:
                     text = os.read(fd, 50)
                     return text
@@ -177,3 +177,9 @@
             os.dup(4)
             os.dup2(5, 6)
         compile(f, ())
+
+    def test_open(self):
+        def f():
+            os.open('/tmp/t', 0, 0)
+            os.open(u'/tmp/t', 0, 0)
+        compile(f, ())
diff --git a/rpython/rtyper/module/ll_os.py b/rpython/rtyper/module/ll_os.py
--- a/rpython/rtyper/module/ll_os.py
+++ b/rpython/rtyper/module/ll_os.py
@@ -910,20 +910,6 @@
         return extdef([int, int, int], None, llimpl=c_setresgid_llimpl,
                       export_name='ll_os.ll_os_setresgid')
 
-    @registering_str_unicode(os.open)
-    def register_os_open(self, traits):
-        os_open = self.llexternal(traits.posix_function_name('open'),
-                                  [traits.CCHARP, rffi.INT, rffi.MODE_T],
-                                  rffi.INT)
-        def os_open_llimpl(path, flags, mode):
-            result = rffi.cast(lltype.Signed, os_open(path, flags, mode))
-            if result == -1:
-                raise OSError(rposix.get_errno(), "os_open failed")
-            return result
-
-        return extdef([traits.str0, int, int], int, traits.ll_os_name('open'),
-                      llimpl=os_open_llimpl)
-
     @registering_if(os, 'getloadavg')
     def register_os_getloadavg(self):
         AP = rffi.CArrayPtr(lltype.Float)

From noreply at buildbot.pypy.org  Wed Nov  5 17:07:21 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Wed,  5 Nov 2014 17:07:21 +0100 (CET)
Subject: [pypy-commit] pypy default: Add comment
Message-ID: <20141105160721.0C5261C3570@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r74343:fb5c0ab7038f
Date: 2014-11-05 17:06 +0100
http://bitbucket.org/pypy/pypy/changeset/fb5c0ab7038f/

Log:	Add comment

diff --git a/rpython/jit/metainterp/history.py b/rpython/jit/metainterp/history.py
--- a/rpython/jit/metainterp/history.py
+++ b/rpython/jit/metainterp/history.py
@@ -279,7 +279,8 @@
             # careful in this comparison: if self.value and other.value
             # are both NaN, stored as regular floats (i.e. on 64-bit),
             # then just using "==" would say False: two NaNs are always
-            # different from each other.
+            # different from each other.  Conversely, "0.0 == -0.0" but
+            # they are not the same constant.
             return (longlong.extract_bits(self.value) ==
                     longlong.extract_bits(other.value))
         return False

From noreply at buildbot.pypy.org  Wed Nov  5 18:38:30 2014
From: noreply at buildbot.pypy.org (amauryfa)
Date: Wed,  5 Nov 2014 18:38:30 +0100 (CET)
Subject: [pypy-commit] pypy default: Remove an old workaround for gcc on
	OS/X.
Message-ID: <20141105173830.C39E11C35F7@cobra.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc 
Branch: 
Changeset: r74344:107f2d89cecc
Date: 2014-11-05 18:37 +0100
http://bitbucket.org/pypy/pypy/changeset/107f2d89cecc/

Log:	Remove an old workaround for gcc on OS/X. It was already disabled
	anyway.

diff --git a/rpython/rtyper/module/ll_os.py b/rpython/rtyper/module/ll_os.py
--- a/rpython/rtyper/module/ll_os.py
+++ b/rpython/rtyper/module/ll_os.py
@@ -249,12 +249,9 @@
 
     @registering_if(os, 'execv')
     def register_os_execv(self):
-        eci = self.gcc_profiling_bug_workaround(
-            'RPY_EXPORTED_FOR_TESTS int _noprof_execv(char *path, char *argv[])',
-            'return execv(path, argv);')
-        os_execv = self.llexternal('_noprof_execv',
-                                   [rffi.CCHARP, rffi.CCHARPP],
-                                   rffi.INT, compilation_info = eci)
+        os_execv = self.llexternal(
+            'execv',
+            [rffi.CCHARP, rffi.CCHARPP], rffi.INT)
 
         def execv_llimpl(path, args):
             l_args = rffi.ll_liststr2charpp(args)
@@ -268,12 +265,9 @@
 
     @registering_if(os, 'execve')
     def register_os_execve(self):
-        eci = self.gcc_profiling_bug_workaround(
-            'RPY_EXPORTED_FOR_TESTS int _noprof_execve(char *filename, char *argv[], char *envp[])',
-            'return execve(filename, argv, envp);')
         os_execve = self.llexternal(
-            '_noprof_execve', [rffi.CCHARP, rffi.CCHARPP, rffi.CCHARPP],
-            rffi.INT, compilation_info = eci)
+            'execve',
+            [rffi.CCHARP, rffi.CCHARPP, rffi.CCHARPP], rffi.INT)
 
         def execve_llimpl(path, args, env):
             # XXX Check path, args, env for \0 and raise TypeErrors as
@@ -1723,10 +1717,7 @@
     @registering_if(os, 'fork')
     def register_os_fork(self):
         from rpython.rlib import debug, rthread
-        eci = self.gcc_profiling_bug_workaround('RPY_EXPORTED_FOR_TESTS pid_t _noprof_fork(void)',
-                                                'return fork();')
-        os_fork = self.llexternal('_noprof_fork', [], rffi.PID_T,
-                                  compilation_info = eci,
+        os_fork = self.llexternal('fork', [], rffi.PID_T,
                                   _nowrapper = True)
 
         def fork_llimpl():
@@ -1927,21 +1918,6 @@
         return extdef([int], str, "ll_os.ttyname",
                       llimpl=ttyname_llimpl)
 
-    # ____________________________________________________________
-    # XXX horrible workaround for a bug of profiling in gcc on
-    # OS X with functions containing a direct call to some system calls
-    # like fork(), execv(), execve()
-    def gcc_profiling_bug_workaround(self, decl, body):
-        body = ('/*--no-profiling-for-this-file!--*/\n'
-                '#include "src/precommondefs.h"\n'
-                '%s {\n'
-                '\t%s\n'
-                '}\n' % (decl, body,))
-        return ExternalCompilationInfo(
-            include_dirs=[cdir],
-            post_include_bits = [decl + ';'],
-            separate_module_sources = [body])
-
 # ____________________________________________________________
 # Support for os.environ
 
diff --git a/rpython/translator/c/genc.py b/rpython/translator/c/genc.py
--- a/rpython/translator/c/genc.py
+++ b/rpython/translator/c/genc.py
@@ -42,26 +42,7 @@
         self.compiler = compiler
 
     def first(self):
-        platform = self.compiler.platform
-        if platform.name.startswith('darwin'):
-            # XXX incredible hack for darwin
-            STR = '/*--no-profiling-for-this-file!--*/'
-            no_prof = []
-            prof = []
-            for cfile in self.compiler.cfiles:
-                if STR in cfile.read():
-                    no_prof.append(cfile)
-                else:
-                    prof.append(cfile)
-            p_eci = self.compiler.eci.merge(
-                ExternalCompilationInfo(compile_extra=['-fprofile-generate'],
-                                        link_extra=['-fprofile-generate']))
-            ofiles = platform._compile_o_files(prof, p_eci)
-            _, eci = self.compiler.eci.get_module_files()
-            ofiles += platform._compile_o_files(no_prof, eci)
-            return platform._finish_linking(ofiles, p_eci, None, True)
-        else:
-            return self.build('-fprofile-generate')
+        return self.build('-fprofile-generate')
 
     def probe(self, exe, args):
         # 'args' is a single string typically containing spaces
diff --git a/rpython/translator/platform/distutils_platform.py b/rpython/translator/platform/distutils_platform.py
--- a/rpython/translator/platform/distutils_platform.py
+++ b/rpython/translator/platform/distutils_platform.py
@@ -127,14 +127,6 @@
         for cfile in self.cfilenames:
             cfile = py.path.local(cfile)
             compile_extra = self.compile_extra[:]
-            # -frandom-seed is only to try to be as reproducable as possible
-            if 0 and self.fix_gcc_random_seed:
-                compile_extra.append('-frandom-seed=%s' % (cfile.basename,))
-                # XXX horrible workaround for a bug of profiling in gcc on
-                # OS X with functions containing a direct call to fork()
-                if '/*--no-profiling-for-this-file!--*/' in cfile.read():
-                    compile_extra = [arg for arg in compile_extra
-                                     if not arg.startswith('-fprofile-')]
 
             old = cfile.dirpath().chdir()
             try:

From noreply at buildbot.pypy.org  Wed Nov  5 20:04:38 2014
From: noreply at buildbot.pypy.org (amauryfa)
Date: Wed,  5 Nov 2014 20:04:38 +0100 (CET)
Subject: [pypy-commit] pypy more-rposix: Fix translation
Message-ID: <20141105190439.08EC31C1056@cobra.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc 
Branch: more-rposix
Changeset: r74345:85b8060d8eaa
Date: 2014-11-05 17:48 +0100
http://bitbucket.org/pypy/pypy/changeset/85b8060d8eaa/

Log:	Fix translation

diff --git a/rpython/rlib/rposix.py b/rpython/rlib/rposix.py
--- a/rpython/rlib/rposix.py
+++ b/rpython/rlib/rposix.py
@@ -175,8 +175,9 @@
 
 @specialize.argtype(0)
 def _as_bytes0(path):
+    """Crashes translation if the path contains NUL characters."""
     res = _as_bytes(path)
-    rstring.check_str0(path)
+    rstring.check_str0(res)
     return res
 
 @specialize.argtype(0)
@@ -189,8 +190,9 @@
 
 @specialize.argtype(0)
 def _as_unicode0(path):
+    """Crashes translation if the path contains NUL characters."""
     res = _as_unicode(path)
-    rstring.check_str0(path)
+    rstring.check_str0(res)
     return res
 
 # Returns True when the unicode function should be called:

From noreply at buildbot.pypy.org  Wed Nov  5 20:04:40 2014
From: noreply at buildbot.pypy.org (amauryfa)
Date: Wed,  5 Nov 2014 20:04:40 +0100 (CET)
Subject: [pypy-commit] pypy more-rposix: hg merge default
Message-ID: <20141105190440.5337A1C1056@cobra.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc 
Branch: more-rposix
Changeset: r74346:6f17e9b7ff10
Date: 2014-11-05 18:41 +0100
http://bitbucket.org/pypy/pypy/changeset/6f17e9b7ff10/

Log:	hg merge default

diff --git a/README.rst b/README.rst
--- a/README.rst
+++ b/README.rst
@@ -37,4 +37,4 @@
 to use virtualenv with the resulting pypy-c as the interpreter; you can
 find more details about various installation schemes here:
 
-http://doc.pypy.org/en/latest/getting-started.html#installing-pypy
+    http://doc.pypy.org/en/latest/install.html
diff --git a/rpython/jit/metainterp/history.py b/rpython/jit/metainterp/history.py
--- a/rpython/jit/metainterp/history.py
+++ b/rpython/jit/metainterp/history.py
@@ -279,7 +279,8 @@
             # careful in this comparison: if self.value and other.value
             # are both NaN, stored as regular floats (i.e. on 64-bit),
             # then just using "==" would say False: two NaNs are always
-            # different from each other.
+            # different from each other.  Conversely, "0.0 == -0.0" but
+            # they are not the same constant.
             return (longlong.extract_bits(self.value) ==
                     longlong.extract_bits(other.value))
         return False
diff --git a/rpython/rlib/objectmodel.py b/rpython/rlib/objectmodel.py
--- a/rpython/rlib/objectmodel.py
+++ b/rpython/rlib/objectmodel.py
@@ -201,6 +201,11 @@
         return result
     return decorator
 
+def always_inline(func):
+    """ mark the function as to-be-inlined by the RPython optimizations (not
+    the JIT!), no matter its size."""
+    func._always_inline_ = True
+    return func
 
 
 # ____________________________________________________________
diff --git a/rpython/rlib/test/test_objectmodel.py b/rpython/rlib/test/test_objectmodel.py
--- a/rpython/rlib/test/test_objectmodel.py
+++ b/rpython/rlib/test/test_objectmodel.py
@@ -438,6 +438,11 @@
     assert exc.value.message == "f argument 'b' must be of type "
     py.test.raises(TypeError, "f('hello', 'world', 3)")
 
+def test_always_inline():
+    @always_inline
+    def f(a, b, c):
+        return a, b, c
+    assert f._always_inline_ == True
 
 def test_enforceargs_defaults():
     @enforceargs(int, int)
diff --git a/rpython/rtyper/module/ll_os.py b/rpython/rtyper/module/ll_os.py
--- a/rpython/rtyper/module/ll_os.py
+++ b/rpython/rtyper/module/ll_os.py
@@ -22,8 +22,6 @@
 from rpython.rtyper.tool import rffi_platform as platform
 from rpython.rlib import rposix
 from rpython.translator.tool.cbuild import ExternalCompilationInfo
-from rpython.rtyper.lltypesystem.llmemory import itemoffsetof, offsetof
-from rpython.rtyper.lltypesystem.rstr import STR
 from rpython.rlib.objectmodel import specialize
 from rpython.translator import cdir
 
@@ -251,12 +249,9 @@
 
     @registering_if(os, 'execv')
     def register_os_execv(self):
-        eci = self.gcc_profiling_bug_workaround(
-            'RPY_EXPORTED_FOR_TESTS int _noprof_execv(char *path, char *argv[])',
-            'return execv(path, argv);')
-        os_execv = self.llexternal('_noprof_execv',
-                                   [rffi.CCHARP, rffi.CCHARPP],
-                                   rffi.INT, compilation_info = eci)
+        os_execv = self.llexternal(
+            'execv',
+            [rffi.CCHARP, rffi.CCHARPP], rffi.INT)
 
         def execv_llimpl(path, args):
             l_args = rffi.ll_liststr2charpp(args)
@@ -270,12 +265,9 @@
 
     @registering_if(os, 'execve')
     def register_os_execve(self):
-        eci = self.gcc_profiling_bug_workaround(
-            'RPY_EXPORTED_FOR_TESTS int _noprof_execve(char *filename, char *argv[], char *envp[])',
-            'return execve(filename, argv, envp);')
         os_execve = self.llexternal(
-            '_noprof_execve', [rffi.CCHARP, rffi.CCHARPP, rffi.CCHARPP],
-            rffi.INT, compilation_info = eci)
+            'execve',
+            [rffi.CCHARP, rffi.CCHARPP, rffi.CCHARPP], rffi.INT)
 
         def execve_llimpl(path, args, env):
             # XXX Check path, args, env for \0 and raise TypeErrors as
@@ -958,8 +950,6 @@
                                   [rffi.INT, rffi.VOIDP, rffi.SIZE_T],
                                   rffi.SIZE_T)
 
-        offset = offsetof(STR, 'chars') + itemoffsetof(STR.chars, 0)
-
         def os_read_llimpl(fd, count):
             if count < 0:
                 raise OSError(errno.EINVAL, None)
@@ -1684,10 +1674,7 @@
     @registering_if(os, 'fork')
     def register_os_fork(self):
         from rpython.rlib import debug, rthread
-        eci = self.gcc_profiling_bug_workaround('RPY_EXPORTED_FOR_TESTS pid_t _noprof_fork(void)',
-                                                'return fork();')
-        os_fork = self.llexternal('_noprof_fork', [], rffi.PID_T,
-                                  compilation_info = eci,
+        os_fork = self.llexternal('fork', [], rffi.PID_T,
                                   _nowrapper = True)
 
         def fork_llimpl():
@@ -1888,21 +1875,6 @@
         return extdef([int], str, "ll_os.ttyname",
                       llimpl=ttyname_llimpl)
 
-    # ____________________________________________________________
-    # XXX horrible workaround for a bug of profiling in gcc on
-    # OS X with functions containing a direct call to some system calls
-    # like fork(), execv(), execve()
-    def gcc_profiling_bug_workaround(self, decl, body):
-        body = ('/*--no-profiling-for-this-file!--*/\n'
-                '#include "src/precommondefs.h"\n'
-                '%s {\n'
-                '\t%s\n'
-                '}\n' % (decl, body,))
-        return ExternalCompilationInfo(
-            include_dirs=[cdir],
-            post_include_bits = [decl + ';'],
-            separate_module_sources = [body])
-
 # ____________________________________________________________
 # Support for os.environ
 
diff --git a/rpython/translator/c/genc.py b/rpython/translator/c/genc.py
--- a/rpython/translator/c/genc.py
+++ b/rpython/translator/c/genc.py
@@ -42,26 +42,7 @@
         self.compiler = compiler
 
     def first(self):
-        platform = self.compiler.platform
-        if platform.name.startswith('darwin'):
-            # XXX incredible hack for darwin
-            STR = '/*--no-profiling-for-this-file!--*/'
-            no_prof = []
-            prof = []
-            for cfile in self.compiler.cfiles:
-                if STR in cfile.read():
-                    no_prof.append(cfile)
-                else:
-                    prof.append(cfile)
-            p_eci = self.compiler.eci.merge(
-                ExternalCompilationInfo(compile_extra=['-fprofile-generate'],
-                                        link_extra=['-fprofile-generate']))
-            ofiles = platform._compile_o_files(prof, p_eci)
-            _, eci = self.compiler.eci.get_module_files()
-            ofiles += platform._compile_o_files(no_prof, eci)
-            return platform._finish_linking(ofiles, p_eci, None, True)
-        else:
-            return self.build('-fprofile-generate')
+        return self.build('-fprofile-generate')
 
     def probe(self, exe, args):
         # 'args' is a single string typically containing spaces
diff --git a/rpython/translator/platform/distutils_platform.py b/rpython/translator/platform/distutils_platform.py
--- a/rpython/translator/platform/distutils_platform.py
+++ b/rpython/translator/platform/distutils_platform.py
@@ -127,14 +127,6 @@
         for cfile in self.cfilenames:
             cfile = py.path.local(cfile)
             compile_extra = self.compile_extra[:]
-            # -frandom-seed is only to try to be as reproducable as possible
-            if 0 and self.fix_gcc_random_seed:
-                compile_extra.append('-frandom-seed=%s' % (cfile.basename,))
-                # XXX horrible workaround for a bug of profiling in gcc on
-                # OS X with functions containing a direct call to fork()
-                if '/*--no-profiling-for-this-file!--*/' in cfile.read():
-                    compile_extra = [arg for arg in compile_extra
-                                     if not arg.startswith('-fprofile-')]
 
             old = cfile.dirpath().chdir()
             try:

From noreply at buildbot.pypy.org  Wed Nov  5 20:04:41 2014
From: noreply at buildbot.pypy.org (amauryfa)
Date: Wed,  5 Nov 2014 20:04:41 +0100 (CET)
Subject: [pypy-commit] pypy more-rposix: Port os.execv and os.execve
Message-ID: <20141105190441.8E1A61C1056@cobra.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc 
Branch: more-rposix
Changeset: r74347:e010a14828c5
Date: 2014-11-05 19:24 +0100
http://bitbucket.org/pypy/pypy/changeset/e010a14828c5/

Log:	Port os.execv and os.execve

diff --git a/rpython/rlib/rposix.py b/rpython/rlib/rposix.py
--- a/rpython/rlib/rposix.py
+++ b/rpython/rlib/rposix.py
@@ -149,6 +149,9 @@
 c_dup2 = external(UNDERSCORE_ON_WIN32 + 'dup2', [rffi.INT, rffi.INT], rffi.INT)
 c_open = external(UNDERSCORE_ON_WIN32 + 'open',
                   [rffi.CCHARP, rffi.INT, rffi.MODE_T], rffi.INT)
+c_execv = external('execv', [rffi.CCHARP, rffi.CCHARPP], rffi.INT)
+c_execve = external('execve',
+                    [rffi.CCHARP, rffi.CCHARPP, rffi.CCHARPP], rffi.INT)
 # Win32 specific functions
 c_wopen = external(UNDERSCORE_ON_WIN32 + 'wopen',
                    [rffi.CWCHARP, rffi.INT, rffi.MODE_T], rffi.INT)
@@ -325,3 +328,31 @@
         raise OSError(get_errno(), "open failed")
     return intmask(fd)
         
+ at register_replacement_for(getattr(os, 'execv', None),
+                          sandboxed_name='ll_os.ll_os_execv')
+def execv(path, args):
+    rstring.check_str0(path)
+    # This list conversion already takes care of NUL bytes.
+    l_args = rffi.ll_liststr2charpp(args)
+    c_execv(path, l_args)
+    rffi.free_charpp(l_args)
+    raise OSError(get_errno(), "execv failed")
+
+ at register_replacement_for(getattr(os, 'execve', None),
+                          sandboxed_name='ll_os.ll_os_execve')
+def execve(path, args, env):
+    envstrs = []
+    for item in env.iteritems():
+        envstr = "%s=%s" % item
+        envstrs.append(envstr)
+
+    rstring.check_str0(path)
+    # This list conversion already takes care of NUL bytes.
+    l_args = rffi.ll_liststr2charpp(args)
+    l_env = rffi.ll_liststr2charpp(envstrs)
+    c_execve(path, l_args, l_env)
+
+    
+    rffi.free_charpp(l_env)
+    rffi.free_charpp(l_args)
+    raise OSError(get_errno(), "execve failed")
diff --git a/rpython/rtyper/module/ll_os.py b/rpython/rtyper/module/ll_os.py
--- a/rpython/rtyper/module/ll_os.py
+++ b/rpython/rtyper/module/ll_os.py
@@ -247,53 +247,6 @@
         return extdef([int], int, llimpl=c_func_llimpl,
                       export_name='ll_os.ll_os_' + name)
 
-    @registering_if(os, 'execv')
-    def register_os_execv(self):
-        os_execv = self.llexternal(
-            'execv',
-            [rffi.CCHARP, rffi.CCHARPP], rffi.INT)
-
-        def execv_llimpl(path, args):
-            l_args = rffi.ll_liststr2charpp(args)
-            os_execv(path, l_args)
-            rffi.free_charpp(l_args)
-            raise OSError(rposix.get_errno(), "execv failed")
-
-        return extdef([str0, [str0]], s_ImpossibleValue, llimpl=execv_llimpl,
-                      export_name="ll_os.ll_os_execv")
-
-
-    @registering_if(os, 'execve')
-    def register_os_execve(self):
-        os_execve = self.llexternal(
-            'execve',
-            [rffi.CCHARP, rffi.CCHARPP, rffi.CCHARPP], rffi.INT)
-
-        def execve_llimpl(path, args, env):
-            # XXX Check path, args, env for \0 and raise TypeErrors as
-            # appropriate
-            envstrs = []
-            for item in env.iteritems():
-                envstr = "%s=%s" % item
-                envstrs.append(envstr)
-
-            l_args = rffi.ll_liststr2charpp(args)
-            l_env = rffi.ll_liststr2charpp(envstrs)
-            os_execve(path, l_args, l_env)
-
-            # XXX untested
-            rffi.free_charpp(l_env)
-            rffi.free_charpp(l_args)
-
-            raise OSError(rposix.get_errno(), "execve failed")
-
-        return extdef(
-            [str0, [str0], {str0: str0}],
-            s_ImpossibleValue,
-            llimpl=execve_llimpl,
-            export_name="ll_os.ll_os_execve")
-
-
     @registering_if(posix, 'spawnv')
     def register_os_spawnv(self):
         os_spawnv = self.llexternal('spawnv',

From noreply at buildbot.pypy.org  Wed Nov  5 20:04:42 2014
From: noreply at buildbot.pypy.org (amauryfa)
Date: Wed,  5 Nov 2014 20:04:42 +0100 (CET)
Subject: [pypy-commit] pypy more-rposix: Translation and test fixes
Message-ID: <20141105190442.BDA4E1C1056@cobra.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc 
Branch: more-rposix
Changeset: r74348:96cace28d98e
Date: 2014-11-05 20:03 +0100
http://bitbucket.org/pypy/pypy/changeset/96cace28d98e/

Log:	Translation and test fixes

diff --git a/rpython/annotator/bookkeeper.py b/rpython/annotator/bookkeeper.py
--- a/rpython/annotator/bookkeeper.py
+++ b/rpython/annotator/bookkeeper.py
@@ -238,10 +238,11 @@
             else:
                 result = SomeString(no_nul=no_nul)
         elif tp is unicode:
+            no_nul = not u'\x00' in x
             if len(x) == 1:
-                result = SomeUnicodeCodePoint()
+                result = SomeUnicodeCodePoint(no_nul=no_nul)
             else:
-                result = SomeUnicodeString()
+                result = SomeUnicodeString(no_nul=no_nul)
         elif tp is bytearray:
             result = SomeByteArray()
         elif tp is tuple:
diff --git a/rpython/annotator/unaryop.py b/rpython/annotator/unaryop.py
--- a/rpython/annotator/unaryop.py
+++ b/rpython/annotator/unaryop.py
@@ -537,7 +537,7 @@
         enc = s_enc.const
         if enc not in ('ascii', 'latin-1', 'utf-8'):
             raise AnnotatorError("Encoding %s not supported for unicode" % (enc,))
-        return SomeString()
+        return SomeString(no_nul=self.no_nul)
     method_encode.can_only_throw = [UnicodeEncodeError]
 
 
@@ -570,7 +570,7 @@
         enc = s_enc.const
         if enc not in ('ascii', 'latin-1', 'utf-8'):
             raise AnnotatorError("Encoding %s not supported for strings" % (enc,))
-        return SomeUnicodeString()
+        return SomeUnicodeString(no_nul=self.no_nul)
     method_decode.can_only_throw = [UnicodeDecodeError]
 
 class __extend__(SomeChar, SomeUnicodeCodePoint):
diff --git a/rpython/rlib/test/test_rposix.py b/rpython/rlib/test/test_rposix.py
--- a/rpython/rlib/test/test_rposix.py
+++ b/rpython/rlib/test/test_rposix.py
@@ -1,7 +1,7 @@
 from rpython.rtyper.test.test_llinterp import interpret
 from rpython.translator.c.test.test_genc import compile
 from rpython.tool.udir import udir
-from rpython.rlib import rposix
+from rpython.rlib import rposix, rstring
 import os, sys
 import py
 
@@ -15,13 +15,15 @@
     if sys.platform == 'win32':
         def as_bytes(self):
             from rpython.rlib.runicode import unicode_encode_mbcs
-            return unicode_encode_mbcs(self.unistr, len(self.unistr),
-                                       "strict")
+            res = unicode_encode_mbcs(self.unistr, len(self.unistr),
+                                      "strict")
+            return rstring.assert_str0(res)
     else:
         def as_bytes(self):
             from rpython.rlib.runicode import unicode_encode_utf_8
-            return unicode_encode_utf_8(self.unistr, len(self.unistr),
-                                        "strict")
+            res = unicode_encode_utf_8(self.unistr, len(self.unistr),
+                                       "strict")
+            return rstring.assert_str0(res)
 
     def as_unicode(self):
         return self.unistr

From noreply at buildbot.pypy.org  Wed Nov  5 20:04:43 2014
From: noreply at buildbot.pypy.org (amauryfa)
Date: Wed,  5 Nov 2014 20:04:43 +0100 (CET)
Subject: [pypy-commit] pypy more-rposix: Port os.spawn* functions.
Message-ID: <20141105190443.EE1B51C1056@cobra.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc 
Branch: more-rposix
Changeset: r74349:3ad549fc500c
Date: 2014-11-05 20:03 +0100
http://bitbucket.org/pypy/pypy/changeset/3ad549fc500c/

Log:	Port os.spawn* functions.

	Untested, need to kick a win32 buildbot.

diff --git a/rpython/rlib/objectmodel.py b/rpython/rlib/objectmodel.py
--- a/rpython/rlib/objectmodel.py
+++ b/rpython/rlib/objectmodel.py
@@ -295,9 +295,10 @@
     """Decorator that causes RPython to replace the function passed as parameter
     with the function being defined."""
     def wrap(func):
-        @register_flow_sc(replaced_function)
-        def sc_redirected_function(ctx, *args_w):
-            return ctx.appcall(func, *args_w)
+        if replaced_function is not None:
+            @register_flow_sc(replaced_function)
+            def sc_redirected_function(ctx, *args_w):
+                return ctx.appcall(func, *args_w)
         if sandboxed_name:
             func._sandbox_external_name = sandboxed_name
         return func
diff --git a/rpython/rlib/rposix.py b/rpython/rlib/rposix.py
--- a/rpython/rlib/rposix.py
+++ b/rpython/rlib/rposix.py
@@ -153,6 +153,12 @@
 c_execve = external('execve',
                     [rffi.CCHARP, rffi.CCHARPP, rffi.CCHARPP], rffi.INT)
 # Win32 specific functions
+c_spawnv = external('spawnv',
+                    [rffi.INT, rffi.CCHARP, rffi.CCHARPP], rffi.INT)
+c_spawnve = external('spawnve',
+                    [rffi.INT, rffi.CCHARP, rffi.CCHARPP, rffi.CCHARP],
+                     rffi.INT)
+# Win32 Unicode functions
 c_wopen = external(UNDERSCORE_ON_WIN32 + 'wopen',
                    [rffi.CWCHARP, rffi.INT, rffi.MODE_T], rffi.INT)
 
@@ -352,7 +358,34 @@
     l_env = rffi.ll_liststr2charpp(envstrs)
     c_execve(path, l_args, l_env)
 
-    
     rffi.free_charpp(l_env)
     rffi.free_charpp(l_args)
     raise OSError(get_errno(), "execve failed")
+
+ at register_replacement_for(getattr(os, 'spawnv', None),
+                          sandboxed_name='ll_os.ll_os_spawnv')
+def spawnv(mode, path, args):
+    rstring.check_str0(path)
+    l_args = rffi.ll_liststr2charpp(args)
+    childpid = c_spawnv(mode, path, l_args)
+    rffi.free_charpp(l_args)
+    if childpid < 0:
+        raise OSError(get_errno(), "os_spawnv failed")
+    return intmask(childpid)
+
+ at register_replacement_for(getattr(os, 'spawnve', None),
+                          sandboxed_name='ll_os.ll_os_spawnve')
+def spawnve(mode, path, args, env):
+    envstrs = []
+    for item in env.iteritems():
+        envstrs.append("%s=%s" % item)
+    rstring.check_str0(path)
+    l_args = rffi.ll_liststr2charpp(args)
+    l_env = rffi.ll_liststr2charpp(envstrs)
+    childpid = c_spawnve(mode, path, l_args, l_env)
+    rffi.free_charpp(l_env)
+    rffi.free_charpp(l_args)
+    if childpid == -1:
+        raise OSError(rposix.get_errno(), "os_spawnve failed")
+    return intmask(childpid)
+

From noreply at buildbot.pypy.org  Wed Nov  5 20:17:22 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Wed,  5 Nov 2014 20:17:22 +0100 (CET)
Subject: [pypy-commit] pypy default: Hah,
 suggesting "translation.shared" to be equal to "sys.platform ==
Message-ID: <20141105191722.22CAB1C003C@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r74350:b8b322eb7b99
Date: 2014-11-05 20:16 +0100
http://bitbucket.org/pypy/pypy/changeset/b8b322eb7b99/

Log:	Hah, suggesting "translation.shared" to be equal to "sys.platform ==
	'win32'" is a bad idea because that is False on non-Windows. It
	hides completely the default value choosen for it in
	translationoption.py.

diff --git a/pypy/config/pypyoption.py b/pypy/config/pypyoption.py
--- a/pypy/config/pypyoption.py
+++ b/pypy/config/pypyoption.py
@@ -86,9 +86,10 @@
     # itself needs the interp-level struct module
     # because 'P' is missing from the app-level one
     "_rawffi": [("objspace.usemodules.struct", True)],
-    "cpyext": [("translation.secondaryentrypoints", "cpyext,main"),
-               ("translation.shared", sys.platform == "win32")],
+    "cpyext": [("translation.secondaryentrypoints", "cpyext,main")],
 }
+if sys.platform == "win32":
+    module_suggests["cpyext"].append(("translation.shared", True))
 
 module_import_dependencies = {
     # no _rawffi if importing rpython.rlib.clibffi raises ImportError

From noreply at buildbot.pypy.org  Thu Nov  6 07:38:42 2014
From: noreply at buildbot.pypy.org (fijal)
Date: Thu,  6 Nov 2014 07:38:42 +0100 (CET)
Subject: [pypy-commit] pypy default: use the official way to malloc strings,
	fixes issue1919
Message-ID: <20141106063842.41F251D28DC@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: 
Changeset: r74351:34433ac979df
Date: 2014-11-06 08:38 +0200
http://bitbucket.org/pypy/pypy/changeset/34433ac979df/

Log:	use the official way to malloc strings, fixes issue1919

diff --git a/rpython/rtyper/lltypesystem/rffi.py b/rpython/rtyper/lltypesystem/rffi.py
--- a/rpython/rtyper/lltypesystem/rffi.py
+++ b/rpython/rtyper/lltypesystem/rffi.py
@@ -825,7 +825,7 @@
         allows for the process to be performed without an extra copy.
         Make sure to call keep_buffer_alive_until_here on the returned values.
         """
-        new_buf = lltype.malloc(STRTYPE, count)
+        new_buf = mallocfn(count)
         pinned = 0
         if rgc.can_move(new_buf):
             if rgc.pin(new_buf):
@@ -857,7 +857,7 @@
             if llop.shrink_array(lltype.Bool, gc_buf, needed_size):
                 pass     # now 'gc_buf' is smaller
             else:
-                gc_buf = lltype.malloc(STRTYPE, needed_size)
+                gc_buf = mallocfn(needed_size)
                 case_num = 2
         if case_num == 2:
             copy_raw_to_string(raw_buf, gc_buf, 0, needed_size)

From noreply at buildbot.pypy.org  Thu Nov  6 12:51:53 2014
From: noreply at buildbot.pypy.org (cfbolz)
Date: Thu,  6 Nov 2014 12:51:53 +0100 (CET)
Subject: [pypy-commit] pypy default: add a small fastpath for comparisons of
 a box with itself
Message-ID: <20141106115153.2BA821C3559@cobra.cs.uni-duesseldorf.de>

Author: Carl Friedrich Bolz 
Branch: 
Changeset: r74352:27a0edfa55b5
Date: 2014-11-06 12:47 +0100
http://bitbucket.org/pypy/pypy/changeset/27a0edfa55b5/

Log:	add a small fastpath for comparisons of a box with itself

	these either directly return true or false, depending on the
	comparison

diff --git a/rpython/jit/metainterp/pyjitpl.py b/rpython/jit/metainterp/pyjitpl.py
--- a/rpython/jit/metainterp/pyjitpl.py
+++ b/rpython/jit/metainterp/pyjitpl.py
@@ -33,6 +33,14 @@
 
 # ____________________________________________________________
 
+FASTPATHS_SAME_BOXES = {
+    "ne": "history.CONST_FALSE",
+    "eq": "history.CONST_TRUE",
+    "lt": "history.CONST_FALSE",
+    "le": "history.CONST_TRUE",
+    "gt": "history.CONST_FALSE",
+    "ge": "history.CONST_TRUE",
+}
 
 class MIFrame(object):
     debug = False
@@ -188,8 +196,6 @@
     # ------------------------------
 
     for _opimpl in ['int_add', 'int_sub', 'int_mul', 'int_floordiv', 'int_mod',
-                    'int_lt', 'int_le', 'int_eq',
-                    'int_ne', 'int_gt', 'int_ge',
                     'int_and', 'int_or', 'int_xor',
                     'int_rshift', 'int_lshift', 'uint_rshift',
                     'uint_lt', 'uint_le', 'uint_gt', 'uint_ge',
@@ -197,7 +203,6 @@
                     'float_add', 'float_sub', 'float_mul', 'float_truediv',
                     'float_lt', 'float_le', 'float_eq',
                     'float_ne', 'float_gt', 'float_ge',
-                    'ptr_eq', 'ptr_ne', 'instance_ptr_eq', 'instance_ptr_ne',
                     ]:
         exec py.code.Source('''
             @arguments("box", "box")
@@ -205,6 +210,18 @@
                 return self.execute(rop.%s, b1, b2)
         ''' % (_opimpl, _opimpl.upper())).compile()
 
+    for _opimpl in ['int_eq', 'int_ne', 'int_lt', 'int_le', 'int_gt', 'int_ge',
+                    'ptr_eq', 'ptr_ne',
+                    'instance_ptr_eq', 'instance_ptr_ne']:
+        exec py.code.Source('''
+            @arguments("box", "box")
+            def opimpl_%s(self, b1, b2):
+                if b1 is b2: # crude fast check
+                    return %s
+                return self.execute(rop.%s, b1, b2)
+        ''' % (_opimpl, FASTPATHS_SAME_BOXES[_opimpl.split("_")[-1]], _opimpl.upper())
+        ).compile()
+
     for _opimpl in ['int_add_ovf', 'int_sub_ovf', 'int_mul_ovf']:
         exec py.code.Source('''
             @arguments("box", "box")
@@ -340,10 +357,13 @@
         exec py.code.Source('''
             @arguments("box", "box", "label")
             def opimpl_goto_if_not_%s(self, b1, b2, target):
-                condbox = self.execute(rop.%s, b1, b2)
+                if b1 is b2:
+                    condbox = %s
+                else:
+                    condbox = self.execute(rop.%s, b1, b2)
                 self.opimpl_goto_if_not(condbox, target)
-        ''' % (_opimpl, _opimpl.upper())).compile()
-
+        ''' % (_opimpl, FASTPATHS_SAME_BOXES[_opimpl.split("_")[-1]], _opimpl.upper())
+        ).compile()
 
     def _establish_nullity(self, box, orgpc):
         value = box.nonnull()
diff --git a/rpython/jit/metainterp/test/test_ajit.py b/rpython/jit/metainterp/test/test_ajit.py
--- a/rpython/jit/metainterp/test/test_ajit.py
+++ b/rpython/jit/metainterp/test/test_ajit.py
@@ -4119,3 +4119,64 @@
         assert res == 42
         res = self.interp_operations(f, [-42])
         assert res == 0
+
+    def test_cmp_fastpaths(self):
+        class Z: pass
+        def make_int(cmp):
+            def f(x):
+                if cmp == 'eq':
+                    return x == x and x == x
+                if cmp == 'ne':
+                    return x != x or x != x
+                if cmp == 'lt':
+                    return x < x or x != x
+                if cmp == 'le':
+                    return x <= x and x <= x
+                if cmp == 'gt':
+                    return x > x or x > x
+                if cmp == 'ge':
+                    return x >= x and x >= x
+                assert 0
+            return f
+
+        def make_str(cmp):
+            def f(x):
+                x = str(x)
+                if cmp == 'eq':
+                    return x is x or x is x
+                if cmp == 'ne':
+                    return x is not x and x is not x
+                assert 0
+            return f
+
+        def make_object(cmp):
+            def f(x):
+                y = Z()
+                y.x = x
+                x = y
+                if cmp == 'eq':
+                    return x is x
+                if cmp == 'ne':
+                    return x is not x
+                assert 0
+            return f
+
+        for cmp in 'eq ne lt le gt ge'.split():
+            f = make_int(cmp)
+            res = self.interp_operations(f, [42])
+            assert res == f(42)
+            opname = "int_%s" % cmp
+            self.check_operations_history(**{opname: 0})
+
+        for cmp in 'eq ne'.split():
+            f = make_str(cmp)
+            res = self.interp_operations(f, [42])
+            assert res == f(42)
+            opname = "ptr_%s" % cmp
+            self.check_operations_history(**{opname: 0})
+
+            f = make_object(cmp)
+            res = self.interp_operations(f, [42])
+            assert res == f(42)
+            opname = "instance_ptr_%s" % cmp
+            self.check_operations_history(**{opname: 0})

From noreply at buildbot.pypy.org  Thu Nov  6 13:21:42 2014
From: noreply at buildbot.pypy.org (cfbolz)
Date: Thu,  6 Nov 2014 13:21:42 +0100 (CET)
Subject: [pypy-commit] pypy default: move computation of flattened marks
 into its own function since it only needs
Message-ID: <20141106122142.DB80E1D2900@cobra.cs.uni-duesseldorf.de>

Author: Carl Friedrich Bolz 
Branch: 
Changeset: r74353:bc52afdac223
Date: 2014-11-06 13:17 +0100
http://bitbucket.org/pypy/pypy/changeset/bc52afdac223/

Log:	move computation of flattened marks into its own function since it
	only needs to be run once

diff --git a/rpython/rlib/rsre/rsre_core.py b/rpython/rlib/rsre/rsre_core.py
--- a/rpython/rlib/rsre/rsre_core.py
+++ b/rpython/rlib/rsre/rsre_core.py
@@ -137,22 +137,25 @@
     def flatten_marks(self):
         # for testing
         if self.match_marks_flat is None:
-            self.match_marks_flat = [self.match_start, self.match_end]
-            mark = self.match_marks
-            if mark is not None:
-                self.match_lastindex = mark.gid
-            else:
-                self.match_lastindex = -1
-            while mark is not None:
-                index = mark.gid + 2
-                while index >= len(self.match_marks_flat):
-                    self.match_marks_flat.append(-1)
-                if self.match_marks_flat[index] == -1:
-                    self.match_marks_flat[index] = mark.position
-                mark = mark.prev
-            self.match_marks = None    # clear
+            self._compute_flattened_marks()
         return self.match_marks_flat
 
+    def _compute_flattened_marks(self):
+        self.match_marks_flat = [self.match_start, self.match_end]
+        mark = self.match_marks
+        if mark is not None:
+            self.match_lastindex = mark.gid
+        else:
+            self.match_lastindex = -1
+        while mark is not None:
+            index = mark.gid + 2
+            while index >= len(self.match_marks_flat):
+                self.match_marks_flat.append(-1)
+            if self.match_marks_flat[index] == -1:
+                self.match_marks_flat[index] = mark.position
+            mark = mark.prev
+        self.match_marks = None    # clear
+
     def span(self, groupnum=0):
         # compatibility
         fmarks = self.flatten_marks()

From noreply at buildbot.pypy.org  Thu Nov  6 14:16:36 2014
From: noreply at buildbot.pypy.org (krono)
Date: Thu,  6 Nov 2014 14:16:36 +0100 (CET)
Subject: [pypy-commit] lang-smalltalk default: Remove untrue elidables
Message-ID: <20141106131636.4E9C51D2D84@cobra.cs.uni-duesseldorf.de>

Author: Tobias Pape 
Branch: 
Changeset: r1060:2255d770994f
Date: 2014-11-06 14:16 +0100
http://bitbucket.org/pypy/lang-smalltalk/changeset/2255d770994f/

Log:	Remove untrue elidables

diff --git a/spyvm/model.py b/spyvm/model.py
--- a/spyvm/model.py
+++ b/spyvm/model.py
@@ -195,7 +195,6 @@
         return r_uint(val)
 
 
-    @jit.elidable
     def as_repr_string(self):
         return "W_SmallInteger(%d)" % self.value
 
@@ -450,7 +449,6 @@
                 name = self.s_class.name
             return "a %s" % (name or '?',)
 
-    @jit.elidable
     def as_repr_string(self):
         return self.as_embellished_string("W_O /w Class", "")
 
@@ -625,7 +623,6 @@
         w_other.changed()
         return True
 
-    @jit.elidable
     def as_repr_string(self):
         return W_AbstractObjectWithClassReference.as_embellished_string(self,
                                 className='W_PointersObject',

From noreply at buildbot.pypy.org  Thu Nov  6 17:43:42 2014
From: noreply at buildbot.pypy.org (rlamy)
Date: Thu,  6 Nov 2014 17:43:42 +0100 (CET)
Subject: [pypy-commit] pypy online-transforms: Simplify use of
 normalize_method(): return non-methods unchanged
Message-ID: <20141106164342.0E7821C030E@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy 
Branch: online-transforms
Changeset: r74354:a98b25085355
Date: 2014-11-06 03:59 +0000
http://bitbucket.org/pypy/pypy/changeset/a98b25085355/

Log:	Simplify use of normalize_method(): return non-methods unchanged

diff --git a/rpython/annotator/bookkeeper.py b/rpython/annotator/bookkeeper.py
--- a/rpython/annotator/bookkeeper.py
+++ b/rpython/annotator/bookkeeper.py
@@ -231,10 +231,7 @@
         """The most precise SomeValue instance that contains the
         immutable value x."""
         if callable(x):
-            try:
-                x = normalize_method(x)
-            except ValueError:
-                pass
+            x = normalize_method(x)
         tp = type(x)
         if issubclass(tp, Symbolic): # symbolic constants support
             result = x.annotation()
@@ -372,10 +369,7 @@
         #  * a user-defined bound or unbound method object
         #  * a frozen pre-built constant (with _freeze_() == True)
         #  * a bound method of a frozen pre-built constant
-        try:
-            pyobj = normalize_method(pyobj)
-        except ValueError:
-            pass
+        pyobj = normalize_method(pyobj)
         try:
             return self.descs[pyobj]
         except KeyError:
diff --git a/rpython/annotator/description.py b/rpython/annotator/description.py
--- a/rpython/annotator/description.py
+++ b/rpython/annotator/description.py
@@ -489,10 +489,7 @@
             if name in cls.__dict__:
                 return  # ignore misbehaving descriptors and the like
             raise
-        try:
-            value = normalize_method(value)
-        except ValueError:
-            pass
+        value = normalize_method(value)
         if isinstance(value, InstanceMethod):
             func = value.im_func
             if isinstance(func, types.FunctionType):
diff --git a/rpython/rtyper/extregistry.py b/rpython/rtyper/extregistry.py
--- a/rpython/rtyper/extregistry.py
+++ b/rpython/rtyper/extregistry.py
@@ -22,10 +22,7 @@
         else:
             if key in dict:
                 raise ValueError("duplicate extregistry entry %r" % (selfcls,))
-            try:
-                key = normalize_method(key)
-            except ValueError:
-                pass
+            key = normalize_method(key)
             dict[key] = selfcls
 
     def _register_value(selfcls, key):
@@ -131,10 +128,7 @@
         return _lookup_type_cls(type(instance))
 
 def lookup(instance):
-    try:
-        instance = normalize_method(instance)
-    except ValueError:
-        pass
+    instance = normalize_method(instance)
     Entry = _lookup_cls(instance)
     return Entry(type(instance), instance)
 
diff --git a/rpython/tool/descriptor.py b/rpython/tool/descriptor.py
--- a/rpython/tool/descriptor.py
+++ b/rpython/tool/descriptor.py
@@ -40,7 +40,7 @@
         if isinstance(method, types.MethodType):
             return InstanceMethod(method.__func__, method.__self__, method.im_class)
         else:
-            raise ValueError('Not a method')
+            return method
 
 else:
     slot_wrapper = type(object.__init__)
@@ -66,6 +66,6 @@
         elif isinstance(method, method_descriptor):
             cls = method.__objclass__
             return InstanceMethod(method, None, method.__objclass__)
-        raise ValueError('Not a method')
+        return method
 
 

From noreply at buildbot.pypy.org  Thu Nov  6 17:43:44 2014
From: noreply at buildbot.pypy.org (rlamy)
Date: Thu,  6 Nov 2014 17:43:44 +0100 (CET)
Subject: [pypy-commit] pypy online-transforms: hg merge default
Message-ID: <20141106164344.23B0C1C030E@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy 
Branch: online-transforms
Changeset: r74355:b4bdf3e9a6b6
Date: 2014-11-06 16:41 +0000
http://bitbucket.org/pypy/pypy/changeset/b4bdf3e9a6b6/

Log:	hg merge default

diff --git a/README.rst b/README.rst
--- a/README.rst
+++ b/README.rst
@@ -37,4 +37,4 @@
 to use virtualenv with the resulting pypy-c as the interpreter; you can
 find more details about various installation schemes here:
 
-http://doc.pypy.org/en/latest/getting-started.html#installing-pypy
+    http://doc.pypy.org/en/latest/install.html
diff --git a/lib_pypy/grp.py b/lib_pypy/grp.py
--- a/lib_pypy/grp.py
+++ b/lib_pypy/grp.py
@@ -66,11 +66,12 @@
 
 @builtinify
 def getgrnam(name):
-    if not isinstance(name, str):
+    if not isinstance(name, basestring):
         raise TypeError("expected string")
+    name = str(name)
     res = libc.getgrnam(name)
     if not res:
-        raise KeyError(name)
+        raise KeyError("'getgrnam(): name not found: %s'" % name)
     return _group_from_gstruct(res)
 
 @builtinify
diff --git a/pypy/config/pypyoption.py b/pypy/config/pypyoption.py
--- a/pypy/config/pypyoption.py
+++ b/pypy/config/pypyoption.py
@@ -86,9 +86,10 @@
     # itself needs the interp-level struct module
     # because 'P' is missing from the app-level one
     "_rawffi": [("objspace.usemodules.struct", True)],
-    "cpyext": [("translation.secondaryentrypoints", "cpyext,main"),
-               ("translation.shared", sys.platform == "win32")],
+    "cpyext": [("translation.secondaryentrypoints", "cpyext,main")],
 }
+if sys.platform == "win32":
+    module_suggests["cpyext"].append(("translation.shared", True))
 
 module_import_dependencies = {
     # no _rawffi if importing rpython.rlib.clibffi raises ImportError
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -39,3 +39,7 @@
 .. branch: kill-multimethod
 
 Kill multimethod machinery, all multimethods were removed earlier.
+
+.. branch nditer-external_loop
+
+Implement `external_loop` arguement to numpy's nditer
diff --git a/pypy/interpreter/executioncontext.py b/pypy/interpreter/executioncontext.py
--- a/pypy/interpreter/executioncontext.py
+++ b/pypy/interpreter/executioncontext.py
@@ -32,6 +32,17 @@
         self.compiler = space.createcompiler()
         self.profilefunc = None
         self.w_profilefuncarg = None
+        self.thread_disappeared = False   # might be set to True after os.fork()
+
+    @staticmethod
+    def _mark_thread_disappeared(space):
+        # Called in the child process after os.fork() by interp_posix.py.
+        # Marks all ExecutionContexts except the current one
+        # with 'thread_disappeared = True'.
+        me = space.getexecutioncontext()
+        for ec in space.threadlocals.getallvalues().values():
+            if ec is not me:
+                ec.thread_disappeared = True
 
     def gettopframe(self):
         return self.topframeref()
diff --git a/pypy/module/_file/interp_stream.py b/pypy/module/_file/interp_stream.py
--- a/pypy/module/_file/interp_stream.py
+++ b/pypy/module/_file/interp_stream.py
@@ -34,8 +34,12 @@
         # this function runs with the GIL acquired so there is no race
         # condition in the creation of the lock
         me = self.space.getexecutioncontext()   # used as thread ident
-        if self.slockowner is me:
-            return False    # already acquired by the current thread
+        if self.slockowner is not None:
+            if self.slockowner is me:
+                return False    # already acquired by the current thread
+            if self.slockowner.thread_disappeared:
+                self.slockowner = None
+                self.slock = None
         try:
             if self.slock is None:
                 self.slock = self.space.allocate_lock()
diff --git a/pypy/module/micronumpy/concrete.py b/pypy/module/micronumpy/concrete.py
--- a/pypy/module/micronumpy/concrete.py
+++ b/pypy/module/micronumpy/concrete.py
@@ -449,7 +449,7 @@
                 strides.reverse()
                 backstrides.reverse()
                 new_shape.reverse()
-            return SliceArray(self.start, strides, backstrides, new_shape,
+            return self.__class__(self.start, strides, backstrides, new_shape,
                               self, orig_array)
         new_strides = calc_new_strides(new_shape, self.get_shape(),
                                        self.get_strides(),
@@ -460,10 +460,16 @@
         new_backstrides = [0] * len(new_shape)
         for nd in range(len(new_shape)):
             new_backstrides[nd] = (new_shape[nd] - 1) * new_strides[nd]
-        return SliceArray(self.start, new_strides, new_backstrides, new_shape,
+        return self.__class__(self.start, new_strides, new_backstrides, new_shape,
                           self, orig_array)
 
 
+class NonWritableSliceArray(SliceArray):
+    def descr_setitem(self, space, orig_array, w_index, w_value):
+        raise OperationError(space.w_ValueError, space.wrap(
+            "assignment destination is read-only"))
+
+
 class VoidBoxStorage(BaseConcreteArray):
     def __init__(self, size, dtype):
         self.storage = alloc_raw_storage(size)
diff --git a/pypy/module/micronumpy/iterators.py b/pypy/module/micronumpy/iterators.py
--- a/pypy/module/micronumpy/iterators.py
+++ b/pypy/module/micronumpy/iterators.py
@@ -8,8 +8,8 @@
 At which byte in x.data does the item x[3,4] begin?
 if x.strides==[1,5]:
     pData = x.pData + (x.start + 3*1 + 4*5)*sizeof(x.pData[0])
-    pData = x.pData + (x.start + 24) * sizeof(x.pData[0])
-so the offset of the element is 24 elements after the first
+    pData = x.pData + (x.start + 23) * sizeof(x.pData[0])
+so the offset of the element is 23 elements after the first
 
 What is the next element in x after coordinates [3,4]?
 if x.order =='C':
@@ -33,7 +33,7 @@
   which is x.strides[1] * (x.shape[1] - 1) + x.strides[0]
 so if we precalculate the overflow backstride as
 [x.strides[i] * (x.shape[i] - 1) for i in range(len(x.shape))]
-we can go faster.
+we can do only addition while iterating
 All the calculations happen in next()
 """
 from rpython.rlib import jit
@@ -41,6 +41,16 @@
 from pypy.module.micronumpy.base import W_NDimArray
 from pypy.module.micronumpy.flagsobj import _update_contiguous_flags
 
+class OpFlag(object):
+    def __init__(self):
+        self.rw = ''
+        self.broadcast = True
+        self.force_contig = False
+        self.force_align = False
+        self.native_byte_order = False
+        self.tmp_copy = ''
+        self.allocate = False
+
 
 class PureShapeIter(object):
     def __init__(self, shape, idx_w):
@@ -89,11 +99,13 @@
 class ArrayIter(object):
     _immutable_fields_ = ['contiguous', 'array', 'size', 'ndim_m1', 'shape_m1[*]',
                           'strides[*]', 'backstrides[*]', 'factors[*]',
-                          'track_index']
+                          'slice_shape', 'slice_stride', 'slice_backstride',
+                          'track_index', 'operand_type', 'slice_operand_type']
 
     track_index = True
 
-    def __init__(self, array, size, shape, strides, backstrides):
+    def __init__(self, array, size, shape, strides, backstrides, op_flags=OpFlag()):
+        from pypy.module.micronumpy import concrete
         assert len(shape) == len(strides) == len(backstrides)
         _update_contiguous_flags(array)
         self.contiguous = (array.flags & NPY.ARRAY_C_CONTIGUOUS and
@@ -105,6 +117,12 @@
         self.shape_m1 = [s - 1 for s in shape]
         self.strides = strides
         self.backstrides = backstrides
+        self.slice_shape = 1
+        self.slice_stride = -1
+        if strides:
+            self.slice_stride = strides[-1]
+        self.slice_backstride = 1
+        self.slice_operand_type = concrete.SliceArray
 
         ndim = len(shape)
         factors = [0] * ndim
@@ -114,6 +132,10 @@
             else:
                 factors[ndim-i-1] = factors[ndim-i] * shape[ndim-i]
         self.factors = factors
+        if op_flags.rw == 'r':
+            self.operand_type = concrete.ConcreteNonWritableArrayWithBase
+        else:
+            self.operand_type = concrete.ConcreteArrayWithBase
 
     @jit.unroll_safe
     def reset(self, state=None):
@@ -193,6 +215,12 @@
         assert state.iterator is self
         self.array.setitem(state.offset, elem)
 
+    def getoperand(self, st, base):
+        impl = self.operand_type
+        res = impl([], self.array.dtype, self.array.order, [], [],
+                   self.array.storage, base)
+        res.start = st.offset
+        return res
 
 def AxisIter(array, shape, axis, cumulative):
     strides = array.get_strides()
@@ -216,3 +244,42 @@
         size /= shape[axis]
     shape[axis] = backstrides[axis] = 0
     return ArrayIter(array, size, shape, array.strides, backstrides)
+
+class SliceIter(ArrayIter):
+    '''
+    used with external loops, getitem and setitem return a SliceArray
+    view into the original array
+    '''
+    _immutable_fields_ = ['base', 'slice_shape[*]', 'slice_stride[*]', 'slice_backstride[*]']
+
+    def __init__(self, array, size, shape, strides, backstrides, slice_shape,
+                 slice_stride, slice_backstride, op_flags, base):
+        from pypy.module.micronumpy import concrete
+        ArrayIter.__init__(self, array, size, shape, strides, backstrides, op_flags)
+        self.slice_shape = slice_shape
+        self.slice_stride = slice_stride
+        self.slice_backstride = slice_backstride
+        self.base = base
+        if op_flags.rw == 'r':
+            self.slice_operand_type = concrete.NonWritableSliceArray
+        else:
+            self.slice_operand_type = concrete.SliceArray
+
+    def getitem(self, state):
+        # XXX cannot be called - must return a boxed value
+        assert False
+
+    def getitem_bool(self, state):
+        # XXX cannot be called - must return a boxed value
+        assert False
+
+    def setitem(self, state, elem):
+        # XXX cannot be called - must return a boxed value
+        assert False
+
+    def getoperand(self, state, base):
+        assert state.iterator is self
+        impl = self.slice_operand_type
+        arr = impl(state.offset, [self.slice_stride], [self.slice_backstride],
+                   [self.slice_shape], self.array, self.base)
+        return arr
diff --git a/pypy/module/micronumpy/ndarray.py b/pypy/module/micronumpy/ndarray.py
--- a/pypy/module/micronumpy/ndarray.py
+++ b/pypy/module/micronumpy/ndarray.py
@@ -83,8 +83,12 @@
         raise OperationError(space.w_AttributeError, space.wrap(
             "Cannot delete array dtype"))
 
+    def ndims(self):
+        return len(self.get_shape())
+    ndims._always_inline_ = True
+
     def descr_get_ndim(self, space):
-        return space.wrap(len(self.get_shape()))
+        return space.wrap(self.ndims())
 
     def descr_get_itemsize(self, space):
         return space.wrap(self.get_dtype().elsize)
@@ -103,14 +107,14 @@
         return space.wrap(loop.tostring(space, self))
 
     def getitem_filter(self, space, arr):
-        if len(arr.get_shape()) > 1 and arr.get_shape() != self.get_shape():
+        if arr.ndims() > 1 and arr.get_shape() != self.get_shape():
             raise OperationError(space.w_ValueError, space.wrap(
                 "boolean index array should have 1 dimension"))
         if arr.get_size() > self.get_size():
             raise OperationError(space.w_ValueError, space.wrap(
                 "index out of range for array"))
         size = loop.count_all_true(arr)
-        if len(arr.get_shape()) == 1:
+        if arr.ndims() == 1:
             res_shape = [size] + self.get_shape()[1:]
         else:
             res_shape = [size]
@@ -119,7 +123,7 @@
         return loop.getitem_filter(w_res, self, arr)
 
     def setitem_filter(self, space, idx, val):
-        if len(idx.get_shape()) > 1 and idx.get_shape() != self.get_shape():
+        if idx.ndims() > 1 and idx.get_shape() != self.get_shape():
             raise OperationError(space.w_ValueError, space.wrap(
                 "boolean index array should have 1 dimension"))
         if idx.get_size() > self.get_size():
@@ -210,7 +214,7 @@
         if space.is_w(w_idx, space.w_Ellipsis):
             return self
         elif isinstance(w_idx, W_NDimArray) and w_idx.get_dtype().is_bool() \
-                and len(w_idx.get_shape()) > 0:
+                and w_idx.ndims() > 0:
             return self.getitem_filter(space, w_idx)
         try:
             return self.implementation.descr_getitem(space, self, w_idx)
@@ -228,7 +232,7 @@
             self.implementation.setslice(space, convert_to_array(space, w_value))
             return
         elif isinstance(w_idx, W_NDimArray) and w_idx.get_dtype().is_bool() \
-                and len(w_idx.get_shape()) > 0:
+                and w_idx.ndims() > 0:
             self.setitem_filter(space, w_idx, convert_to_array(space, w_value))
             return
         try:
@@ -289,7 +293,7 @@
             shape=shape, backward_broadcast=backward_broadcast)
 
     def is_scalar(self):
-        return len(self.get_shape()) == 0
+        return self.ndims() == 0
 
     def set_scalar_value(self, w_val):
         return self.implementation.setitem(self.implementation.start, w_val)
@@ -408,7 +412,7 @@
         """
         if axis1 == axis2:
             return self
-        n = len(self.get_shape())
+        n = self.ndims()
         if n <= 1:
             return self
         if axis1 < 0:
@@ -426,7 +430,7 @@
         return self.implementation.nonzero(space, index_type)
 
     def descr_tolist(self, space):
-        if len(self.get_shape()) == 0:
+        if self.ndims() == 0:
             return self.get_scalar_value().item(space)
         l_w = []
         for i in range(self.get_shape()[0]):
@@ -514,7 +518,7 @@
         if len(args_w) == 0:
             raise OperationError(space.w_ValueError, space.wrap(
                 "itemset must have at least one argument"))
-        if len(args_w) != len(self.get_shape()) + 1:
+        if len(args_w) != self.ndims() + 1:
             raise OperationError(space.w_ValueError, space.wrap(
                 "incorrect number of indices for array"))
         self.descr_setitem(space, space.newtuple(args_w[:-1]), args_w[-1])
@@ -647,14 +651,14 @@
 
     @unwrap_spec(offset=int, axis1=int, axis2=int)
     def descr_diagonal(self, space, offset=0, axis1=0, axis2=1):
-        if len(self.get_shape()) < 2:
+        if self.ndims() < 2:
             raise OperationError(space.w_ValueError, space.wrap(
                 "need at least 2 dimensions for diagonal"))
-        if (axis1 < 0 or axis2 < 0 or axis1 >= len(self.get_shape()) or
-                axis2 >= len(self.get_shape())):
+        if (axis1 < 0 or axis2 < 0 or axis1 >= self.ndims() or
+                axis2 >= self.ndims()):
             raise oefmt(space.w_ValueError,
                         "axis1(=%d) and axis2(=%d) must be withing range "
-                        "(ndim=%d)", axis1, axis2, len(self.get_shape()))
+                        "(ndim=%d)", axis1, axis2, self.ndims())
         if axis1 == axis2:
             raise OperationError(space.w_ValueError, space.wrap(
                 "axis1 and axis2 cannot be the same"))
@@ -733,7 +737,7 @@
             raise OperationError(space.w_NotImplementedError, space.wrap(
                 'sorter not supported in searchsort'))
         side = searchside_converter(space, w_side)
-        if len(self.get_shape()) != 1:
+        if self.ndims() != 1:
             raise oefmt(space.w_ValueError, "a must be a 1-d array")
         v = convert_to_array(space, w_v)
         ret = W_NDimArray.from_shape(
@@ -972,7 +976,7 @@
         if other.is_scalar():
             #Note: w_out is not modified, this is numpy compliant.
             return self.descr_mul(space, other)
-        elif len(self.get_shape()) < 2 and len(other.get_shape()) < 2:
+        elif self.ndims() < 2 and other.ndims() < 2:
             w_res = self.descr_mul(space, other)
             assert isinstance(w_res, W_NDimArray)
             return w_res.descr_sum(space, space.wrap(-1), out)
@@ -989,7 +993,7 @@
                 matches = False
             elif not out.implementation.order == "C":
                 matches = False
-            elif len(out.get_shape()) != len(out_shape):
+            elif out.ndims() != len(out_shape):
                 matches = False
             else:
                 for i in range(len(out_shape)):
diff --git a/pypy/module/micronumpy/nditer.py b/pypy/module/micronumpy/nditer.py
--- a/pypy/module/micronumpy/nditer.py
+++ b/pypy/module/micronumpy/nditer.py
@@ -5,7 +5,7 @@
 from pypy.module.micronumpy import ufuncs, support, concrete
 from pypy.module.micronumpy.base import W_NDimArray, convert_to_array
 from pypy.module.micronumpy.descriptor import decode_w_dtype
-from pypy.module.micronumpy.iterators import ArrayIter
+from pypy.module.micronumpy.iterators import ArrayIter, SliceIter, OpFlag
 from pypy.module.micronumpy.strides import (calculate_broadcast_strides,
                                             shape_agreement, shape_agreement_multiple)
 
@@ -35,17 +35,6 @@
     return ret
 
 
-class OpFlag(object):
-    def __init__(self):
-        self.rw = ''
-        self.broadcast = True
-        self.force_contig = False
-        self.force_align = False
-        self.native_byte_order = False
-        self.tmp_copy = ''
-        self.allocate = False
-
-
 def parse_op_flag(space, lst):
     op_flag = OpFlag()
     for w_item in lst:
@@ -71,17 +60,17 @@
         elif item == 'allocate':
             op_flag.allocate = True
         elif item == 'no_subtype':
-            raise OperationError(space.w_NotImplementedError, space.wrap(
-                '"no_subtype" op_flag not implemented yet'))
+            raise oefmt(space.w_NotImplementedError,
+                '"no_subtype" op_flag not implemented yet')
         elif item == 'arraymask':
-            raise OperationError(space.w_NotImplementedError, space.wrap(
-                '"arraymask" op_flag not implemented yet'))
+            raise oefmt(space.w_NotImplementedError,
+                '"arraymask" op_flag not implemented yet')
         elif item == 'writemask':
-            raise OperationError(space.w_NotImplementedError, space.wrap(
-                '"writemask" op_flag not implemented yet'))
+            raise oefmt(space.w_NotImplementedError,
+                '"writemask" op_flag not implemented yet')
         else:
-            raise OperationError(space.w_ValueError, space.wrap(
-                'op_flags must be a tuple or array of per-op flag-tuples'))
+            raise oefmt(space.w_ValueError,
+                'op_flags must be a tuple or array of per-op flag-tuples')
     if op_flag.rw == '':
         raise oefmt(space.w_ValueError,
                     "None of the iterator flags READWRITE, READONLY, or "
@@ -94,8 +83,8 @@
         return
     elif not space.isinstance_w(w_flags, space.w_tuple) and not \
             space.isinstance_w(w_flags, space.w_list):
-        raise OperationError(space.w_ValueError, space.wrap(
-            'Iter global flags must be a list or tuple of strings'))
+        raise oefmt(space.w_ValueError,
+            'Iter global flags must be a list or tuple of strings')
     lst = space.listview(w_flags)
     for w_item in lst:
         if not space.isinstance_w(w_item, space.w_str) and not \
@@ -106,12 +95,10 @@
                         typename)
         item = space.str_w(w_item)
         if item == 'external_loop':
-            raise OperationError(space.w_NotImplementedError, space.wrap(
-                'nditer external_loop not implemented yet'))
             nditer.external_loop = True
         elif item == 'buffered':
-            raise OperationError(space.w_NotImplementedError, space.wrap(
-                'nditer buffered not implemented yet'))
+            raise oefmt(space.w_NotImplementedError,
+                'nditer buffered not implemented yet')
             # For numpy compatability
             nditer.buffered = True
         elif item == 'c_index':
@@ -131,8 +118,8 @@
         elif item == 'refs_ok':
             nditer.refs_ok = True
         elif item == 'reduce_ok':
-            raise OperationError(space.w_NotImplementedError, space.wrap(
-                'nditer reduce_ok not implemented yet'))
+            raise oefmt(space.w_NotImplementedError,
+                'nditer reduce_ok not implemented yet')
             nditer.reduce_ok = True
         elif item == 'zerosize_ok':
             nditer.zerosize_ok = True
@@ -141,9 +128,9 @@
                         'Unexpected iterator global flag "%s"',
                         item)
     if nditer.tracked_index and nditer.external_loop:
-        raise OperationError(space.w_ValueError, space.wrap(
+        raise oefmt(space.w_ValueError,
             'Iterator flag EXTERNAL_LOOP cannot be used if an index or '
-            'multi-index is being tracked'))
+            'multi-index is being tracked')
 
 
 def is_backward(imp, order):
@@ -155,11 +142,11 @@
         raise NotImplementedError('not implemented yet')
 
 
-def get_iter(space, order, arr, shape, dtype):
+def get_iter(space, order, arr, shape, dtype, op_flags):
     imp = arr.implementation
     backward = is_backward(imp, order)
     if arr.is_scalar():
-        return ArrayIter(imp, 1, [], [], [])
+        return ArrayIter(imp, 1, [], [], [], op_flags=op_flags)
     if (imp.strides[0] < imp.strides[-1] and not backward) or \
        (imp.strides[0] > imp.strides[-1] and backward):
         # flip the strides. Is this always true for multidimension?
@@ -174,8 +161,103 @@
         backstrides = imp.backstrides
     r = calculate_broadcast_strides(strides, backstrides, imp.shape,
                                     shape, backward)
-    return ArrayIter(imp, imp.get_size(), shape, r[0], r[1])
+    return ArrayIter(imp, imp.get_size(), shape, r[0], r[1], op_flags=op_flags)
 
+def calculate_ndim(op_in, oa_ndim):
+    if oa_ndim >=0:
+        return oa_ndim
+    else:
+        ndim = 0
+        for op in op_in:
+            if op is None:
+                continue
+            assert isinstance(op, W_NDimArray)
+            ndim = max(ndim, op.ndims())
+    return ndim
+
+def coalesce_axes(it, space):
+    # Copy logic from npyiter_coalesce_axes, used in ufunc iterators
+    # and in nditer's with 'external_loop' flag
+    can_coalesce = True
+    if it.order == 'F':
+        fastest = 0
+    else:
+        fastest = -1
+    for idim in range(it.ndim - 1):
+        for op_it, _ in it.iters:
+            if op_it is None:
+                continue
+            assert isinstance(op_it, ArrayIter)
+            indx = len(op_it.strides)
+            if it.order == 'F':
+                indx = len(op_it.array.strides) - indx
+                assert indx >=0
+                astrides = op_it.array.strides[indx:]
+            else:
+                astrides = op_it.array.strides[:indx]
+            # does op_it iters over array "naturally"
+            if astrides != op_it.strides:
+                can_coalesce = False
+                break
+        if can_coalesce:
+            for i in range(len(it.iters)):
+                old_iter = it.iters[i][0]
+                shape = [s+1 for s in old_iter.shape_m1]
+                strides = old_iter.strides
+                backstrides = old_iter.backstrides
+                if it.order == 'F':
+                    new_shape = shape[1:]
+                    new_strides = strides[1:]
+                    new_backstrides = backstrides[1:]
+                    _stride = min(strides[0], old_iter.slice_stride)
+                else:
+                    new_shape = shape[:-1]
+                    new_strides = strides[:-1]
+                    new_backstrides = backstrides[:-1]
+                    _stride = old_iter.slice_stride
+                # We always want the "fastest" iterator in external loops
+                _shape = shape[fastest] * old_iter.slice_shape
+                _backstride = (_shape - 1) * _stride
+                new_iter = SliceIter(old_iter.array, old_iter.size / shape[fastest],
+                            new_shape, new_strides, new_backstrides,
+                            _shape, _stride, _backstride,
+                            it.op_flags[i], it)
+                it.iters[i] = (new_iter, new_iter.reset())
+            if len(it.shape) > 1:
+                if it.order == 'F':
+                    it.shape = it.shape[1:]
+                else:
+                    it.shape = it.shape[:-1]
+            else:
+                it.shape = [1]
+        else:
+            break
+    # Always coalesce at least one
+    for i in range(len(it.iters)):
+        old_iter = it.iters[i][0]
+        shape = [s+1 for s in old_iter.shape_m1]
+        strides = old_iter.strides
+        backstrides = old_iter.backstrides
+        new_shape = shape[:-1]
+        new_strides = strides[:-1]
+        new_backstrides = backstrides[:-1]
+        _shape = shape[-1] * old_iter.slice_shape
+        # use the operand's iterator's rightmost stride,
+        # even if it is not the fastest (for 'F' or swapped axis)
+        _stride = old_iter.slice_stride
+        _backstride = (_shape - 1) * _stride
+        new_iter = SliceIter(old_iter.array, old_iter.size / shape[-1],
+                    new_shape, new_strides, new_backstrides,
+                    _shape, _stride, _backstride,
+                    it.op_flags[i], it)
+        it.iters[i] = (new_iter, new_iter.reset())
+    if len(it.shape) > 1:
+        if it.order == 'F':
+            it.shape = it.shape[1:]
+        else:
+            it.shape = it.shape[:-1]
+    else:
+        it.shape = [1]
 
 class IndexIterator(object):
     def __init__(self, shape, backward=False):
@@ -205,6 +287,7 @@
 
 
 class W_NDIter(W_Root):
+    _immutable_fields_ = ['ndim', ]
     def __init__(self, space, w_seq, w_flags, w_op_flags, w_op_dtypes, w_casting,
                  w_op_axes, w_itershape, w_buffersize, order):
         self.order = order
@@ -236,28 +319,29 @@
         self.op_flags = parse_op_arg(space, 'op_flags', w_op_flags,
                                      len(self.seq), parse_op_flag)
         # handle w_op_axes
+        oa_ndim = -1
         if not space.is_none(w_op_axes):
-            self.set_op_axes(space, w_op_axes)
+            oa_ndim = self.set_op_axes(space, w_op_axes)
+        self.ndim = calculate_ndim(self.seq, oa_ndim)
 
         # handle w_op_dtypes part 1: creating self.dtypes list from input
         if not space.is_none(w_op_dtypes):
             w_seq_as_list = space.listview(w_op_dtypes)
             self.dtypes = [decode_w_dtype(space, w_elem) for w_elem in w_seq_as_list]
             if len(self.dtypes) != len(self.seq):
-                raise OperationError(space.w_ValueError, space.wrap(
-                    "op_dtypes must be a tuple/list matching the number of ops"))
+                raise oefmt(space.w_ValueError,
+                    "op_dtypes must be a tuple/list matching the number of ops")
         else:
             self.dtypes = []
 
         # handle None or writable operands, calculate my shape
-        self.iters = []
         outargs = [i for i in range(len(self.seq))
                    if self.seq[i] is None or self.op_flags[i].rw == 'w']
         if len(outargs) > 0:
             out_shape = shape_agreement_multiple(space, [self.seq[i] for i in outargs])
         else:
             out_shape = None
-        self.shape = iter_shape = shape_agreement_multiple(space, self.seq,
+        self.shape = shape_agreement_multiple(space, self.seq,
                                                            shape=out_shape)
         if len(outargs) > 0:
             # Make None operands writeonly and flagged for allocation
@@ -276,11 +360,11 @@
             for i in outargs:
                 if self.seq[i] is None:
                     # XXX can we postpone allocation to later?
-                    self.seq[i] = W_NDimArray.from_shape(space, iter_shape, out_dtype)
+                    self.seq[i] = W_NDimArray.from_shape(space, self.shape, out_dtype)
                 else:
                     if not self.op_flags[i].broadcast:
                         # Raises if ooutput cannot be broadcast
-                        shape_agreement(space, iter_shape, self.seq[i], False)
+                        shape_agreement(space, self.shape, self.seq[i], False)
 
         if self.tracked_index != "":
             if self.order == "K":
@@ -289,7 +373,7 @@
                 backward = False
             else:
                 backward = self.order != self.tracked_index
-            self.index_iter = IndexIterator(iter_shape, backward=backward)
+            self.index_iter = IndexIterator(self.shape, backward=backward)
 
         # handle w_op_dtypes part 2: copy where needed if possible
         if len(self.dtypes) > 0:
@@ -311,49 +395,49 @@
             self.dtypes = [s.get_dtype() for s in self.seq]
 
         # create an iterator for each operand
+        self.iters = []
         for i in range(len(self.seq)):
-            it = get_iter(space, self.order, self.seq[i], iter_shape, self.dtypes[i])
+            it = get_iter(space, self.order, self.seq[i], self.shape,
+                          self.dtypes[i], self.op_flags[i])
             it.contiguous = False
             self.iters.append((it, it.reset()))
 
+        if self.external_loop:
+            coalesce_axes(self, space)
+
     def set_op_axes(self, space, w_op_axes):
         if space.len_w(w_op_axes) != len(self.seq):
             raise oefmt(space.w_ValueError,
                         "op_axes must be a tuple/list matching the number of ops")
         op_axes = space.listview(w_op_axes)
-        l = -1
+        oa_ndim = -1
         for w_axis in op_axes:
             if not space.is_none(w_axis):
                 axis_len = space.len_w(w_axis)
-                if l == -1:
-                    l = axis_len
-                elif axis_len != l:
+                if oa_ndim == -1:
+                    oa_ndim = axis_len
+                elif axis_len != oa_ndim:
                     raise oefmt(space.w_ValueError,
                                 "Each entry of op_axes must have the same size")
                 self.op_axes.append([space.int_w(x) if not space.is_none(x) else -1
                                      for x in space.listview(w_axis)])
-        if l == -1:
+        if oa_ndim == -1:
             raise oefmt(space.w_ValueError,
                         "If op_axes is provided, at least one list of axes "
                         "must be contained within it")
-        raise Exception('xxx TODO')
+        raise oefmt(space.w_NotImplementedError, "op_axis not finished yet")
         # Check that values make sense:
         # - in bounds for each operand
         # ValueError: Iterator input op_axes[0][3] (==3) is not a valid axis of op[0], which has 2 dimensions
         # - no repeat axis
         # ValueError: The 'op_axes' provided to the iterator constructor for operand 1 contained duplicate value 0
+        return oa_ndim
 
     def descr_iter(self, space):
         return space.wrap(self)
 
-    def getitem(self, it, st, op_flags):
-        if op_flags.rw == 'r':
-            impl = concrete.ConcreteNonWritableArrayWithBase
-        else:
-            impl = concrete.ConcreteArrayWithBase
-        res = impl([], it.array.dtype, it.array.order, [], [],
-                   it.array.storage, self)
-        res.start = st.offset
+    def getitem(self, it, st):
+        res = it.getoperand(st, self)
         return W_NDimArray(res)
 
     def descr_getitem(self, space, w_idx):
@@ -363,7 +447,7 @@
         except IndexError:
             raise oefmt(space.w_IndexError,
                         "Iterator operand index %d is out of bounds", idx)
-        return self.getitem(it, st, self.op_flags[idx])
+        return self.getitem(it, st)
 
     def descr_setitem(self, space, w_idx, w_value):
         raise oefmt(space.w_NotImplementedError, "not implemented yet")
@@ -385,7 +469,7 @@
             else:
                 self.first_next = False
         for i, (it, st) in enumerate(self.iters):
-            res.append(self.getitem(it, st, self.op_flags[i]))
+            res.append(self.getitem(it, st))
             self.iters[i] = (it, it.next(st))
         if len(res) < 2:
             return res[0]
@@ -477,7 +561,7 @@
         raise oefmt(space.w_NotImplementedError, "not implemented yet")
 
     def descr_get_ndim(self, space):
-        raise oefmt(space.w_NotImplementedError, "not implemented yet")
+        return space.wrap(self.ndim)
 
     def descr_get_nop(self, space):
         raise oefmt(space.w_NotImplementedError, "not implemented yet")
diff --git a/pypy/module/micronumpy/test/test_nditer.py b/pypy/module/micronumpy/test/test_nditer.py
--- a/pypy/module/micronumpy/test/test_nditer.py
+++ b/pypy/module/micronumpy/test/test_nditer.py
@@ -63,9 +63,6 @@
         from numpy import arange, nditer, array
         a = arange(24).reshape(2, 3, 4)
         import sys
-        if '__pypy__' in sys.builtin_module_names:
-            raises(NotImplementedError, nditer, a, flags=['external_loop'])
-            skip('nditer external_loop not implmented')
         r = []
         n = 0
         for x in nditer(a, flags=['external_loop']):
@@ -79,7 +76,9 @@
             r.append(x)
             n += 1
         assert n == 12
-        assert (array(r) == [[ 0, 12], [ 4, 16], [ 8, 20], [ 1, 13], [ 5, 17], [ 9, 21], [ 2, 14], [ 6, 18], [10, 22], [ 3, 15], [ 7, 19], [11, 23]]).all()
+        assert (array(r) == [[ 0, 12], [ 4, 16], [ 8, 20], [ 1, 13], [ 5, 17], [ 9, 21],
+                             [ 2, 14], [ 6, 18], [10, 22], [ 3, 15], [ 7, 19], [11, 23],
+                            ]).all()
         e = raises(ValueError, 'r[0][0] = 0')
         assert str(e.value) == 'assignment destination is read-only'
         r = []
@@ -222,9 +221,6 @@
     def test_outarg(self):
         from numpy import nditer, zeros, arange
         import sys
-        if '__pypy__' in sys.builtin_module_names:
-            raises(NotImplementedError, nditer, [1, 2], flags=['external_loop'])
-            skip('nditer external_loop not implmented')
 
         def square1(a):
             it = nditer([a, None])
@@ -233,6 +229,9 @@
             return it.operands[1]
         assert (square1([1, 2, 3]) == [1, 4, 9]).all()
 
+        if '__pypy__' in sys.builtin_module_names:
+            raises(NotImplementedError, nditer, [1, 2], flags=['buffered'])
+            skip('nditer buffered not implmented')
         def square2(a, out=None):
             it = nditer([a, out], flags=['external_loop', 'buffered'],
                         op_flags=[['readonly'],
@@ -252,10 +251,11 @@
         from numpy import nditer, arange
         a = arange(3)
         import sys
+        b = arange(8).reshape(2,4)
         if '__pypy__' in sys.builtin_module_names:
-            raises(NotImplementedError, nditer, a, flags=['external_loop'])
-            skip('nditer external_loop not implmented')
-        b = arange(8).reshape(2,4)
+            raises(NotImplementedError, nditer, [a, b, None], flags=['external_loop'],
+                   op_axes=[[0, -1, -1], [-1, 0, 1], None])
+            skip('nditer op_axes not implemented yet')
         it = nditer([a, b, None], flags=['external_loop'],
                     op_axes=[[0, -1, -1], [-1, 0, 1], None])
         for x, y, z in it:
diff --git a/pypy/module/posix/interp_posix.py b/pypy/module/posix/interp_posix.py
--- a/pypy/module/posix/interp_posix.py
+++ b/pypy/module/posix/interp_posix.py
@@ -10,6 +10,7 @@
 
 from pypy.interpreter.gateway import unwrap_spec
 from pypy.interpreter.error import OperationError, wrap_oserror, wrap_oserror2
+from pypy.interpreter.executioncontext import ExecutionContext
 from pypy.module.sys.interp_encoding import getfilesystemencoding
 
 
@@ -721,6 +722,8 @@
     "NOT_RPYTHON"
     get_fork_hooks(where).append(hook)
 
+add_fork_hook('child', ExecutionContext._mark_thread_disappeared)
+
 @specialize.arg(0)
 def run_fork_hooks(where, space):
     for hook in get_fork_hooks(where):
diff --git a/pypy/module/test_lib_pypy/test_grp_extra.py b/pypy/module/test_lib_pypy/test_grp_extra.py
--- a/pypy/module/test_lib_pypy/test_grp_extra.py
+++ b/pypy/module/test_lib_pypy/test_grp_extra.py
@@ -9,7 +9,8 @@
                                     "No grp module on this platform")
 
     def test_basic(self):
-        raises(KeyError, self.grp.getgrnam, "dEkLofcG")
+        e = raises(KeyError, self.grp.getgrnam, "dEkLofcG")
+        assert e.value.args[0] == "'getgrnam(): name not found: dEkLofcG'"
         for name in ["root", "wheel"]:
             try:
                 g = self.grp.getgrnam(name)
@@ -19,6 +20,8 @@
             assert 'root' in g.gr_mem or g.gr_mem == []
             assert g.gr_name == name
             assert isinstance(g.gr_passwd, str)    # usually just 'x', don't hope :-)
+            g2 = self.grp.getgrnam(unicode(name))
+            assert g2 == g
             break
         else:
             raise
diff --git a/pypy/module/test_lib_pypy/test_posix_extra.py b/pypy/module/test_lib_pypy/test_posix_extra.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/test_lib_pypy/test_posix_extra.py
@@ -0,0 +1,40 @@
+import py
+import sys, os, subprocess
+
+
+CODE = """
+import sys, os, thread, time
+
+fd1, fd2 = os.pipe()
+f1 = os.fdopen(fd1, 'r', 0)
+f2 = os.fdopen(fd2, 'w', 0)
+
+def f():
+    print "thread started"
+    x = f1.read(1)
+    assert x == "X"
+    print "thread exit"
+
+thread.start_new_thread(f, ())
+time.sleep(0.5)
+if os.fork() == 0:   # in the child
+    time.sleep(0.5)
+    x = f1.read(1)
+    assert x == "Y"
+    print "ok!"
+    sys.exit()
+
+f2.write("X")   # in the parent
+f2.write("Y")   # in the parent
+time.sleep(1.0)
+"""
+
+
+def test_thread_fork_file_lock():
+    if not hasattr(os, 'fork'):
+        py.test.skip("requires 'fork'")
+    output = subprocess.check_output([sys.executable, '-u', '-c', CODE])
+    assert output.splitlines() == [
+        'thread started',
+        'thread exit',
+        'ok!']
diff --git a/rpython/jit/metainterp/history.py b/rpython/jit/metainterp/history.py
--- a/rpython/jit/metainterp/history.py
+++ b/rpython/jit/metainterp/history.py
@@ -279,7 +279,8 @@
             # careful in this comparison: if self.value and other.value
             # are both NaN, stored as regular floats (i.e. on 64-bit),
             # then just using "==" would say False: two NaNs are always
-            # different from each other.
+            # different from each other.  Conversely, "0.0 == -0.0" but
+            # they are not the same constant.
             return (longlong.extract_bits(self.value) ==
                     longlong.extract_bits(other.value))
         return False
diff --git a/rpython/jit/metainterp/pyjitpl.py b/rpython/jit/metainterp/pyjitpl.py
--- a/rpython/jit/metainterp/pyjitpl.py
+++ b/rpython/jit/metainterp/pyjitpl.py
@@ -33,6 +33,14 @@
 
 # ____________________________________________________________
 
+FASTPATHS_SAME_BOXES = {
+    "ne": "history.CONST_FALSE",
+    "eq": "history.CONST_TRUE",
+    "lt": "history.CONST_FALSE",
+    "le": "history.CONST_TRUE",
+    "gt": "history.CONST_FALSE",
+    "ge": "history.CONST_TRUE",
+}
 
 class MIFrame(object):
     debug = False
@@ -188,8 +196,6 @@
     # ------------------------------
 
     for _opimpl in ['int_add', 'int_sub', 'int_mul', 'int_floordiv', 'int_mod',
-                    'int_lt', 'int_le', 'int_eq',
-                    'int_ne', 'int_gt', 'int_ge',
                     'int_and', 'int_or', 'int_xor',
                     'int_rshift', 'int_lshift', 'uint_rshift',
                     'uint_lt', 'uint_le', 'uint_gt', 'uint_ge',
@@ -197,7 +203,6 @@
                     'float_add', 'float_sub', 'float_mul', 'float_truediv',
                     'float_lt', 'float_le', 'float_eq',
                     'float_ne', 'float_gt', 'float_ge',
-                    'ptr_eq', 'ptr_ne', 'instance_ptr_eq', 'instance_ptr_ne',
                     ]:
         exec py.code.Source('''
             @arguments("box", "box")
@@ -205,6 +210,18 @@
                 return self.execute(rop.%s, b1, b2)
         ''' % (_opimpl, _opimpl.upper())).compile()
 
+    for _opimpl in ['int_eq', 'int_ne', 'int_lt', 'int_le', 'int_gt', 'int_ge',
+                    'ptr_eq', 'ptr_ne',
+                    'instance_ptr_eq', 'instance_ptr_ne']:
+        exec py.code.Source('''
+            @arguments("box", "box")
+            def opimpl_%s(self, b1, b2):
+                if b1 is b2: # crude fast check
+                    return %s
+                return self.execute(rop.%s, b1, b2)
+        ''' % (_opimpl, FASTPATHS_SAME_BOXES[_opimpl.split("_")[-1]], _opimpl.upper())
+        ).compile()
+
     for _opimpl in ['int_add_ovf', 'int_sub_ovf', 'int_mul_ovf']:
         exec py.code.Source('''
             @arguments("box", "box")
@@ -340,10 +357,13 @@
         exec py.code.Source('''
             @arguments("box", "box", "label")
             def opimpl_goto_if_not_%s(self, b1, b2, target):
-                condbox = self.execute(rop.%s, b1, b2)
+                if b1 is b2:
+                    condbox = %s
+                else:
+                    condbox = self.execute(rop.%s, b1, b2)
                 self.opimpl_goto_if_not(condbox, target)
-        ''' % (_opimpl, _opimpl.upper())).compile()
-
+        ''' % (_opimpl, FASTPATHS_SAME_BOXES[_opimpl.split("_")[-1]], _opimpl.upper())
+        ).compile()
 
     def _establish_nullity(self, box, orgpc):
         value = box.nonnull()
diff --git a/rpython/jit/metainterp/test/test_ajit.py b/rpython/jit/metainterp/test/test_ajit.py
--- a/rpython/jit/metainterp/test/test_ajit.py
+++ b/rpython/jit/metainterp/test/test_ajit.py
@@ -4119,3 +4119,64 @@
         assert res == 42
         res = self.interp_operations(f, [-42])
         assert res == 0
+
+    def test_cmp_fastpaths(self):
+        class Z: pass
+        def make_int(cmp):
+            def f(x):
+                if cmp == 'eq':
+                    return x == x and x == x
+                if cmp == 'ne':
+                    return x != x or x != x
+                if cmp == 'lt':
+                    return x < x or x != x
+                if cmp == 'le':
+                    return x <= x and x <= x
+                if cmp == 'gt':
+                    return x > x or x > x
+                if cmp == 'ge':
+                    return x >= x and x >= x
+                assert 0
+            return f
+
+        def make_str(cmp):
+            def f(x):
+                x = str(x)
+                if cmp == 'eq':
+                    return x is x or x is x
+                if cmp == 'ne':
+                    return x is not x and x is not x
+                assert 0
+            return f
+
+        def make_object(cmp):
+            def f(x):
+                y = Z()
+                y.x = x
+                x = y
+                if cmp == 'eq':
+                    return x is x
+                if cmp == 'ne':
+                    return x is not x
+                assert 0
+            return f
+
+        for cmp in 'eq ne lt le gt ge'.split():
+            f = make_int(cmp)
+            res = self.interp_operations(f, [42])
+            assert res == f(42)
+            opname = "int_%s" % cmp
+            self.check_operations_history(**{opname: 0})
+
+        for cmp in 'eq ne'.split():
+            f = make_str(cmp)
+            res = self.interp_operations(f, [42])
+            assert res == f(42)
+            opname = "ptr_%s" % cmp
+            self.check_operations_history(**{opname: 0})
+
+            f = make_object(cmp)
+            res = self.interp_operations(f, [42])
+            assert res == f(42)
+            opname = "instance_ptr_%s" % cmp
+            self.check_operations_history(**{opname: 0})
diff --git a/rpython/rlib/objectmodel.py b/rpython/rlib/objectmodel.py
--- a/rpython/rlib/objectmodel.py
+++ b/rpython/rlib/objectmodel.py
@@ -201,6 +201,11 @@
         return result
     return decorator
 
+def always_inline(func):
+    """ mark the function as to-be-inlined by the RPython optimizations (not
+    the JIT!), no matter its size."""
+    func._always_inline_ = True
+    return func
 
 
 # ____________________________________________________________
diff --git a/rpython/rlib/rsre/rsre_core.py b/rpython/rlib/rsre/rsre_core.py
--- a/rpython/rlib/rsre/rsre_core.py
+++ b/rpython/rlib/rsre/rsre_core.py
@@ -137,22 +137,25 @@
     def flatten_marks(self):
         # for testing
         if self.match_marks_flat is None:
-            self.match_marks_flat = [self.match_start, self.match_end]
-            mark = self.match_marks
-            if mark is not None:
-                self.match_lastindex = mark.gid
-            else:
-                self.match_lastindex = -1
-            while mark is not None:
-                index = mark.gid + 2
-                while index >= len(self.match_marks_flat):
-                    self.match_marks_flat.append(-1)
-                if self.match_marks_flat[index] == -1:
-                    self.match_marks_flat[index] = mark.position
-                mark = mark.prev
-            self.match_marks = None    # clear
+            self._compute_flattened_marks()
         return self.match_marks_flat
 
+    def _compute_flattened_marks(self):
+        self.match_marks_flat = [self.match_start, self.match_end]
+        mark = self.match_marks
+        if mark is not None:
+            self.match_lastindex = mark.gid
+        else:
+            self.match_lastindex = -1
+        while mark is not None:
+            index = mark.gid + 2
+            while index >= len(self.match_marks_flat):
+                self.match_marks_flat.append(-1)
+            if self.match_marks_flat[index] == -1:
+                self.match_marks_flat[index] = mark.position
+            mark = mark.prev
+        self.match_marks = None    # clear
+
     def span(self, groupnum=0):
         # compatibility
         fmarks = self.flatten_marks()
diff --git a/rpython/rlib/test/test_objectmodel.py b/rpython/rlib/test/test_objectmodel.py
--- a/rpython/rlib/test/test_objectmodel.py
+++ b/rpython/rlib/test/test_objectmodel.py
@@ -438,6 +438,11 @@
     assert exc.value.message == "f argument 'b' must be of type "
     py.test.raises(TypeError, "f('hello', 'world', 3)")
 
+def test_always_inline():
+    @always_inline
+    def f(a, b, c):
+        return a, b, c
+    assert f._always_inline_ == True
 
 def test_enforceargs_defaults():
     @enforceargs(int, int)
diff --git a/rpython/rtyper/lltypesystem/rffi.py b/rpython/rtyper/lltypesystem/rffi.py
--- a/rpython/rtyper/lltypesystem/rffi.py
+++ b/rpython/rtyper/lltypesystem/rffi.py
@@ -825,7 +825,7 @@
         allows for the process to be performed without an extra copy.
         Make sure to call keep_buffer_alive_until_here on the returned values.
         """
-        new_buf = lltype.malloc(STRTYPE, count)
+        new_buf = mallocfn(count)
         pinned = 0
         if rgc.can_move(new_buf):
             if rgc.pin(new_buf):
@@ -857,7 +857,7 @@
             if llop.shrink_array(lltype.Bool, gc_buf, needed_size):
                 pass     # now 'gc_buf' is smaller
             else:
-                gc_buf = lltype.malloc(STRTYPE, needed_size)
+                gc_buf = mallocfn(needed_size)
                 case_num = 2
         if case_num == 2:
             copy_raw_to_string(raw_buf, gc_buf, 0, needed_size)
diff --git a/rpython/rtyper/module/ll_os.py b/rpython/rtyper/module/ll_os.py
--- a/rpython/rtyper/module/ll_os.py
+++ b/rpython/rtyper/module/ll_os.py
@@ -22,8 +22,6 @@
 from rpython.rtyper.tool import rffi_platform as platform
 from rpython.rlib import rposix
 from rpython.translator.tool.cbuild import ExternalCompilationInfo
-from rpython.rtyper.lltypesystem.llmemory import itemoffsetof, offsetof
-from rpython.rtyper.lltypesystem.rstr import STR
 from rpython.rlib.objectmodel import specialize
 from rpython.translator import cdir
 
@@ -251,12 +249,9 @@
 
     @registering_if(os, 'execv')
     def register_os_execv(self):
-        eci = self.gcc_profiling_bug_workaround(
-            'RPY_EXPORTED_FOR_TESTS int _noprof_execv(char *path, char *argv[])',
-            'return execv(path, argv);')
-        os_execv = self.llexternal('_noprof_execv',
-                                   [rffi.CCHARP, rffi.CCHARPP],
-                                   rffi.INT, compilation_info = eci)
+        os_execv = self.llexternal(
+            'execv',
+            [rffi.CCHARP, rffi.CCHARPP], rffi.INT)
 
         def execv_llimpl(path, args):
             l_args = rffi.ll_liststr2charpp(args)
@@ -270,12 +265,9 @@
 
     @registering_if(os, 'execve')
     def register_os_execve(self):
-        eci = self.gcc_profiling_bug_workaround(
-            'RPY_EXPORTED_FOR_TESTS int _noprof_execve(char *filename, char *argv[], char *envp[])',
-            'return execve(filename, argv, envp);')
         os_execve = self.llexternal(
-            '_noprof_execve', [rffi.CCHARP, rffi.CCHARPP, rffi.CCHARPP],
-            rffi.INT, compilation_info = eci)
+            'execve',
+            [rffi.CCHARP, rffi.CCHARPP, rffi.CCHARPP], rffi.INT)
 
         def execve_llimpl(path, args, env):
             # XXX Check path, args, env for \0 and raise TypeErrors as
@@ -1001,8 +993,6 @@
                                   [rffi.INT, rffi.VOIDP, rffi.SIZE_T],
                                   rffi.SIZE_T)
 
-        offset = offsetof(STR, 'chars') + itemoffsetof(STR.chars, 0)
-
         def os_read_llimpl(fd, count):
             if count < 0:
                 raise OSError(errno.EINVAL, None)
@@ -1727,10 +1717,7 @@
     @registering_if(os, 'fork')
     def register_os_fork(self):
         from rpython.rlib import debug, rthread
-        eci = self.gcc_profiling_bug_workaround('RPY_EXPORTED_FOR_TESTS pid_t _noprof_fork(void)',
-                                                'return fork();')
-        os_fork = self.llexternal('_noprof_fork', [], rffi.PID_T,
-                                  compilation_info = eci,
+        os_fork = self.llexternal('fork', [], rffi.PID_T,
                                   _nowrapper = True)
 
         def fork_llimpl():
@@ -1931,21 +1918,6 @@
         return extdef([int], str, "ll_os.ttyname",
                       llimpl=ttyname_llimpl)
 
-    # ____________________________________________________________
-    # XXX horrible workaround for a bug of profiling in gcc on
-    # OS X with functions containing a direct call to some system calls
-    # like fork(), execv(), execve()
-    def gcc_profiling_bug_workaround(self, decl, body):
-        body = ('/*--no-profiling-for-this-file!--*/\n'
-                '#include "src/precommondefs.h"\n'
-                '%s {\n'
-                '\t%s\n'
-                '}\n' % (decl, body,))
-        return ExternalCompilationInfo(
-            include_dirs=[cdir],
-            post_include_bits = [decl + ';'],
-            separate_module_sources = [body])
-
 # ____________________________________________________________
 # Support for os.environ
 
diff --git a/rpython/translator/c/genc.py b/rpython/translator/c/genc.py
--- a/rpython/translator/c/genc.py
+++ b/rpython/translator/c/genc.py
@@ -42,26 +42,7 @@
         self.compiler = compiler
 
     def first(self):
-        platform = self.compiler.platform
-        if platform.name.startswith('darwin'):
-            # XXX incredible hack for darwin
-            STR = '/*--no-profiling-for-this-file!--*/'
-            no_prof = []
-            prof = []
-            for cfile in self.compiler.cfiles:
-                if STR in cfile.read():
-                    no_prof.append(cfile)
-                else:
-                    prof.append(cfile)
-            p_eci = self.compiler.eci.merge(
-                ExternalCompilationInfo(compile_extra=['-fprofile-generate'],
-                                        link_extra=['-fprofile-generate']))
-            ofiles = platform._compile_o_files(prof, p_eci)
-            _, eci = self.compiler.eci.get_module_files()
-            ofiles += platform._compile_o_files(no_prof, eci)
-            return platform._finish_linking(ofiles, p_eci, None, True)
-        else:
-            return self.build('-fprofile-generate')
+        return self.build('-fprofile-generate')
 
     def probe(self, exe, args):
         # 'args' is a single string typically containing spaces
diff --git a/rpython/translator/platform/distutils_platform.py b/rpython/translator/platform/distutils_platform.py
--- a/rpython/translator/platform/distutils_platform.py
+++ b/rpython/translator/platform/distutils_platform.py
@@ -127,14 +127,6 @@
         for cfile in self.cfilenames:
             cfile = py.path.local(cfile)
             compile_extra = self.compile_extra[:]
-            # -frandom-seed is only to try to be as reproducable as possible
-            if 0 and self.fix_gcc_random_seed:
-                compile_extra.append('-frandom-seed=%s' % (cfile.basename,))
-                # XXX horrible workaround for a bug of profiling in gcc on
-                # OS X with functions containing a direct call to fork()
-                if '/*--no-profiling-for-this-file!--*/' in cfile.read():
-                    compile_extra = [arg for arg in compile_extra
-                                     if not arg.startswith('-fprofile-')]
 
             old = cfile.dirpath().chdir()
             try:

From noreply at buildbot.pypy.org  Thu Nov  6 18:51:06 2014
From: noreply at buildbot.pypy.org (rlamy)
Date: Thu,  6 Nov 2014 18:51:06 +0100 (CET)
Subject: [pypy-commit] pypy online-transforms: create annotator-level
 expression object V_Type
Message-ID: <20141106175106.B88961C0483@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy 
Branch: online-transforms
Changeset: r74356:4cd0db0438b3
Date: 2014-11-06 17:50 +0000
http://bitbucket.org/pypy/pypy/changeset/4cd0db0438b3/

Log:	create annotator-level expression object V_Type

diff --git a/rpython/annotator/expression.py b/rpython/annotator/expression.py
new file mode 100644
--- /dev/null
+++ b/rpython/annotator/expression.py
@@ -0,0 +1,14 @@
+from rpython.flowspace.model import Variable
+from rpython.flowspace.operation import op
+from rpython.annotator.model import SomeType
+
+class V_Type(Variable):
+    def __init__(self, v_obj):
+        Variable.__init__(self)
+        self.arg = v_obj
+        s = SomeType()
+        s.is_type_of = [v_obj]
+        self.annotation = s
+
+    def as_operation(self):
+        return op.type(self.arg)
diff --git a/rpython/annotator/unaryop.py b/rpython/annotator/unaryop.py
--- a/rpython/annotator/unaryop.py
+++ b/rpython/annotator/unaryop.py
@@ -17,16 +17,19 @@
 from rpython.annotator.binaryop import _clone ## XXX where to put this?
 from rpython.annotator.model import AnnotatorError
 from rpython.annotator.argument import simple_args, complex_args
+from rpython.annotator.expression import V_Type
 
 UNARY_OPERATIONS = set([oper.opname for oper in op.__dict__.values()
                         if oper.dispatch == 1])
 UNARY_OPERATIONS.remove('contains')
 
- at op.type.register(SomeObject)
-def type_SomeObject(annotator, arg):
-    r = SomeType()
-    r.is_type_of = [arg]
-    return r
+ at op.assign.register(SomeObject)
+def assign(annotator, v_obj):
+    return annotator.annotation(v_obj)
+
+ at op.type.register_transform(SomeObject)
+def type_SomeObject(annotator, v_arg):
+    return [op.assign(V_Type(v_arg))]
 
 @op.bool.register(SomeObject)
 def bool_SomeObject(annotator, obj):
diff --git a/rpython/flowspace/model.py b/rpython/flowspace/model.py
--- a/rpython/flowspace/model.py
+++ b/rpython/flowspace/model.py
@@ -578,7 +578,8 @@
                 for v in op.args:
                     assert isinstance(v, (Constant, Variable))
                     if isinstance(v, Variable):
-                        usevar(v)
+                        if type(v) is Variable:
+                            usevar(v)
                     else:
                         assert v.value is not last_exception
                         #assert v.value != last_exc_value
diff --git a/rpython/flowspace/operation.py b/rpython/flowspace/operation.py
--- a/rpython/flowspace/operation.py
+++ b/rpython/flowspace/operation.py
@@ -486,6 +486,7 @@
 add_operator('yield_', 1)
 add_operator('newslice', 3)
 add_operator('hint', None, dispatch=1)
+add_operator('assign', 1, dispatch=1)
 
 class Contains(SingleDispatchMixin, PureOperation):
     opname = 'contains'
diff --git a/rpython/translator/transform.py b/rpython/translator/transform.py
--- a/rpython/translator/transform.py
+++ b/rpython/translator/transform.py
@@ -134,6 +134,16 @@
                     s_dict = self.annotation(op.args[0])
                     s_dict.dictdef.generalize_key(self.binding(op.args[1]))
 
+def remove_assign(ann, block_subset):
+    for block in block_subset:
+        for i in range(len(block.operations)):
+            op = block.operations[i]
+            if op.opname == 'assign':
+                new_op = op.args[0].as_operation()
+                new_op.result = op.result
+                block.operations[i] = new_op
+
+
 
 def transform_dead_op_vars(self, block_subset):
     # we redo the same simplification from simplify.py,
@@ -249,6 +259,7 @@
     transform_extend_with_str_slice,
     transform_extend_with_char_count,
     transform_list_contains,
+    remove_assign,
     ]
 
 def transform_graph(ann, extra_passes=None, block_subset=None):

From noreply at buildbot.pypy.org  Thu Nov  6 21:12:37 2014
From: noreply at buildbot.pypy.org (pjenvey)
Date: Thu,  6 Nov 2014 21:12:37 +0100 (CET)
Subject: [pypy-commit] pypy py3k: merge default
Message-ID: <20141106201237.4A9051C07EE@cobra.cs.uni-duesseldorf.de>

Author: Philip Jenvey 
Branch: py3k
Changeset: r74357:03d54c00a4a8
Date: 2014-11-06 12:11 -0800
http://bitbucket.org/pypy/pypy/changeset/03d54c00a4a8/

Log:	merge default

diff --git a/README.rst b/README.rst
--- a/README.rst
+++ b/README.rst
@@ -37,4 +37,4 @@
 to use virtualenv with the resulting pypy-c as the interpreter; you can
 find more details about various installation schemes here:
 
-http://doc.pypy.org/en/latest/getting-started.html#installing-pypy
+    http://doc.pypy.org/en/latest/install.html
diff --git a/lib_pypy/grp.py b/lib_pypy/grp.py
--- a/lib_pypy/grp.py
+++ b/lib_pypy/grp.py
@@ -72,7 +72,7 @@
         raise TypeError("expected string")
     res = libc.getgrnam(os.fsencode(name))
     if not res:
-        raise KeyError(name)
+        raise KeyError("'getgrnam(): name not found: %s'" % name)
     return _group_from_gstruct(res)
 
 @builtinify
diff --git a/pypy/config/pypyoption.py b/pypy/config/pypyoption.py
--- a/pypy/config/pypyoption.py
+++ b/pypy/config/pypyoption.py
@@ -88,9 +88,10 @@
     # itself needs the interp-level struct module
     # because 'P' is missing from the app-level one
     "_rawffi": [("objspace.usemodules.struct", True)],
-    "cpyext": [("translation.secondaryentrypoints", "cpyext,main"),
-               ("translation.shared", sys.platform == "win32")],
+    "cpyext": [("translation.secondaryentrypoints", "cpyext,main")],
 }
+if sys.platform == "win32":
+    module_suggests["cpyext"].append(("translation.shared", True))
 
 module_import_dependencies = {
     # no _rawffi if importing rpython.rlib.clibffi raises ImportError
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -39,3 +39,7 @@
 .. branch: kill-multimethod
 
 Kill multimethod machinery, all multimethods were removed earlier.
+
+.. branch nditer-external_loop
+
+Implement `external_loop` arguement to numpy's nditer
diff --git a/pypy/interpreter/executioncontext.py b/pypy/interpreter/executioncontext.py
--- a/pypy/interpreter/executioncontext.py
+++ b/pypy/interpreter/executioncontext.py
@@ -32,6 +32,17 @@
         self.compiler = space.createcompiler()
         self.profilefunc = None
         self.w_profilefuncarg = None
+        self.thread_disappeared = False   # might be set to True after os.fork()
+
+    @staticmethod
+    def _mark_thread_disappeared(space):
+        # Called in the child process after os.fork() by interp_posix.py.
+        # Marks all ExecutionContexts except the current one
+        # with 'thread_disappeared = True'.
+        me = space.getexecutioncontext()
+        for ec in space.threadlocals.getallvalues().values():
+            if ec is not me:
+                ec.thread_disappeared = True
 
     def gettopframe(self):
         return self.topframeref()
diff --git a/pypy/module/micronumpy/concrete.py b/pypy/module/micronumpy/concrete.py
--- a/pypy/module/micronumpy/concrete.py
+++ b/pypy/module/micronumpy/concrete.py
@@ -449,7 +449,7 @@
                 strides.reverse()
                 backstrides.reverse()
                 new_shape.reverse()
-            return SliceArray(self.start, strides, backstrides, new_shape,
+            return self.__class__(self.start, strides, backstrides, new_shape,
                               self, orig_array)
         new_strides = calc_new_strides(new_shape, self.get_shape(),
                                        self.get_strides(),
@@ -460,10 +460,16 @@
         new_backstrides = [0] * len(new_shape)
         for nd in range(len(new_shape)):
             new_backstrides[nd] = (new_shape[nd] - 1) * new_strides[nd]
-        return SliceArray(self.start, new_strides, new_backstrides, new_shape,
+        return self.__class__(self.start, new_strides, new_backstrides, new_shape,
                           self, orig_array)
 
 
+class NonWritableSliceArray(SliceArray):
+    def descr_setitem(self, space, orig_array, w_index, w_value):
+        raise OperationError(space.w_ValueError, space.wrap(
+            "assignment destination is read-only"))
+
+
 class VoidBoxStorage(BaseConcreteArray):
     def __init__(self, size, dtype):
         self.storage = alloc_raw_storage(size)
diff --git a/pypy/module/micronumpy/iterators.py b/pypy/module/micronumpy/iterators.py
--- a/pypy/module/micronumpy/iterators.py
+++ b/pypy/module/micronumpy/iterators.py
@@ -8,8 +8,8 @@
 At which byte in x.data does the item x[3,4] begin?
 if x.strides==[1,5]:
     pData = x.pData + (x.start + 3*1 + 4*5)*sizeof(x.pData[0])
-    pData = x.pData + (x.start + 24) * sizeof(x.pData[0])
-so the offset of the element is 24 elements after the first
+    pData = x.pData + (x.start + 23) * sizeof(x.pData[0])
+so the offset of the element is 23 elements after the first
 
 What is the next element in x after coordinates [3,4]?
 if x.order =='C':
@@ -33,7 +33,7 @@
   which is x.strides[1] * (x.shape[1] - 1) + x.strides[0]
 so if we precalculate the overflow backstride as
 [x.strides[i] * (x.shape[i] - 1) for i in range(len(x.shape))]
-we can go faster.
+we can do only addition while iterating
 All the calculations happen in next()
 """
 from rpython.rlib import jit
@@ -41,6 +41,16 @@
 from pypy.module.micronumpy.base import W_NDimArray
 from pypy.module.micronumpy.flagsobj import _update_contiguous_flags
 
+class OpFlag(object):
+    def __init__(self):
+        self.rw = ''
+        self.broadcast = True
+        self.force_contig = False
+        self.force_align = False
+        self.native_byte_order = False
+        self.tmp_copy = ''
+        self.allocate = False
+
 
 class PureShapeIter(object):
     def __init__(self, shape, idx_w):
@@ -89,11 +99,13 @@
 class ArrayIter(object):
     _immutable_fields_ = ['contiguous', 'array', 'size', 'ndim_m1', 'shape_m1[*]',
                           'strides[*]', 'backstrides[*]', 'factors[*]',
-                          'track_index']
+                          'slice_shape', 'slice_stride', 'slice_backstride',
+                          'track_index', 'operand_type', 'slice_operand_type']
 
     track_index = True
 
-    def __init__(self, array, size, shape, strides, backstrides):
+    def __init__(self, array, size, shape, strides, backstrides, op_flags=OpFlag()):
+        from pypy.module.micronumpy import concrete
         assert len(shape) == len(strides) == len(backstrides)
         _update_contiguous_flags(array)
         self.contiguous = (array.flags & NPY.ARRAY_C_CONTIGUOUS and
@@ -105,6 +117,12 @@
         self.shape_m1 = [s - 1 for s in shape]
         self.strides = strides
         self.backstrides = backstrides
+        self.slice_shape = 1
+        self.slice_stride = -1
+        if strides:
+            self.slice_stride = strides[-1]
+        self.slice_backstride = 1
+        self.slice_operand_type = concrete.SliceArray
 
         ndim = len(shape)
         factors = [0] * ndim
@@ -114,6 +132,10 @@
             else:
                 factors[ndim-i-1] = factors[ndim-i] * shape[ndim-i]
         self.factors = factors
+        if op_flags.rw == 'r':
+            self.operand_type = concrete.ConcreteNonWritableArrayWithBase
+        else:
+            self.operand_type = concrete.ConcreteArrayWithBase
 
     @jit.unroll_safe
     def reset(self, state=None):
@@ -193,6 +215,12 @@
         assert state.iterator is self
         self.array.setitem(state.offset, elem)
 
+    def getoperand(self, st, base):
+        impl = self.operand_type
+        res = impl([], self.array.dtype, self.array.order, [], [],
+                   self.array.storage, base)
+        res.start = st.offset
+        return res
 
 def AxisIter(array, shape, axis, cumulative):
     strides = array.get_strides()
@@ -216,3 +244,42 @@
         size /= shape[axis]
     shape[axis] = backstrides[axis] = 0
     return ArrayIter(array, size, shape, array.strides, backstrides)
+
+class SliceIter(ArrayIter):
+    '''
+    used with external loops, getitem and setitem return a SliceArray
+    view into the original array
+    '''
+    _immutable_fields_ = ['base', 'slice_shape[*]', 'slice_stride[*]', 'slice_backstride[*]']
+
+    def __init__(self, array, size, shape, strides, backstrides, slice_shape,
+                 slice_stride, slice_backstride, op_flags, base):
+        from pypy.module.micronumpy import concrete
+        ArrayIter.__init__(self, array, size, shape, strides, backstrides, op_flags)
+        self.slice_shape = slice_shape
+        self.slice_stride = slice_stride
+        self.slice_backstride = slice_backstride
+        self.base = base
+        if op_flags.rw == 'r':
+            self.slice_operand_type = concrete.NonWritableSliceArray
+        else:
+            self.slice_operand_type = concrete.SliceArray
+
+    def getitem(self, state):
+        # XXX cannot be called - must return a boxed value
+        assert False
+
+    def getitem_bool(self, state):
+        # XXX cannot be called - must return a boxed value
+        assert False
+
+    def setitem(self, state, elem):
+        # XXX cannot be called - must return a boxed value
+        assert False
+
+    def getoperand(self, state, base):
+        assert state.iterator is self
+        impl = self.slice_operand_type
+        arr = impl(state.offset, [self.slice_stride], [self.slice_backstride],
+                   [self.slice_shape], self.array, self.base)
+        return arr
diff --git a/pypy/module/micronumpy/ndarray.py b/pypy/module/micronumpy/ndarray.py
--- a/pypy/module/micronumpy/ndarray.py
+++ b/pypy/module/micronumpy/ndarray.py
@@ -83,8 +83,12 @@
         raise OperationError(space.w_AttributeError, space.wrap(
             "Cannot delete array dtype"))
 
+    def ndims(self):
+        return len(self.get_shape())
+    ndims._always_inline_ = True
+
     def descr_get_ndim(self, space):
-        return space.wrap(len(self.get_shape()))
+        return space.wrap(self.ndims())
 
     def descr_get_itemsize(self, space):
         return space.wrap(self.get_dtype().elsize)
@@ -103,14 +107,14 @@
         return space.wrap(loop.tostring(space, self))
 
     def getitem_filter(self, space, arr):
-        if len(arr.get_shape()) > 1 and arr.get_shape() != self.get_shape():
+        if arr.ndims() > 1 and arr.get_shape() != self.get_shape():
             raise OperationError(space.w_ValueError, space.wrap(
                 "boolean index array should have 1 dimension"))
         if arr.get_size() > self.get_size():
             raise OperationError(space.w_ValueError, space.wrap(
                 "index out of range for array"))
         size = loop.count_all_true(arr)
-        if len(arr.get_shape()) == 1:
+        if arr.ndims() == 1:
             res_shape = [size] + self.get_shape()[1:]
         else:
             res_shape = [size]
@@ -119,7 +123,7 @@
         return loop.getitem_filter(w_res, self, arr)
 
     def setitem_filter(self, space, idx, val):
-        if len(idx.get_shape()) > 1 and idx.get_shape() != self.get_shape():
+        if idx.ndims() > 1 and idx.get_shape() != self.get_shape():
             raise OperationError(space.w_ValueError, space.wrap(
                 "boolean index array should have 1 dimension"))
         if idx.get_size() > self.get_size():
@@ -210,7 +214,7 @@
         if space.is_w(w_idx, space.w_Ellipsis):
             return self
         elif isinstance(w_idx, W_NDimArray) and w_idx.get_dtype().is_bool() \
-                and len(w_idx.get_shape()) > 0:
+                and w_idx.ndims() > 0:
             return self.getitem_filter(space, w_idx)
         try:
             return self.implementation.descr_getitem(space, self, w_idx)
@@ -228,7 +232,7 @@
             self.implementation.setslice(space, convert_to_array(space, w_value))
             return
         elif isinstance(w_idx, W_NDimArray) and w_idx.get_dtype().is_bool() \
-                and len(w_idx.get_shape()) > 0:
+                and w_idx.ndims() > 0:
             self.setitem_filter(space, w_idx, convert_to_array(space, w_value))
             return
         try:
@@ -289,7 +293,7 @@
             shape=shape, backward_broadcast=backward_broadcast)
 
     def is_scalar(self):
-        return len(self.get_shape()) == 0
+        return self.ndims() == 0
 
     def set_scalar_value(self, w_val):
         return self.implementation.setitem(self.implementation.start, w_val)
@@ -408,7 +412,7 @@
         """
         if axis1 == axis2:
             return self
-        n = len(self.get_shape())
+        n = self.ndims()
         if n <= 1:
             return self
         if axis1 < 0:
@@ -426,7 +430,7 @@
         return self.implementation.nonzero(space, index_type)
 
     def descr_tolist(self, space):
-        if len(self.get_shape()) == 0:
+        if self.ndims() == 0:
             return self.get_scalar_value().item(space)
         l_w = []
         for i in range(self.get_shape()[0]):
@@ -514,7 +518,7 @@
         if len(args_w) == 0:
             raise OperationError(space.w_ValueError, space.wrap(
                 "itemset must have at least one argument"))
-        if len(args_w) != len(self.get_shape()) + 1:
+        if len(args_w) != self.ndims() + 1:
             raise OperationError(space.w_ValueError, space.wrap(
                 "incorrect number of indices for array"))
         self.descr_setitem(space, space.newtuple(args_w[:-1]), args_w[-1])
@@ -638,14 +642,14 @@
 
     @unwrap_spec(offset=int, axis1=int, axis2=int)
     def descr_diagonal(self, space, offset=0, axis1=0, axis2=1):
-        if len(self.get_shape()) < 2:
+        if self.ndims() < 2:
             raise OperationError(space.w_ValueError, space.wrap(
                 "need at least 2 dimensions for diagonal"))
-        if (axis1 < 0 or axis2 < 0 or axis1 >= len(self.get_shape()) or
-                axis2 >= len(self.get_shape())):
+        if (axis1 < 0 or axis2 < 0 or axis1 >= self.ndims() or
+                axis2 >= self.ndims()):
             raise oefmt(space.w_ValueError,
                         "axis1(=%d) and axis2(=%d) must be withing range "
-                        "(ndim=%d)", axis1, axis2, len(self.get_shape()))
+                        "(ndim=%d)", axis1, axis2, self.ndims())
         if axis1 == axis2:
             raise OperationError(space.w_ValueError, space.wrap(
                 "axis1 and axis2 cannot be the same"))
@@ -724,7 +728,7 @@
             raise OperationError(space.w_NotImplementedError, space.wrap(
                 'sorter not supported in searchsort'))
         side = searchside_converter(space, w_side)
-        if len(self.get_shape()) != 1:
+        if self.ndims() != 1:
             raise oefmt(space.w_ValueError, "a must be a 1-d array")
         v = convert_to_array(space, w_v)
         ret = W_NDimArray.from_shape(
@@ -963,7 +967,7 @@
         if other.is_scalar():
             #Note: w_out is not modified, this is numpy compliant.
             return self.descr_mul(space, other)
-        elif len(self.get_shape()) < 2 and len(other.get_shape()) < 2:
+        elif self.ndims() < 2 and other.ndims() < 2:
             w_res = self.descr_mul(space, other)
             assert isinstance(w_res, W_NDimArray)
             return w_res.descr_sum(space, space.wrap(-1), out)
@@ -980,7 +984,7 @@
                 matches = False
             elif not out.implementation.order == "C":
                 matches = False
-            elif len(out.get_shape()) != len(out_shape):
+            elif out.ndims() != len(out_shape):
                 matches = False
             else:
                 for i in range(len(out_shape)):
diff --git a/pypy/module/micronumpy/nditer.py b/pypy/module/micronumpy/nditer.py
--- a/pypy/module/micronumpy/nditer.py
+++ b/pypy/module/micronumpy/nditer.py
@@ -5,7 +5,7 @@
 from pypy.module.micronumpy import ufuncs, support, concrete
 from pypy.module.micronumpy.base import W_NDimArray, convert_to_array
 from pypy.module.micronumpy.descriptor import decode_w_dtype
-from pypy.module.micronumpy.iterators import ArrayIter
+from pypy.module.micronumpy.iterators import ArrayIter, SliceIter, OpFlag
 from pypy.module.micronumpy.strides import (calculate_broadcast_strides,
                                             shape_agreement, shape_agreement_multiple)
 
@@ -35,17 +35,6 @@
     return ret
 
 
-class OpFlag(object):
-    def __init__(self):
-        self.rw = ''
-        self.broadcast = True
-        self.force_contig = False
-        self.force_align = False
-        self.native_byte_order = False
-        self.tmp_copy = ''
-        self.allocate = False
-
-
 def parse_op_flag(space, lst):
     op_flag = OpFlag()
     for w_item in lst:
@@ -71,17 +60,17 @@
         elif item == 'allocate':
             op_flag.allocate = True
         elif item == 'no_subtype':
-            raise OperationError(space.w_NotImplementedError, space.wrap(
-                '"no_subtype" op_flag not implemented yet'))
+            raise oefmt(space.w_NotImplementedError,
+                '"no_subtype" op_flag not implemented yet')
         elif item == 'arraymask':
-            raise OperationError(space.w_NotImplementedError, space.wrap(
-                '"arraymask" op_flag not implemented yet'))
+            raise oefmt(space.w_NotImplementedError,
+                '"arraymask" op_flag not implemented yet')
         elif item == 'writemask':
-            raise OperationError(space.w_NotImplementedError, space.wrap(
-                '"writemask" op_flag not implemented yet'))
+            raise oefmt(space.w_NotImplementedError,
+                '"writemask" op_flag not implemented yet')
         else:
-            raise OperationError(space.w_ValueError, space.wrap(
-                'op_flags must be a tuple or array of per-op flag-tuples'))
+            raise oefmt(space.w_ValueError,
+                'op_flags must be a tuple or array of per-op flag-tuples')
     if op_flag.rw == '':
         raise oefmt(space.w_ValueError,
                     "None of the iterator flags READWRITE, READONLY, or "
@@ -94,8 +83,8 @@
         return
     elif not space.isinstance_w(w_flags, space.w_tuple) and not \
             space.isinstance_w(w_flags, space.w_list):
-        raise OperationError(space.w_ValueError, space.wrap(
-            'Iter global flags must be a list or tuple of strings'))
+        raise oefmt(space.w_ValueError,
+            'Iter global flags must be a list or tuple of strings')
     lst = space.listview(w_flags)
     for w_item in lst:
         if not space.isinstance_w(w_item, space.w_str) and not \
@@ -106,12 +95,10 @@
                         typename)
         item = space.str_w(w_item)
         if item == 'external_loop':
-            raise OperationError(space.w_NotImplementedError, space.wrap(
-                'nditer external_loop not implemented yet'))
             nditer.external_loop = True
         elif item == 'buffered':
-            raise OperationError(space.w_NotImplementedError, space.wrap(
-                'nditer buffered not implemented yet'))
+            raise oefmt(space.w_NotImplementedError,
+                'nditer buffered not implemented yet')
             # For numpy compatability
             nditer.buffered = True
         elif item == 'c_index':
@@ -131,8 +118,8 @@
         elif item == 'refs_ok':
             nditer.refs_ok = True
         elif item == 'reduce_ok':
-            raise OperationError(space.w_NotImplementedError, space.wrap(
-                'nditer reduce_ok not implemented yet'))
+            raise oefmt(space.w_NotImplementedError,
+                'nditer reduce_ok not implemented yet')
             nditer.reduce_ok = True
         elif item == 'zerosize_ok':
             nditer.zerosize_ok = True
@@ -141,9 +128,9 @@
                         'Unexpected iterator global flag "%s"',
                         item)
     if nditer.tracked_index and nditer.external_loop:
-        raise OperationError(space.w_ValueError, space.wrap(
+        raise oefmt(space.w_ValueError,
             'Iterator flag EXTERNAL_LOOP cannot be used if an index or '
-            'multi-index is being tracked'))
+            'multi-index is being tracked')
 
 
 def is_backward(imp, order):
@@ -155,11 +142,11 @@
         raise NotImplementedError('not implemented yet')
 
 
-def get_iter(space, order, arr, shape, dtype):
+def get_iter(space, order, arr, shape, dtype, op_flags):
     imp = arr.implementation
     backward = is_backward(imp, order)
     if arr.is_scalar():
-        return ArrayIter(imp, 1, [], [], [])
+        return ArrayIter(imp, 1, [], [], [], op_flags=op_flags)
     if (imp.strides[0] < imp.strides[-1] and not backward) or \
        (imp.strides[0] > imp.strides[-1] and backward):
         # flip the strides. Is this always true for multidimension?
@@ -174,8 +161,103 @@
         backstrides = imp.backstrides
     r = calculate_broadcast_strides(strides, backstrides, imp.shape,
                                     shape, backward)
-    return ArrayIter(imp, imp.get_size(), shape, r[0], r[1])
+    return ArrayIter(imp, imp.get_size(), shape, r[0], r[1], op_flags=op_flags)
 
+def calculate_ndim(op_in, oa_ndim):
+    if oa_ndim >=0:
+        return oa_ndim
+    else:
+        ndim = 0
+        for op in op_in:
+            if op is None:
+                continue
+            assert isinstance(op, W_NDimArray)
+            ndim = max(ndim, op.ndims())
+    return ndim
+
+def coalesce_axes(it, space):
+    # Copy logic from npyiter_coalesce_axes, used in ufunc iterators
+    # and in nditer's with 'external_loop' flag
+    can_coalesce = True
+    if it.order == 'F':
+        fastest = 0
+    else:
+        fastest = -1
+    for idim in range(it.ndim - 1):
+        for op_it, _ in it.iters:
+            if op_it is None:
+                continue
+            assert isinstance(op_it, ArrayIter)
+            indx = len(op_it.strides)
+            if it.order == 'F':
+                indx = len(op_it.array.strides) - indx
+                assert indx >=0
+                astrides = op_it.array.strides[indx:]
+            else:
+                astrides = op_it.array.strides[:indx]
+            # does op_it iters over array "naturally"
+            if astrides != op_it.strides:
+                can_coalesce = False
+                break
+        if can_coalesce:
+            for i in range(len(it.iters)):
+                old_iter = it.iters[i][0]
+                shape = [s+1 for s in old_iter.shape_m1]
+                strides = old_iter.strides
+                backstrides = old_iter.backstrides
+                if it.order == 'F':
+                    new_shape = shape[1:]
+                    new_strides = strides[1:]
+                    new_backstrides = backstrides[1:]
+                    _stride = min(strides[0], old_iter.slice_stride)
+                else:
+                    new_shape = shape[:-1]
+                    new_strides = strides[:-1]
+                    new_backstrides = backstrides[:-1]
+                    _stride = old_iter.slice_stride
+                # We always want the "fastest" iterator in external loops
+                _shape = shape[fastest] * old_iter.slice_shape
+                _backstride = (_shape - 1) * _stride
+                new_iter = SliceIter(old_iter.array, old_iter.size / shape[fastest],
+                            new_shape, new_strides, new_backstrides,
+                            _shape, _stride, _backstride,
+                            it.op_flags[i], it)
+                it.iters[i] = (new_iter, new_iter.reset())
+            if len(it.shape) > 1:
+                if it.order == 'F':
+                    it.shape = it.shape[1:]
+                else:
+                    it.shape = it.shape[:-1]
+            else:
+                it.shape = [1]
+        else:
+            break
+    # Always coalesce at least one
+    for i in range(len(it.iters)):
+        old_iter = it.iters[i][0]
+        shape = [s+1 for s in old_iter.shape_m1]
+        strides = old_iter.strides
+        backstrides = old_iter.backstrides
+        new_shape = shape[:-1]
+        new_strides = strides[:-1]
+        new_backstrides = backstrides[:-1]
+        _shape = shape[-1] * old_iter.slice_shape
+        # use the operand's iterator's rightmost stride,
+        # even if it is not the fastest (for 'F' or swapped axis)
+        _stride = old_iter.slice_stride
+        _backstride = (_shape - 1) * _stride
+        new_iter = SliceIter(old_iter.array, old_iter.size / shape[-1],
+                    new_shape, new_strides, new_backstrides,
+                    _shape, _stride, _backstride,
+                    it.op_flags[i], it)
+        it.iters[i] = (new_iter, new_iter.reset())
+    if len(it.shape) > 1:
+        if it.order == 'F':
+            it.shape = it.shape[1:]
+        else:
+            it.shape = it.shape[:-1]
+    else:
+        it.shape = [1]
 
 class IndexIterator(object):
     def __init__(self, shape, backward=False):
@@ -205,6 +287,7 @@
 
 
 class W_NDIter(W_Root):
+    _immutable_fields_ = ['ndim', ]
     def __init__(self, space, w_seq, w_flags, w_op_flags, w_op_dtypes, w_casting,
                  w_op_axes, w_itershape, w_buffersize, order):
         self.order = order
@@ -236,28 +319,29 @@
         self.op_flags = parse_op_arg(space, 'op_flags', w_op_flags,
                                      len(self.seq), parse_op_flag)
         # handle w_op_axes
+        oa_ndim = -1
         if not space.is_none(w_op_axes):
-            self.set_op_axes(space, w_op_axes)
+            oa_ndim = self.set_op_axes(space, w_op_axes)
+        self.ndim = calculate_ndim(self.seq, oa_ndim)
 
         # handle w_op_dtypes part 1: creating self.dtypes list from input
         if not space.is_none(w_op_dtypes):
             w_seq_as_list = space.listview(w_op_dtypes)
             self.dtypes = [decode_w_dtype(space, w_elem) for w_elem in w_seq_as_list]
             if len(self.dtypes) != len(self.seq):
-                raise OperationError(space.w_ValueError, space.wrap(
-                    "op_dtypes must be a tuple/list matching the number of ops"))
+                raise oefmt(space.w_ValueError,
+                    "op_dtypes must be a tuple/list matching the number of ops")
         else:
             self.dtypes = []
 
         # handle None or writable operands, calculate my shape
-        self.iters = []
         outargs = [i for i in range(len(self.seq))
                    if self.seq[i] is None or self.op_flags[i].rw == 'w']
         if len(outargs) > 0:
             out_shape = shape_agreement_multiple(space, [self.seq[i] for i in outargs])
         else:
             out_shape = None
-        self.shape = iter_shape = shape_agreement_multiple(space, self.seq,
+        self.shape = shape_agreement_multiple(space, self.seq,
                                                            shape=out_shape)
         if len(outargs) > 0:
             # Make None operands writeonly and flagged for allocation
@@ -276,11 +360,11 @@
             for i in outargs:
                 if self.seq[i] is None:
                     # XXX can we postpone allocation to later?
-                    self.seq[i] = W_NDimArray.from_shape(space, iter_shape, out_dtype)
+                    self.seq[i] = W_NDimArray.from_shape(space, self.shape, out_dtype)
                 else:
                     if not self.op_flags[i].broadcast:
                         # Raises if ooutput cannot be broadcast
-                        shape_agreement(space, iter_shape, self.seq[i], False)
+                        shape_agreement(space, self.shape, self.seq[i], False)
 
         if self.tracked_index != "":
             if self.order == "K":
@@ -289,7 +373,7 @@
                 backward = False
             else:
                 backward = self.order != self.tracked_index
-            self.index_iter = IndexIterator(iter_shape, backward=backward)
+            self.index_iter = IndexIterator(self.shape, backward=backward)
 
         # handle w_op_dtypes part 2: copy where needed if possible
         if len(self.dtypes) > 0:
@@ -311,49 +395,49 @@
             self.dtypes = [s.get_dtype() for s in self.seq]
 
         # create an iterator for each operand
+        self.iters = []
         for i in range(len(self.seq)):
-            it = get_iter(space, self.order, self.seq[i], iter_shape, self.dtypes[i])
+            it = get_iter(space, self.order, self.seq[i], self.shape,
+                          self.dtypes[i], self.op_flags[i])
             it.contiguous = False
             self.iters.append((it, it.reset()))
 
+        if self.external_loop:
+            coalesce_axes(self, space)
+
     def set_op_axes(self, space, w_op_axes):
         if space.len_w(w_op_axes) != len(self.seq):
             raise oefmt(space.w_ValueError,
                         "op_axes must be a tuple/list matching the number of ops")
         op_axes = space.listview(w_op_axes)
-        l = -1
+        oa_ndim = -1
         for w_axis in op_axes:
             if not space.is_none(w_axis):
                 axis_len = space.len_w(w_axis)
-                if l == -1:
-                    l = axis_len
-                elif axis_len != l:
+                if oa_ndim == -1:
+                    oa_ndim = axis_len
+                elif axis_len != oa_ndim:
                     raise oefmt(space.w_ValueError,
                                 "Each entry of op_axes must have the same size")
                 self.op_axes.append([space.int_w(x) if not space.is_none(x) else -1
                                      for x in space.listview(w_axis)])
-        if l == -1:
+        if oa_ndim == -1:
             raise oefmt(space.w_ValueError,
                         "If op_axes is provided, at least one list of axes "
                         "must be contained within it")
-        raise Exception('xxx TODO')
+        raise oefmt(space.w_NotImplementedError, "op_axis not finished yet")
         # Check that values make sense:
         # - in bounds for each operand
         # ValueError: Iterator input op_axes[0][3] (==3) is not a valid axis of op[0], which has 2 dimensions
         # - no repeat axis
         # ValueError: The 'op_axes' provided to the iterator constructor for operand 1 contained duplicate value 0
+        return oa_ndim
 
     def descr_iter(self, space):
         return space.wrap(self)
 
-    def getitem(self, it, st, op_flags):
-        if op_flags.rw == 'r':
-            impl = concrete.ConcreteNonWritableArrayWithBase
-        else:
-            impl = concrete.ConcreteArrayWithBase
-        res = impl([], it.array.dtype, it.array.order, [], [],
-                   it.array.storage, self)
-        res.start = st.offset
+    def getitem(self, it, st):
+        res = it.getoperand(st, self)
         return W_NDimArray(res)
 
     def descr_getitem(self, space, w_idx):
@@ -363,7 +447,7 @@
         except IndexError:
             raise oefmt(space.w_IndexError,
                         "Iterator operand index %d is out of bounds", idx)
-        return self.getitem(it, st, self.op_flags[idx])
+        return self.getitem(it, st)
 
     def descr_setitem(self, space, w_idx, w_value):
         raise oefmt(space.w_NotImplementedError, "not implemented yet")
@@ -385,7 +469,7 @@
             else:
                 self.first_next = False
         for i, (it, st) in enumerate(self.iters):
-            res.append(self.getitem(it, st, self.op_flags[i]))
+            res.append(self.getitem(it, st))
             self.iters[i] = (it, it.next(st))
         if len(res) < 2:
             return res[0]
@@ -477,7 +561,7 @@
         raise oefmt(space.w_NotImplementedError, "not implemented yet")
 
     def descr_get_ndim(self, space):
-        raise oefmt(space.w_NotImplementedError, "not implemented yet")
+        return space.wrap(self.ndim)
 
     def descr_get_nop(self, space):
         raise oefmt(space.w_NotImplementedError, "not implemented yet")
diff --git a/pypy/module/micronumpy/test/test_nditer.py b/pypy/module/micronumpy/test/test_nditer.py
--- a/pypy/module/micronumpy/test/test_nditer.py
+++ b/pypy/module/micronumpy/test/test_nditer.py
@@ -63,9 +63,6 @@
         from numpy import arange, nditer, array
         a = arange(24).reshape(2, 3, 4)
         import sys
-        if '__pypy__' in sys.builtin_module_names:
-            raises(NotImplementedError, nditer, a, flags=['external_loop'])
-            skip('nditer external_loop not implmented')
         r = []
         n = 0
         for x in nditer(a, flags=['external_loop']):
@@ -79,7 +76,9 @@
             r.append(x)
             n += 1
         assert n == 12
-        assert (array(r) == [[ 0, 12], [ 4, 16], [ 8, 20], [ 1, 13], [ 5, 17], [ 9, 21], [ 2, 14], [ 6, 18], [10, 22], [ 3, 15], [ 7, 19], [11, 23]]).all()
+        assert (array(r) == [[ 0, 12], [ 4, 16], [ 8, 20], [ 1, 13], [ 5, 17], [ 9, 21],
+                             [ 2, 14], [ 6, 18], [10, 22], [ 3, 15], [ 7, 19], [11, 23],
+                            ]).all()
         e = raises(ValueError, 'r[0][0] = 0')
         assert str(e.value) == 'assignment destination is read-only'
         r = []
@@ -222,9 +221,6 @@
     def test_outarg(self):
         from numpy import nditer, zeros, arange
         import sys
-        if '__pypy__' in sys.builtin_module_names:
-            raises(NotImplementedError, nditer, [1, 2], flags=['external_loop'])
-            skip('nditer external_loop not implmented')
 
         def square1(a):
             it = nditer([a, None])
@@ -233,6 +229,9 @@
             return it.operands[1]
         assert (square1([1, 2, 3]) == [1, 4, 9]).all()
 
+        if '__pypy__' in sys.builtin_module_names:
+            raises(NotImplementedError, nditer, [1, 2], flags=['buffered'])
+            skip('nditer buffered not implmented')
         def square2(a, out=None):
             it = nditer([a, out], flags=['external_loop', 'buffered'],
                         op_flags=[['readonly'],
@@ -252,10 +251,11 @@
         from numpy import nditer, arange
         a = arange(3)
         import sys
+        b = arange(8).reshape(2,4)
         if '__pypy__' in sys.builtin_module_names:
-            raises(NotImplementedError, nditer, a, flags=['external_loop'])
-            skip('nditer external_loop not implmented')
-        b = arange(8).reshape(2,4)
+            raises(NotImplementedError, nditer, [a, b, None], flags=['external_loop'],
+                   op_axes=[[0, -1, -1], [-1, 0, 1], None])
+            skip('nditer op_axes not implemented yet')
         it = nditer([a, b, None], flags=['external_loop'],
                     op_axes=[[0, -1, -1], [-1, 0, 1], None])
         for x, y, z in it:
diff --git a/pypy/module/posix/interp_posix.py b/pypy/module/posix/interp_posix.py
--- a/pypy/module/posix/interp_posix.py
+++ b/pypy/module/posix/interp_posix.py
@@ -10,6 +10,7 @@
 
 from pypy.interpreter.gateway import unwrap_spec, WrappedDefault
 from pypy.interpreter.error import OperationError, wrap_oserror, wrap_oserror2
+from pypy.interpreter.executioncontext import ExecutionContext
 
 
 _WIN32 = sys.platform == 'win32'
@@ -729,6 +730,8 @@
     "NOT_RPYTHON"
     get_fork_hooks(where).append(hook)
 
+add_fork_hook('child', ExecutionContext._mark_thread_disappeared)
+
 @specialize.arg(0)
 def run_fork_hooks(where, space):
     for hook in get_fork_hooks(where):
diff --git a/pypy/module/test_lib_pypy/test_grp_extra.py b/pypy/module/test_lib_pypy/test_grp_extra.py
--- a/pypy/module/test_lib_pypy/test_grp_extra.py
+++ b/pypy/module/test_lib_pypy/test_grp_extra.py
@@ -10,7 +10,8 @@
                                     "No grp module on this platform")
 
     def test_basic(self):
-        raises(KeyError, self.grp.getgrnam, "dEkLofcG")
+        e = raises(KeyError, self.grp.getgrnam, "dEkLofcG")
+        assert e.value.args[0] == "'getgrnam(): name not found: dEkLofcG'"
         for name in ["root", "wheel"]:
             try:
                 g = self.grp.getgrnam(name)
diff --git a/pypy/module/test_lib_pypy/test_posix_extra.py b/pypy/module/test_lib_pypy/test_posix_extra.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/test_lib_pypy/test_posix_extra.py
@@ -0,0 +1,40 @@
+import py
+import sys, os, subprocess
+
+
+CODE = """
+import sys, os, thread, time
+
+fd1, fd2 = os.pipe()
+f1 = os.fdopen(fd1, 'r', 0)
+f2 = os.fdopen(fd2, 'w', 0)
+
+def f():
+    print "thread started"
+    x = f1.read(1)
+    assert x == "X"
+    print "thread exit"
+
+thread.start_new_thread(f, ())
+time.sleep(0.5)
+if os.fork() == 0:   # in the child
+    time.sleep(0.5)
+    x = f1.read(1)
+    assert x == "Y"
+    print "ok!"
+    sys.exit()
+
+f2.write("X")   # in the parent
+f2.write("Y")   # in the parent
+time.sleep(1.0)
+"""
+
+
+def test_thread_fork_file_lock():
+    if not hasattr(os, 'fork'):
+        py.test.skip("requires 'fork'")
+    output = subprocess.check_output([sys.executable, '-u', '-c', CODE])
+    assert output.splitlines() == [
+        'thread started',
+        'thread exit',
+        'ok!']
diff --git a/rpython/annotator/description.py b/rpython/annotator/description.py
--- a/rpython/annotator/description.py
+++ b/rpython/annotator/description.py
@@ -388,6 +388,9 @@
 
             return s_sigs
 
+def is_mixin(cls):
+    return cls.__dict__.get('_mixin_', False)
+
 NODEFAULT = object()
 
 class ClassDesc(Desc):
@@ -414,14 +417,13 @@
         self.specialize = specialize
         self._classdefs = {}
 
-        assert cls.__module__ != '__builtin__'
-        base = object
-        baselist = list(cls.__bases__)
-
-        if cls.__dict__.get('_mixin_', False):
+        if is_mixin(cls):
             raise AnnotatorError("cannot use directly the class %r because "
                                  "it is a _mixin_" % (cls,))
 
+        assert cls.__module__ != '__builtin__'
+        baselist = list(cls.__bases__)
+
         # special case: skip BaseException, and pretend
         # that all exceptions ultimately inherit from Exception instead
         # of BaseException (XXX hack)
@@ -432,10 +434,11 @@
 
         mixins_before = []
         mixins_after = []
+        base = object
         for b1 in baselist:
             if b1 is object:
                 continue
-            if b1.__dict__.get('_mixin_', False):
+            if is_mixin(b1):
                 if base is object:
                     mixins_before.append(b1)
                 else:
@@ -539,7 +542,7 @@
         add(check_not_in)
         #
         for base in reversed(mro):
-            assert base.__dict__.get("_mixin_", False), (
+            assert is_mixin(base), (
                 "Mixin class %r has non mixin base class %r" % (mixins, base))
             for name, value in base.__dict__.items():
                 if name in skip:
diff --git a/rpython/annotator/test/test_description.py b/rpython/annotator/test/test_description.py
--- a/rpython/annotator/test/test_description.py
+++ b/rpython/annotator/test/test_description.py
@@ -1,4 +1,4 @@
-from rpython.annotator.description import ClassDesc
+from rpython.annotator.description import ClassDesc, is_mixin
 
 class FakeBookkeeper:
     def __init__(self):
@@ -20,3 +20,13 @@
     dC = bk.getdesc(C)
     dD = bk.getdesc(D)
     assert ClassDesc.getcommonbase([dC, dD]) is dA
+
+def test_is_mixin():
+    class Mixin1(object):
+        _mixin_ = True
+
+    class A(Mixin1):
+        pass
+
+    assert is_mixin(Mixin1)
+    assert not is_mixin(A)
diff --git a/rpython/jit/metainterp/history.py b/rpython/jit/metainterp/history.py
--- a/rpython/jit/metainterp/history.py
+++ b/rpython/jit/metainterp/history.py
@@ -279,7 +279,8 @@
             # careful in this comparison: if self.value and other.value
             # are both NaN, stored as regular floats (i.e. on 64-bit),
             # then just using "==" would say False: two NaNs are always
-            # different from each other.
+            # different from each other.  Conversely, "0.0 == -0.0" but
+            # they are not the same constant.
             return (longlong.extract_bits(self.value) ==
                     longlong.extract_bits(other.value))
         return False
diff --git a/rpython/jit/metainterp/pyjitpl.py b/rpython/jit/metainterp/pyjitpl.py
--- a/rpython/jit/metainterp/pyjitpl.py
+++ b/rpython/jit/metainterp/pyjitpl.py
@@ -33,6 +33,14 @@
 
 # ____________________________________________________________
 
+FASTPATHS_SAME_BOXES = {
+    "ne": "history.CONST_FALSE",
+    "eq": "history.CONST_TRUE",
+    "lt": "history.CONST_FALSE",
+    "le": "history.CONST_TRUE",
+    "gt": "history.CONST_FALSE",
+    "ge": "history.CONST_TRUE",
+}
 
 class MIFrame(object):
     debug = False
@@ -188,8 +196,6 @@
     # ------------------------------
 
     for _opimpl in ['int_add', 'int_sub', 'int_mul', 'int_floordiv', 'int_mod',
-                    'int_lt', 'int_le', 'int_eq',
-                    'int_ne', 'int_gt', 'int_ge',
                     'int_and', 'int_or', 'int_xor',
                     'int_rshift', 'int_lshift', 'uint_rshift',
                     'uint_lt', 'uint_le', 'uint_gt', 'uint_ge',
@@ -197,7 +203,6 @@
                     'float_add', 'float_sub', 'float_mul', 'float_truediv',
                     'float_lt', 'float_le', 'float_eq',
                     'float_ne', 'float_gt', 'float_ge',
-                    'ptr_eq', 'ptr_ne', 'instance_ptr_eq', 'instance_ptr_ne',
                     ]:
         exec py.code.Source('''
             @arguments("box", "box")
@@ -205,6 +210,18 @@
                 return self.execute(rop.%s, b1, b2)
         ''' % (_opimpl, _opimpl.upper())).compile()
 
+    for _opimpl in ['int_eq', 'int_ne', 'int_lt', 'int_le', 'int_gt', 'int_ge',
+                    'ptr_eq', 'ptr_ne',
+                    'instance_ptr_eq', 'instance_ptr_ne']:
+        exec py.code.Source('''
+            @arguments("box", "box")
+            def opimpl_%s(self, b1, b2):
+                if b1 is b2: # crude fast check
+                    return %s
+                return self.execute(rop.%s, b1, b2)
+        ''' % (_opimpl, FASTPATHS_SAME_BOXES[_opimpl.split("_")[-1]], _opimpl.upper())
+        ).compile()
+
     for _opimpl in ['int_add_ovf', 'int_sub_ovf', 'int_mul_ovf']:
         exec py.code.Source('''
             @arguments("box", "box")
@@ -340,10 +357,13 @@
         exec py.code.Source('''
             @arguments("box", "box", "label")
             def opimpl_goto_if_not_%s(self, b1, b2, target):
-                condbox = self.execute(rop.%s, b1, b2)
+                if b1 is b2:
+                    condbox = %s
+                else:
+                    condbox = self.execute(rop.%s, b1, b2)
                 self.opimpl_goto_if_not(condbox, target)
-        ''' % (_opimpl, _opimpl.upper())).compile()
-
+        ''' % (_opimpl, FASTPATHS_SAME_BOXES[_opimpl.split("_")[-1]], _opimpl.upper())
+        ).compile()
 
     def _establish_nullity(self, box, orgpc):
         value = box.nonnull()
diff --git a/rpython/jit/metainterp/test/test_ajit.py b/rpython/jit/metainterp/test/test_ajit.py
--- a/rpython/jit/metainterp/test/test_ajit.py
+++ b/rpython/jit/metainterp/test/test_ajit.py
@@ -4119,3 +4119,64 @@
         assert res == 42
         res = self.interp_operations(f, [-42])
         assert res == 0
+
+    def test_cmp_fastpaths(self):
+        class Z: pass
+        def make_int(cmp):
+            def f(x):
+                if cmp == 'eq':
+                    return x == x and x == x
+                if cmp == 'ne':
+                    return x != x or x != x
+                if cmp == 'lt':
+                    return x < x or x != x
+                if cmp == 'le':
+                    return x <= x and x <= x
+                if cmp == 'gt':
+                    return x > x or x > x
+                if cmp == 'ge':
+                    return x >= x and x >= x
+                assert 0
+            return f
+
+        def make_str(cmp):
+            def f(x):
+                x = str(x)
+                if cmp == 'eq':
+                    return x is x or x is x
+                if cmp == 'ne':
+                    return x is not x and x is not x
+                assert 0
+            return f
+
+        def make_object(cmp):
+            def f(x):
+                y = Z()
+                y.x = x
+                x = y
+                if cmp == 'eq':
+                    return x is x
+                if cmp == 'ne':
+                    return x is not x
+                assert 0
+            return f
+
+        for cmp in 'eq ne lt le gt ge'.split():
+            f = make_int(cmp)
+            res = self.interp_operations(f, [42])
+            assert res == f(42)
+            opname = "int_%s" % cmp
+            self.check_operations_history(**{opname: 0})
+
+        for cmp in 'eq ne'.split():
+            f = make_str(cmp)
+            res = self.interp_operations(f, [42])
+            assert res == f(42)
+            opname = "ptr_%s" % cmp
+            self.check_operations_history(**{opname: 0})
+
+            f = make_object(cmp)
+            res = self.interp_operations(f, [42])
+            assert res == f(42)
+            opname = "instance_ptr_%s" % cmp
+            self.check_operations_history(**{opname: 0})
diff --git a/rpython/rlib/objectmodel.py b/rpython/rlib/objectmodel.py
--- a/rpython/rlib/objectmodel.py
+++ b/rpython/rlib/objectmodel.py
@@ -201,6 +201,11 @@
         return result
     return decorator
 
+def always_inline(func):
+    """ mark the function as to-be-inlined by the RPython optimizations (not
+    the JIT!), no matter its size."""
+    func._always_inline_ = True
+    return func
 
 
 # ____________________________________________________________
diff --git a/rpython/rlib/rsre/rsre_core.py b/rpython/rlib/rsre/rsre_core.py
--- a/rpython/rlib/rsre/rsre_core.py
+++ b/rpython/rlib/rsre/rsre_core.py
@@ -137,22 +137,25 @@
     def flatten_marks(self):
         # for testing
         if self.match_marks_flat is None:
-            self.match_marks_flat = [self.match_start, self.match_end]
-            mark = self.match_marks
-            if mark is not None:
-                self.match_lastindex = mark.gid
-            else:
-                self.match_lastindex = -1
-            while mark is not None:
-                index = mark.gid + 2
-                while index >= len(self.match_marks_flat):
-                    self.match_marks_flat.append(-1)
-                if self.match_marks_flat[index] == -1:
-                    self.match_marks_flat[index] = mark.position
-                mark = mark.prev
-            self.match_marks = None    # clear
+            self._compute_flattened_marks()
         return self.match_marks_flat
 
+    def _compute_flattened_marks(self):
+        self.match_marks_flat = [self.match_start, self.match_end]
+        mark = self.match_marks
+        if mark is not None:
+            self.match_lastindex = mark.gid
+        else:
+            self.match_lastindex = -1
+        while mark is not None:
+            index = mark.gid + 2
+            while index >= len(self.match_marks_flat):
+                self.match_marks_flat.append(-1)
+            if self.match_marks_flat[index] == -1:
+                self.match_marks_flat[index] = mark.position
+            mark = mark.prev
+        self.match_marks = None    # clear
+
     def span(self, groupnum=0):
         # compatibility
         fmarks = self.flatten_marks()
diff --git a/rpython/rlib/test/test_objectmodel.py b/rpython/rlib/test/test_objectmodel.py
--- a/rpython/rlib/test/test_objectmodel.py
+++ b/rpython/rlib/test/test_objectmodel.py
@@ -438,6 +438,11 @@
     assert exc.value.message == "f argument 'b' must be of type "
     py.test.raises(TypeError, "f('hello', 'world', 3)")
 
+def test_always_inline():
+    @always_inline
+    def f(a, b, c):
+        return a, b, c
+    assert f._always_inline_ == True
 
 def test_enforceargs_defaults():
     @enforceargs(int, int)
diff --git a/rpython/rtyper/lltypesystem/rffi.py b/rpython/rtyper/lltypesystem/rffi.py
--- a/rpython/rtyper/lltypesystem/rffi.py
+++ b/rpython/rtyper/lltypesystem/rffi.py
@@ -825,7 +825,7 @@
         allows for the process to be performed without an extra copy.
         Make sure to call keep_buffer_alive_until_here on the returned values.
         """
-        new_buf = lltype.malloc(STRTYPE, count)
+        new_buf = mallocfn(count)
         pinned = 0
         if rgc.can_move(new_buf):
             if rgc.pin(new_buf):
@@ -857,7 +857,7 @@
             if llop.shrink_array(lltype.Bool, gc_buf, needed_size):
                 pass     # now 'gc_buf' is smaller
             else:
-                gc_buf = lltype.malloc(STRTYPE, needed_size)
+                gc_buf = mallocfn(needed_size)
                 case_num = 2
         if case_num == 2:
             copy_raw_to_string(raw_buf, gc_buf, 0, needed_size)
diff --git a/rpython/rtyper/module/ll_os.py b/rpython/rtyper/module/ll_os.py
--- a/rpython/rtyper/module/ll_os.py
+++ b/rpython/rtyper/module/ll_os.py
@@ -22,8 +22,6 @@
 from rpython.rtyper.tool import rffi_platform as platform
 from rpython.rlib import rposix
 from rpython.translator.tool.cbuild import ExternalCompilationInfo
-from rpython.rtyper.lltypesystem.llmemory import itemoffsetof, offsetof
-from rpython.rtyper.lltypesystem.rstr import STR
 from rpython.rlib.objectmodel import specialize
 from rpython.translator import cdir
 
@@ -251,12 +249,9 @@
 
     @registering_if(os, 'execv')
     def register_os_execv(self):
-        eci = self.gcc_profiling_bug_workaround(
-            'RPY_EXPORTED_FOR_TESTS int _noprof_execv(char *path, char *argv[])',
-            'return execv(path, argv);')
-        os_execv = self.llexternal('_noprof_execv',
-                                   [rffi.CCHARP, rffi.CCHARPP],
-                                   rffi.INT, compilation_info = eci)
+        os_execv = self.llexternal(
+            'execv',
+            [rffi.CCHARP, rffi.CCHARPP], rffi.INT)
 
         def execv_llimpl(path, args):
             l_args = rffi.ll_liststr2charpp(args)
@@ -270,12 +265,9 @@
 
     @registering_if(os, 'execve')
     def register_os_execve(self):
-        eci = self.gcc_profiling_bug_workaround(
-            'RPY_EXPORTED_FOR_TESTS int _noprof_execve(char *filename, char *argv[], char *envp[])',
-            'return execve(filename, argv, envp);')
         os_execve = self.llexternal(
-            '_noprof_execve', [rffi.CCHARP, rffi.CCHARPP, rffi.CCHARPP],
-            rffi.INT, compilation_info = eci)
+            'execve',
+            [rffi.CCHARP, rffi.CCHARPP, rffi.CCHARPP], rffi.INT)
 
         def execve_llimpl(path, args, env):
             # XXX Check path, args, env for \0 and raise TypeErrors as
@@ -1001,8 +993,6 @@
                                   [rffi.INT, rffi.VOIDP, rffi.SIZE_T],
                                   rffi.SIZE_T)
 
-        offset = offsetof(STR, 'chars') + itemoffsetof(STR.chars, 0)
-
         def os_read_llimpl(fd, count):
             if count < 0:
                 raise OSError(errno.EINVAL, None)
@@ -1727,10 +1717,7 @@
     @registering_if(os, 'fork')
     def register_os_fork(self):
         from rpython.rlib import debug, rthread
-        eci = self.gcc_profiling_bug_workaround('RPY_EXPORTED_FOR_TESTS pid_t _noprof_fork(void)',
-                                                'return fork();')
-        os_fork = self.llexternal('_noprof_fork', [], rffi.PID_T,
-                                  compilation_info = eci,
+        os_fork = self.llexternal('fork', [], rffi.PID_T,
                                   _nowrapper = True)
 
         def fork_llimpl():
@@ -1931,21 +1918,6 @@
         return extdef([int], str, "ll_os.ttyname",
                       llimpl=ttyname_llimpl)
 
-    # ____________________________________________________________
-    # XXX horrible workaround for a bug of profiling in gcc on
-    # OS X with functions containing a direct call to some system calls
-    # like fork(), execv(), execve()
-    def gcc_profiling_bug_workaround(self, decl, body):
-        body = ('/*--no-profiling-for-this-file!--*/\n'
-                '#include "src/precommondefs.h"\n'
-                '%s {\n'
-                '\t%s\n'
-                '}\n' % (decl, body,))
-        return ExternalCompilationInfo(
-            include_dirs=[cdir],
-            post_include_bits = [decl + ';'],
-            separate_module_sources = [body])
-
 # ____________________________________________________________
 # Support for os.environ
 
diff --git a/rpython/translator/c/genc.py b/rpython/translator/c/genc.py
--- a/rpython/translator/c/genc.py
+++ b/rpython/translator/c/genc.py
@@ -42,26 +42,7 @@
         self.compiler = compiler
 
     def first(self):
-        platform = self.compiler.platform
-        if platform.name.startswith('darwin'):
-            # XXX incredible hack for darwin
-            STR = '/*--no-profiling-for-this-file!--*/'
-            no_prof = []
-            prof = []
-            for cfile in self.compiler.cfiles:
-                if STR in cfile.read():
-                    no_prof.append(cfile)
-                else:
-                    prof.append(cfile)
-            p_eci = self.compiler.eci.merge(
-                ExternalCompilationInfo(compile_extra=['-fprofile-generate'],
-                                        link_extra=['-fprofile-generate']))
-            ofiles = platform._compile_o_files(prof, p_eci)
-            _, eci = self.compiler.eci.get_module_files()
-            ofiles += platform._compile_o_files(no_prof, eci)
-            return platform._finish_linking(ofiles, p_eci, None, True)
-        else:
-            return self.build('-fprofile-generate')
+        return self.build('-fprofile-generate')
 
     def probe(self, exe, args):
         # 'args' is a single string typically containing spaces
diff --git a/rpython/translator/platform/distutils_platform.py b/rpython/translator/platform/distutils_platform.py
--- a/rpython/translator/platform/distutils_platform.py
+++ b/rpython/translator/platform/distutils_platform.py
@@ -127,14 +127,6 @@
         for cfile in self.cfilenames:
             cfile = py.path.local(cfile)
             compile_extra = self.compile_extra[:]
-            # -frandom-seed is only to try to be as reproducable as possible
-            if 0 and self.fix_gcc_random_seed:
-                compile_extra.append('-frandom-seed=%s' % (cfile.basename,))
-                # XXX horrible workaround for a bug of profiling in gcc on
-                # OS X with functions containing a direct call to fork()
-                if '/*--no-profiling-for-this-file!--*/' in cfile.read():
-                    compile_extra = [arg for arg in compile_extra
-                                     if not arg.startswith('-fprofile-')]
 
             old = cfile.dirpath().chdir()
             try:

From noreply at buildbot.pypy.org  Thu Nov  6 22:16:51 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Thu,  6 Nov 2014 22:16:51 +0100 (CET)
Subject: [pypy-commit] pypy default: fix for OS/X translation (shown by
	hippy)
Message-ID: <20141106211651.6B57F1C07EE@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r74358:94d0614584c0
Date: 2014-11-06 22:16 +0100
http://bitbucket.org/pypy/pypy/changeset/94d0614584c0/

Log:	fix for OS/X translation (shown by hippy)

diff --git a/rpython/rlib/_rsocket_rffi.py b/rpython/rlib/_rsocket_rffi.py
--- a/rpython/rlib/_rsocket_rffi.py
+++ b/rpython/rlib/_rsocket_rffi.py
@@ -499,7 +499,7 @@
 getnameinfo = external('getnameinfo', [sockaddr_ptr, socklen_t, CCHARP,
                        size_t, CCHARP, size_t, rffi.INT], rffi.INT)
 
-if sys.platform.startswith("openbsd"):
+if sys.platform.startswith("openbsd") or sys.platform.startswith("darwin"):
     htonl = external('htonl', [rffi.UINT], rffi.UINT, releasegil=False, macro=True)
     htons = external('htons', [rffi.USHORT], rffi.USHORT, releasegil=False, macro=True)
     ntohl = external('ntohl', [rffi.UINT], rffi.UINT, releasegil=False, macro=True)

From noreply at buildbot.pypy.org  Fri Nov  7 14:52:13 2014
From: noreply at buildbot.pypy.org (amauryfa)
Date: Fri,  7 Nov 2014 14:52:13 +0100 (CET)
Subject: [pypy-commit] pypy more-rposix: Remove dead code
Message-ID: <20141107135213.6B76A1C030E@cobra.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc 
Branch: more-rposix
Changeset: r74359:bff0cfc2ba47
Date: 2014-11-05 20:06 +0100
http://bitbucket.org/pypy/pypy/changeset/bff0cfc2ba47/

Log:	Remove dead code

diff --git a/rpython/rtyper/module/ll_os.py b/rpython/rtyper/module/ll_os.py
--- a/rpython/rtyper/module/ll_os.py
+++ b/rpython/rtyper/module/ll_os.py
@@ -247,50 +247,6 @@
         return extdef([int], int, llimpl=c_func_llimpl,
                       export_name='ll_os.ll_os_' + name)
 
-    @registering_if(posix, 'spawnv')
-    def register_os_spawnv(self):
-        os_spawnv = self.llexternal('spawnv',
-                                    [rffi.INT, rffi.CCHARP, rffi.CCHARPP],
-                                    rffi.INT)
-
-        def spawnv_llimpl(mode, path, args):
-            mode = rffi.cast(rffi.INT, mode)
-            l_args = rffi.ll_liststr2charpp(args)
-            childpid = os_spawnv(mode, path, l_args)
-            rffi.free_charpp(l_args)
-            if childpid == -1:
-                raise OSError(rposix.get_errno(), "os_spawnv failed")
-            return rffi.cast(lltype.Signed, childpid)
-
-        return extdef([int, str0, [str0]], int, llimpl=spawnv_llimpl,
-                      export_name="ll_os.ll_os_spawnv")
-
-    @registering_if(os, 'spawnve')
-    def register_os_spawnve(self):
-        os_spawnve = self.llexternal('spawnve',
-                                     [rffi.INT, rffi.CCHARP, rffi.CCHARPP,
-                                      rffi.CCHARPP],
-                                     rffi.INT)
-
-        def spawnve_llimpl(mode, path, args, env):
-            envstrs = []
-            for item in env.iteritems():
-                envstrs.append("%s=%s" % item)
-
-            mode = rffi.cast(rffi.INT, mode)
-            l_args = rffi.ll_liststr2charpp(args)
-            l_env = rffi.ll_liststr2charpp(envstrs)
-            childpid = os_spawnve(mode, path, l_args, l_env)
-            rffi.free_charpp(l_env)
-            rffi.free_charpp(l_args)
-            if childpid == -1:
-                raise OSError(rposix.get_errno(), "os_spawnve failed")
-            return rffi.cast(lltype.Signed, childpid)
-
-        return extdef([int, str0, [str0], {str0: str0}], int,
-                      llimpl=spawnve_llimpl,
-                      export_name="ll_os.ll_os_spawnve")
-
     @registering_if(os, "getlogin", condition=not _WIN32)
     def register_os_getlogin(self):
         os_getlogin = self.llexternal('getlogin', [], rffi.CCHARP, releasegil=False)

From noreply at buildbot.pypy.org  Fri Nov  7 14:52:14 2014
From: noreply at buildbot.pypy.org (amauryfa)
Date: Fri,  7 Nov 2014 14:52:14 +0100 (CET)
Subject: [pypy-commit] pypy more-rposix: Remove an old workaround that was
 useful for the cli backend
Message-ID: <20141107135214.C53B51C030E@cobra.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc 
Branch: more-rposix
Changeset: r74360:8be9ebc6bbbd
Date: 2014-11-06 21:15 +0100
http://bitbucket.org/pypy/pypy/changeset/8be9ebc6bbbd/

Log:	Remove an old workaround that was useful for the cli backend

diff --git a/rpython/rlib/rwin32.py b/rpython/rlib/rwin32.py
--- a/rpython/rlib/rwin32.py
+++ b/rpython/rlib/rwin32.py
@@ -243,7 +243,7 @@
                 buflen -= 1
 
             if buflen <= 0:
-                result = fake_FormatError(code)
+                result = 'Windows Error %d' % (code,)
             else:
                 result = rffi.charpsize2str(s_buf, buflen)
         finally:
@@ -252,9 +252,6 @@
 
         return result
 
-    def fake_FormatError(code):
-        return 'Windows Error %d' % (code,)
-
     def lastWindowsError(context="Windows Error"):
         code = GetLastError()
         return WindowsError(code, context)
diff --git a/rpython/rtyper/module/ll_os.py b/rpython/rtyper/module/ll_os.py
--- a/rpython/rtyper/module/ll_os.py
+++ b/rpython/rtyper/module/ll_os.py
@@ -1799,19 +1799,3 @@
     def getcontroller(self):
         from rpython.rtyper.module.ll_os_environ import OsEnvironController
         return OsEnvironController()
-
-# ____________________________________________________________
-# Support for the WindowsError exception
-
-if sys.platform == 'win32':
-    from rpython.rlib import rwin32
-
-    class RegisterFormatError(BaseLazyRegistering):
-        def __init__(self):
-            pass
-
-        @registering(rwin32.FormatError)
-        def register_rwin32_FormatError(self):
-            return extdef([lltype.Signed], str,
-                          "rwin32_FormatError",
-                          llimpl=rwin32.llimpl_FormatError)

From noreply at buildbot.pypy.org  Fri Nov  7 14:52:15 2014
From: noreply at buildbot.pypy.org (amauryfa)
Date: Fri,  7 Nov 2014 14:52:15 +0100 (CET)
Subject: [pypy-commit] pypy more-rposix: Fix test
Message-ID: <20141107135215.F0E461C030E@cobra.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc 
Branch: more-rposix
Changeset: r74361:755c80edc35b
Date: 2014-11-06 21:52 +0100
http://bitbucket.org/pypy/pypy/changeset/755c80edc35b/

Log:	Fix test

diff --git a/rpython/rtyper/module/test/test_ll_os.py b/rpython/rtyper/module/test/test_ll_os.py
--- a/rpython/rtyper/module/test/test_ll_os.py
+++ b/rpython/rtyper/module/test/test_ll_os.py
@@ -5,6 +5,7 @@
 from rpython.rtyper.module import ll_os
 #has side effect of registering functions
 from rpython.tool.pytest.expecttest import ExpectTest
+from rpython.rlib import rposix
 
 from rpython.rtyper import extregistry
 import errno
@@ -149,8 +150,6 @@
     if os.name != 'posix':
         py.test.skip('posix specific function')
 
-    ll_execve = getllimpl(os.execve)
-
     def run_execve(program, args=None, env=None, do_path_lookup=False):
         if args is None:
             args = [program]
@@ -158,7 +157,7 @@
             args = [program] + args
         if env is None:
             env = {}
-        # we cannot directly call ll_execve() because it replaces the
+        # we cannot directly call execve() because it replaces the
         # current process.
         fd_read, fd_write = os.pipe()
         childpid = os.fork()
@@ -170,7 +169,7 @@
             if do_path_lookup:
                 os.execvp(program, args)
             else:
-                ll_execve(program, args, env)
+                rposix.execve(program, args, env)
             assert 0, "should not arrive here"
         else:
             # in the parent
@@ -205,12 +204,12 @@
 
     # If the target does not exist, an OSError should result
     info = py.test.raises(
-        OSError, ll_execve, "this/file/is/non/existent", [], {})
+        OSError, rposix.execve, "this/file/is/non/existent", [], {})
     assert info.value.errno == errno.ENOENT
 
     # If the target is not executable, an OSError should result
     info = py.test.raises(
-        OSError, ll_execve, "/etc/passwd", [], {})
+        OSError, rposix.execve, "/etc/passwd", [], {})
     assert info.value.errno == errno.EACCES
 
 def test_os_write():

From noreply at buildbot.pypy.org  Fri Nov  7 14:52:17 2014
From: noreply at buildbot.pypy.org (amauryfa)
Date: Fri,  7 Nov 2014 14:52:17 +0100 (CET)
Subject: [pypy-commit] pypy more-rposix: Move os.getlogin()
Message-ID: <20141107135217.3EC311C030E@cobra.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc 
Branch: more-rposix
Changeset: r74362:b25e33df35a3
Date: 2014-11-06 21:53 +0100
http://bitbucket.org/pypy/pypy/changeset/b25e33df35a3/

Log:	Move os.getlogin()

diff --git a/rpython/rlib/rposix.py b/rpython/rlib/rposix.py
--- a/rpython/rlib/rposix.py
+++ b/rpython/rlib/rposix.py
@@ -152,6 +152,8 @@
 c_execv = external('execv', [rffi.CCHARP, rffi.CCHARPP], rffi.INT)
 c_execve = external('execve',
                     [rffi.CCHARP, rffi.CCHARPP, rffi.CCHARPP], rffi.INT)
+c_getlogin = external('getlogin', [], rffi.CCHARP, releasegil=False)
+
 # Win32 specific functions
 c_spawnv = external('spawnv',
                     [rffi.INT, rffi.CCHARP, rffi.CCHARPP], rffi.INT)
@@ -386,6 +388,13 @@
     rffi.free_charpp(l_env)
     rffi.free_charpp(l_args)
     if childpid == -1:
-        raise OSError(rposix.get_errno(), "os_spawnve failed")
+        raise OSError(get_errno(), "os_spawnve failed")
     return intmask(childpid)
 
+ at register_replacement_for(getattr(os, 'getlogin', None),
+                          sandboxed_name='ll_os.ll_os_getlogin')
+def getlogin():
+    result = c_getlogin()
+    if not result:
+        raise OSError(get_errno(), "getlogin failed")
+    return rffi.charp2str(result)
diff --git a/rpython/rtyper/module/ll_os.py b/rpython/rtyper/module/ll_os.py
--- a/rpython/rtyper/module/ll_os.py
+++ b/rpython/rtyper/module/ll_os.py
@@ -247,20 +247,6 @@
         return extdef([int], int, llimpl=c_func_llimpl,
                       export_name='ll_os.ll_os_' + name)
 
-    @registering_if(os, "getlogin", condition=not _WIN32)
-    def register_os_getlogin(self):
-        os_getlogin = self.llexternal('getlogin', [], rffi.CCHARP, releasegil=False)
-
-        def getlogin_llimpl():
-            result = os_getlogin()
-            if not result:
-                raise OSError(rposix.get_errno(), "getlogin failed")
-
-            return rffi.charp2str(result)
-
-        return extdef([], str, llimpl=getlogin_llimpl,
-                      export_name="ll_os.ll_os_getlogin")
-
     @registering_str_unicode(os.utime)
     def register_os_utime(self, traits):
         UTIMBUFP = lltype.Ptr(self.UTIMBUF)
diff --git a/rpython/rtyper/module/test/test_ll_os.py b/rpython/rtyper/module/test/test_ll_os.py
--- a/rpython/rtyper/module/test/test_ll_os.py
+++ b/rpython/rtyper/module/test/test_ll_os.py
@@ -44,7 +44,7 @@
         expected = os.getlogin()
     except OSError, e:
         py.test.skip("the underlying os.getlogin() failed: %s" % e)
-    data = getllimpl(os.getlogin)()
+    data = rposix.getlogin()
     assert data == expected
 
 def test_statvfs():

From noreply at buildbot.pypy.org  Fri Nov  7 14:52:18 2014
From: noreply at buildbot.pypy.org (amauryfa)
Date: Fri,  7 Nov 2014 14:52:18 +0100 (CET)
Subject: [pypy-commit] pypy more-rposix: Port os.utime.
Message-ID: <20141107135218.680B61C030E@cobra.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc 
Branch: more-rposix
Changeset: r74363:586ee1f92428
Date: 2014-11-06 23:51 +0100
http://bitbucket.org/pypy/pypy/changeset/586ee1f92428/

Log:	Port os.utime.

	A lot of changes, but it's actually 30 lines less than before.

diff --git a/rpython/rlib/rposix.py b/rpython/rlib/rposix.py
--- a/rpython/rlib/rposix.py
+++ b/rpython/rlib/rposix.py
@@ -1,7 +1,8 @@
 import os
 import sys
 from rpython.rtyper.lltypesystem.rffi import CConstant, CExternVariable, INT
-from rpython.rtyper.lltypesystem import ll2ctypes, rffi
+from rpython.rtyper.lltypesystem import lltype, ll2ctypes, rffi
+from rpython.rtyper.tool import rffi_platform
 from rpython.translator.tool.cbuild import ExternalCompilationInfo
 from rpython.rlib.rarithmetic import intmask
 from rpython.rlib.objectmodel import (
@@ -135,13 +136,22 @@
             pass
 
 if _WIN32:
-    includes = ['io.h']
+    includes = ['io.h', 'sys/utime.h', 'sys/types.h']
 else:
-    includes = ['unistd.h']
+    includes = ['unistd.h', 'utime.h', 'sys/time.h', 'sys/types.h']
 eci = ExternalCompilationInfo(
     includes=includes,
 )
 
+class CConfig:
+    _compilation_info_ = eci
+    HAVE_UTIMES = rffi_platform.Has('utimes')
+    UTIMBUF = rffi_platform.Struct('struct %sutimbuf' % UNDERSCORE_ON_WIN32,
+                                   [('actime', rffi.INT),
+                                    ('modtime', rffi.INT)])
+config = rffi_platform.configure(CConfig)
+globals().update(config)
+
 def external(name, args, result, **kwds):
     return rffi.llexternal(name, args, result, compilation_info=eci, **kwds)
 
@@ -257,10 +267,6 @@
 def chmod(path, mode):
     return os.chmod(_as_bytes(path), mode)
 
- at specialize.argtype(0, 1)
-def utime(path, times):
-    return os.utime(_as_bytes(path), times)
-
 @specialize.argtype(0)
 def chdir(path):
     return os.chdir(_as_bytes(path))
@@ -398,3 +404,116 @@
     if not result:
         raise OSError(get_errno(), "getlogin failed")
     return rffi.charp2str(result)
+
+
+#_______________________________________________________________
+
+UTIMBUFP = lltype.Ptr(UTIMBUF)
+c_utime = external('utime', [rffi.CCHARP, UTIMBUFP], rffi.INT)
+if HAVE_UTIMES:
+    class CConfig:
+        _compilation_info_ = eci
+        TIMEVAL = rffi_platform.Struct('struct timeval', [
+            ('tv_sec', rffi.LONG),
+            ('tv_usec', rffi.LONG)])
+    config = rffi_platform.configure(CConfig)
+    TIMEVAL = config['TIMEVAL']
+    TIMEVAL2P = rffi.CArrayPtr(TIMEVAL)
+    c_utimes = external('utimes', [rffi.CCHARP, TIMEVAL2P], rffi.INT)
+
+if _WIN32:
+    from rpython.rlib.rposix import rwin32
+    GetSystemTime = external(
+        'GetSystemTime',
+        [lltype.Ptr(rwin32.SYSTEMTIME)],
+        lltype.Void,
+        calling_conv='win')
+
+    SystemTimeToFileTime = external(
+        'SystemTimeToFileTime',
+        [lltype.Ptr(rwin32.SYSTEMTIME),
+         lltype.Ptr(rwin32.FILETIME)],
+        rwin32.BOOL,
+        calling_conv='win')
+
+    SetFileTime = external(
+        'SetFileTime',
+        [rwin32.HANDLE,
+         lltype.Ptr(rwin32.FILETIME),
+         lltype.Ptr(rwin32.FILETIME),
+         lltype.Ptr(rwin32.FILETIME)],
+        rwin32.BOOL,
+        calling_conv='win')
+
+
+ at register_replacement_for(getattr(os, 'utime', None),
+                          sandboxed_name='ll_os.ll_os_utime')
+ at specialize.argtype(0, 1)
+def utime(path, times):
+    if not _WIN32:
+        path = _as_bytes0(path)
+        if times is None:
+            error = c_utime(path, lltype.nullptr(UTIMBUFP.TO))
+        else:
+            actime, modtime = times
+            if HAVE_UTIMES:
+                import math
+                l_times = lltype.malloc(TIMEVAL2P.TO, 2, flavor='raw')
+                fracpart, intpart = math.modf(actime)
+                rffi.setintfield(l_times[0], 'c_tv_sec', int(intpart))
+                rffi.setintfield(l_times[0], 'c_tv_usec', int(fracpart * 1e6))
+                fracpart, intpart = math.modf(modtime)
+                rffi.setintfield(l_times[1], 'c_tv_sec', int(intpart))
+                rffi.setintfield(l_times[1], 'c_tv_usec', int(fracpart * 1e6))
+                error = c_utimes(path, l_times)
+                lltype.free(l_times, flavor='raw')
+            else:
+                l_utimbuf = lltype.malloc(UTIMBUFP.TO, flavor='raw')
+                l_utimbuf.c_actime  = rffi.r_time_t(actime)
+                l_utimbuf.c_modtime = rffi.r_time_t(modtime)
+                error = c_utime(path, l_utimbuf)
+                lltype.free(l_utimbuf, flavor='raw')
+        if error < 0:
+            raise OSError(get_errno(), "os_utime failed")
+    else:  # _WIN32 case
+        from rpython.rlib.rwin32file import make_win32_traits
+        if _prefer_unicode(path):
+            # XXX remove dependency on rtyper.module.  The "traits"
+            # are just used for CreateFile anyway.
+            from rpython.rtyper.module.support import UnicodeTraits
+            win32traits = make_win32_traits(UnicodeTraits())
+            path = _as_unicode0(path)
+        else:
+            from rpython.rtyper.module.support import StringTraits
+            win32traits = make_win32_traits(StringTraits())
+            path = _as_bytes0(path)
+        hFile = win32traits.CreateFile(path,
+                           win32traits.FILE_WRITE_ATTRIBUTES, 0,
+                           None, win32traits.OPEN_EXISTING,
+                           win32traits.FILE_FLAG_BACKUP_SEMANTICS,
+                           rwin32.NULL_HANDLE)
+        if hFile == rwin32.INVALID_HANDLE_VALUE:
+            raise rwin32.lastWindowsError()
+        ctime = lltype.nullptr(rwin32.FILETIME)
+        atime = lltype.malloc(rwin32.FILETIME, flavor='raw')
+        mtime = lltype.malloc(rwin32.FILETIME, flavor='raw')
+        try:
+            if tp is None:
+                now = lltype.malloc(rwin32.SYSTEMTIME, flavor='raw')
+                try:
+                    GetSystemTime(now)
+                    if (not SystemTimeToFileTime(now, atime) or
+                        not SystemTimeToFileTime(now, mtime)):
+                        raise rwin32.lastWindowsError()
+                finally:
+                    lltype.free(now, flavor='raw')
+            else:
+                actime, modtime = tp
+                time_t_to_FILE_TIME(actime, atime)
+                time_t_to_FILE_TIME(modtime, mtime)
+            if not SetFileTime(hFile, ctime, atime, mtime):
+                raise rwin32.lastWindowsError()
+        finally:
+            rwin32.CloseHandle(hFile)
+            lltype.free(atime, flavor='raw')
+            lltype.free(mtime, flavor='raw')
diff --git a/rpython/rlib/rwin32file.py b/rpython/rlib/rwin32file.py
new file mode 100644
--- /dev/null
+++ b/rpython/rlib/rwin32file.py
@@ -0,0 +1,192 @@
+"""
+Win32 API functions around files.
+"""
+
+from rpython.rtyper.lltypesystem import lltype, rffi
+from rpython.translator.tool.cbuild import ExternalCompilationInfo
+from rpython.rtyper.tool import rffi_platform as platform
+
+
+def make_win32_traits(traits):
+    from rpython.rlib import rwin32
+
+    if traits.str is unicode:
+        suffix = 'W'
+    else:
+        suffix = 'A'
+
+    class CConfig:
+        _compilation_info_ = ExternalCompilationInfo(
+            includes = ['windows.h', 'winbase.h', 'sys/stat.h'],
+        )
+        WIN32_FIND_DATA = platform.Struct(
+            'struct _WIN32_FIND_DATA' + suffix,
+            # Only interesting fields
+            [('dwFileAttributes', rwin32.DWORD),
+             ('nFileSizeHigh', rwin32.DWORD),
+             ('nFileSizeLow', rwin32.DWORD),
+             ('ftCreationTime', rwin32.FILETIME),
+             ('ftLastAccessTime', rwin32.FILETIME),
+             ('ftLastWriteTime', rwin32.FILETIME),
+             ('cFileName', lltype.FixedSizeArray(traits.CHAR, 250))])
+        ERROR_FILE_NOT_FOUND = platform.ConstantInteger(
+            'ERROR_FILE_NOT_FOUND')
+        ERROR_NO_MORE_FILES = platform.ConstantInteger(
+            'ERROR_NO_MORE_FILES')
+
+        GetFileExInfoStandard = platform.ConstantInteger(
+            'GetFileExInfoStandard')
+        FILE_ATTRIBUTE_DIRECTORY = platform.ConstantInteger(
+            'FILE_ATTRIBUTE_DIRECTORY')
+        FILE_ATTRIBUTE_READONLY = platform.ConstantInteger(
+            'FILE_ATTRIBUTE_READONLY')
+        INVALID_FILE_ATTRIBUTES = platform.ConstantInteger(
+            'INVALID_FILE_ATTRIBUTES')
+        ERROR_SHARING_VIOLATION = platform.ConstantInteger(
+            'ERROR_SHARING_VIOLATION')
+        _S_IFDIR = platform.ConstantInteger('_S_IFDIR')
+        _S_IFREG = platform.ConstantInteger('_S_IFREG')
+        _S_IFCHR = platform.ConstantInteger('_S_IFCHR')
+        _S_IFIFO = platform.ConstantInteger('_S_IFIFO')
+        FILE_TYPE_UNKNOWN = platform.ConstantInteger('FILE_TYPE_UNKNOWN')
+        FILE_TYPE_CHAR = platform.ConstantInteger('FILE_TYPE_CHAR')
+        FILE_TYPE_PIPE = platform.ConstantInteger('FILE_TYPE_PIPE')
+
+        FILE_WRITE_ATTRIBUTES = platform.ConstantInteger(
+            'FILE_WRITE_ATTRIBUTES')
+        OPEN_EXISTING = platform.ConstantInteger(
+            'OPEN_EXISTING')
+        FILE_FLAG_BACKUP_SEMANTICS = platform.ConstantInteger(
+            'FILE_FLAG_BACKUP_SEMANTICS')
+        VOLUME_NAME_DOS = platform.ConstantInteger('VOLUME_NAME_DOS')
+        VOLUME_NAME_NT = platform.ConstantInteger('VOLUME_NAME_NT')
+
+        WIN32_FILE_ATTRIBUTE_DATA = platform.Struct(
+            'WIN32_FILE_ATTRIBUTE_DATA',
+            [('dwFileAttributes', rwin32.DWORD),
+             ('nFileSizeHigh', rwin32.DWORD),
+             ('nFileSizeLow', rwin32.DWORD),
+             ('ftCreationTime', rwin32.FILETIME),
+             ('ftLastAccessTime', rwin32.FILETIME),
+             ('ftLastWriteTime', rwin32.FILETIME)])
+
+        BY_HANDLE_FILE_INFORMATION = platform.Struct(
+            'BY_HANDLE_FILE_INFORMATION',
+            [('dwFileAttributes', rwin32.DWORD),
+             ('ftCreationTime', rwin32.FILETIME),
+             ('ftLastAccessTime', rwin32.FILETIME),
+             ('ftLastWriteTime', rwin32.FILETIME),
+             ('dwVolumeSerialNumber', rwin32.DWORD),
+             ('nFileSizeHigh', rwin32.DWORD),
+             ('nFileSizeLow', rwin32.DWORD),
+             ('nNumberOfLinks', rwin32.DWORD),
+             ('nFileIndexHigh', rwin32.DWORD),
+             ('nFileIndexLow', rwin32.DWORD)])
+
+    config = platform.configure(CConfig)
+
+    def external(*args, **kwargs):
+        kwargs['compilation_info'] = CConfig._compilation_info_
+        llfunc = rffi.llexternal(calling_conv='win', *args, **kwargs)
+        return staticmethod(llfunc)
+
+    class Win32Traits:
+        apisuffix = suffix
+
+        for name in '''WIN32_FIND_DATA WIN32_FILE_ATTRIBUTE_DATA BY_HANDLE_FILE_INFORMATION
+                       GetFileExInfoStandard
+                       FILE_ATTRIBUTE_DIRECTORY FILE_ATTRIBUTE_READONLY
+                       INVALID_FILE_ATTRIBUTES
+                       _S_IFDIR _S_IFREG _S_IFCHR _S_IFIFO
+                       FILE_TYPE_UNKNOWN FILE_TYPE_CHAR FILE_TYPE_PIPE
+                       FILE_WRITE_ATTRIBUTES OPEN_EXISTING FILE_FLAG_BACKUP_SEMANTICS
+                       VOLUME_NAME_DOS VOLUME_NAME_NT
+                       ERROR_FILE_NOT_FOUND ERROR_NO_MORE_FILES
+                       ERROR_SHARING_VIOLATION
+                    '''.split():
+            locals()[name] = config[name]
+        LPWIN32_FIND_DATA    = lltype.Ptr(WIN32_FIND_DATA)
+        GET_FILEEX_INFO_LEVELS = rffi.ULONG # an enumeration
+
+        FindFirstFile = external('FindFirstFile' + suffix,
+                                 [traits.CCHARP, LPWIN32_FIND_DATA],
+                                 rwin32.HANDLE)
+        FindNextFile = external('FindNextFile' + suffix,
+                                [rwin32.HANDLE, LPWIN32_FIND_DATA],
+                                rwin32.BOOL)
+        FindClose = external('FindClose',
+                             [rwin32.HANDLE],
+                             rwin32.BOOL)
+
+        GetFileAttributes = external(
+            'GetFileAttributes' + suffix,
+            [traits.CCHARP],
+            rwin32.DWORD)
+
+        SetFileAttributes = external(
+            'SetFileAttributes' + suffix,
+            [traits.CCHARP, rwin32.DWORD],
+            rwin32.BOOL)
+
+        GetFileAttributesEx = external(
+            'GetFileAttributesEx' + suffix,
+            [traits.CCHARP, GET_FILEEX_INFO_LEVELS,
+             lltype.Ptr(WIN32_FILE_ATTRIBUTE_DATA)],
+            rwin32.BOOL)
+
+        GetFileInformationByHandle = external(
+            'GetFileInformationByHandle',
+            [rwin32.HANDLE, lltype.Ptr(BY_HANDLE_FILE_INFORMATION)],
+            rwin32.BOOL)
+
+        GetFileType = external(
+            'GetFileType',
+            [rwin32.HANDLE],
+            rwin32.DWORD)
+
+        LPSTRP = rffi.CArrayPtr(traits.CCHARP)
+
+        GetFullPathName = external(
+            'GetFullPathName' + suffix,
+            [traits.CCHARP, rwin32.DWORD,
+             traits.CCHARP, LPSTRP],
+            rwin32.DWORD)
+
+        GetCurrentDirectory = external(
+            'GetCurrentDirectory' + suffix,
+            [rwin32.DWORD, traits.CCHARP],
+            rwin32.DWORD)
+
+        SetCurrentDirectory = external(
+            'SetCurrentDirectory' + suffix,
+            [traits.CCHARP],
+            rwin32.BOOL)
+
+        CreateDirectory = external(
+            'CreateDirectory' + suffix,
+            [traits.CCHARP, rffi.VOIDP],
+            rwin32.BOOL)
+
+        SetEnvironmentVariable = external(
+            'SetEnvironmentVariable' + suffix,
+            [traits.CCHARP, traits.CCHARP],
+            rwin32.BOOL)
+
+        CreateFile = external(
+            'CreateFile' + apisuffix,
+            [traits.CCHARP, rwin32.DWORD, rwin32.DWORD,
+             rwin32.LPSECURITY_ATTRIBUTES, rwin32.DWORD, rwin32.DWORD,
+             rwin32.HANDLE],
+            rwin32.HANDLE)
+
+        DeleteFile = external(
+            'DeleteFile' + suffix,
+            [traits.CCHARP],
+            rwin32.BOOL)
+
+        MoveFile = external(
+            'MoveFile' + suffix,
+            [traits.CCHARP, traits.CCHARP],
+            rwin32.BOOL)
+
+    return Win32Traits
diff --git a/rpython/rtyper/module/ll_os.py b/rpython/rtyper/module/ll_os.py
--- a/rpython/rtyper/module/ll_os.py
+++ b/rpython/rtyper/module/ll_os.py
@@ -247,104 +247,6 @@
         return extdef([int], int, llimpl=c_func_llimpl,
                       export_name='ll_os.ll_os_' + name)
 
-    @registering_str_unicode(os.utime)
-    def register_os_utime(self, traits):
-        UTIMBUFP = lltype.Ptr(self.UTIMBUF)
-        os_utime = self.llexternal('utime', [rffi.CCHARP, UTIMBUFP], rffi.INT)
-
-        if not _WIN32:
-            includes = ['sys/time.h']
-        else:
-            includes = ['time.h']
-        eci = ExternalCompilationInfo(includes=includes)
-
-        class CConfig:
-            _compilation_info_ = eci
-            HAVE_UTIMES = platform.Has('utimes')
-        config = platform.configure(CConfig)
-
-        # XXX note that on Windows, calls to os.utime() are ignored on
-        # directories.  Remove that hack over there once it's fixed here!
-
-        if config['HAVE_UTIMES']:
-            class CConfig:
-                _compilation_info_ = eci
-                TIMEVAL = platform.Struct('struct timeval', [('tv_sec', rffi.LONG),
-                                                             ('tv_usec', rffi.LONG)])
-            config = platform.configure(CConfig)
-            TIMEVAL = config['TIMEVAL']
-            TIMEVAL2P = rffi.CArrayPtr(TIMEVAL)
-            os_utimes = self.llexternal('utimes', [rffi.CCHARP, TIMEVAL2P],
-                                        rffi.INT, compilation_info=eci)
-
-            def os_utime_platform(path, actime, modtime):
-                import math
-                l_times = lltype.malloc(TIMEVAL2P.TO, 2, flavor='raw')
-                fracpart, intpart = math.modf(actime)
-                rffi.setintfield(l_times[0], 'c_tv_sec', int(intpart))
-                rffi.setintfield(l_times[0], 'c_tv_usec', int(fracpart * 1E6))
-                fracpart, intpart = math.modf(modtime)
-                rffi.setintfield(l_times[1], 'c_tv_sec', int(intpart))
-                rffi.setintfield(l_times[1], 'c_tv_usec', int(fracpart * 1E6))
-                error = os_utimes(path, l_times)
-                lltype.free(l_times, flavor='raw')
-                return error
-        else:
-            # we only have utime(), which does not allow sub-second resolution
-            def os_utime_platform(path, actime, modtime):
-                l_utimbuf = lltype.malloc(UTIMBUFP.TO, flavor='raw')
-                l_utimbuf.c_actime  = rffi.r_time_t(actime)
-                l_utimbuf.c_modtime = rffi.r_time_t(modtime)
-                error = os_utime(path, l_utimbuf)
-                lltype.free(l_utimbuf, flavor='raw')
-                return error
-
-        # NB. this function is specialized; we get one version where
-        # tp is known to be None, and one version where it is known
-        # to be a tuple of 2 floats.
-        if not _WIN32:
-            assert traits.str is str
-
-            @specialize.argtype(1)
-            def os_utime_llimpl(path, tp):
-                if tp is None:
-                    error = os_utime(path, lltype.nullptr(UTIMBUFP.TO))
-                else:
-                    actime, modtime = tp
-                    error = os_utime_platform(path, actime, modtime)
-                    error = rffi.cast(lltype.Signed, error)
-                if error == -1:
-                    raise OSError(rposix.get_errno(), "os_utime failed")
-        else:
-            from rpython.rtyper.module.ll_win32file import make_utime_impl
-            os_utime_llimpl = make_utime_impl(traits)
-
-        s_tuple_of_2_floats = SomeTuple([SomeFloat(), SomeFloat()])
-
-        def os_utime_normalize_args(s_path, s_times):
-            # special handling of the arguments: they can be either
-            # [str, (float, float)] or [str, s_None], and get normalized
-            # to exactly one of these two.
-            if not traits.str0.contains(s_path):
-                raise Exception("os.utime() arg 1 must be a string, got %s" % (
-                    s_path,))
-            case1 = s_None.contains(s_times)
-            case2 = s_tuple_of_2_floats.contains(s_times)
-            if case1 and case2:
-                return [traits.str0, s_ImpossibleValue] #don't know which case yet
-            elif case1:
-                return [traits.str0, s_None]
-            elif case2:
-                return [traits.str0, s_tuple_of_2_floats]
-            else:
-                raise Exception("os.utime() arg 2 must be None or a tuple of "
-                                "2 floats, got %s" % (s_times,))
-        os_utime_normalize_args._default_signature_ = [traits.str0, None]
-
-        return extdef(os_utime_normalize_args, s_None,
-                      "ll_os.ll_os_utime",
-                      llimpl=os_utime_llimpl)
-
     @registering(os.times)
     def register_os_times(self):
         if sys.platform.startswith('win'):
@@ -1357,7 +1259,7 @@
                 raise OSError(rposix.get_errno(), "os_unlink failed")
 
         if sys.platform == 'win32':
-            from rpython.rtyper.module.ll_win32file import make_win32_traits
+            from rpython.rlib.rwin32file import make_win32_traits
             win32traits = make_win32_traits(traits)
 
             @func_renamer('unlink_llimpl_%s' % traits.str.__name__)
@@ -1392,7 +1294,7 @@
                                    [traits.CCHARP, rffi.MODE_T], rffi.INT)
 
         if sys.platform == 'win32':
-            from rpython.rtyper.module.ll_win32file import make_win32_traits
+            from rpython.rlib.rwin32file import make_win32_traits
             win32traits = make_win32_traits(traits)
 
             @func_renamer('mkdir_llimpl_%s' % traits.str.__name__)
@@ -1464,7 +1366,7 @@
                 raise OSError(rposix.get_errno(), "os_rename failed")
 
         if sys.platform == 'win32':
-            from rpython.rtyper.module.ll_win32file import make_win32_traits
+            from rpython.rlib.rwin32file import make_win32_traits
             win32traits = make_win32_traits(traits)
 
             @func_renamer('rename_llimpl_%s' % traits.str.__name__)
diff --git a/rpython/rtyper/module/ll_os_stat.py b/rpython/rtyper/module/ll_os_stat.py
--- a/rpython/rtyper/module/ll_os_stat.py
+++ b/rpython/rtyper/module/ll_os_stat.py
@@ -440,7 +440,7 @@
 
 def make_win32_stat_impl(name, traits):
     from rpython.rlib import rwin32
-    from rpython.rtyper.module.ll_win32file import make_win32_traits
+    from rpython.rlib.rwin32file import make_win32_traits
     win32traits = make_win32_traits(traits)
 
     # The CRT of Windows has a number of flaws wrt. its stat() implementation:
diff --git a/rpython/rtyper/module/ll_win32file.py b/rpython/rtyper/module/ll_win32file.py
--- a/rpython/rtyper/module/ll_win32file.py
+++ b/rpython/rtyper/module/ll_win32file.py
@@ -10,195 +10,14 @@
 from rpython.tool.sourcetools import func_renamer
 from rpython.rlib.objectmodel import specialize
 
-def make_win32_traits(traits):
-    from rpython.rlib import rwin32
-
-    if traits.str is unicode:
-        suffix = 'W'
-    else:
-        suffix = 'A'
-
-    class CConfig:
-        _compilation_info_ = ExternalCompilationInfo(
-            includes = ['windows.h', 'winbase.h', 'sys/stat.h'],
-        )
-        WIN32_FIND_DATA = platform.Struct(
-            'struct _WIN32_FIND_DATA' + suffix,
-            # Only interesting fields
-            [('dwFileAttributes', rwin32.DWORD),
-             ('nFileSizeHigh', rwin32.DWORD),
-             ('nFileSizeLow', rwin32.DWORD),
-             ('ftCreationTime', rwin32.FILETIME),
-             ('ftLastAccessTime', rwin32.FILETIME),
-             ('ftLastWriteTime', rwin32.FILETIME),
-             ('cFileName', lltype.FixedSizeArray(traits.CHAR, 250))])
-        ERROR_FILE_NOT_FOUND = platform.ConstantInteger(
-            'ERROR_FILE_NOT_FOUND')
-        ERROR_NO_MORE_FILES = platform.ConstantInteger(
-            'ERROR_NO_MORE_FILES')
-
-        GetFileExInfoStandard = platform.ConstantInteger(
-            'GetFileExInfoStandard')
-        FILE_ATTRIBUTE_DIRECTORY = platform.ConstantInteger(
-            'FILE_ATTRIBUTE_DIRECTORY')
-        FILE_ATTRIBUTE_READONLY = platform.ConstantInteger(
-            'FILE_ATTRIBUTE_READONLY')
-        INVALID_FILE_ATTRIBUTES = platform.ConstantInteger(
-            'INVALID_FILE_ATTRIBUTES')
-        ERROR_SHARING_VIOLATION = platform.ConstantInteger(
-            'ERROR_SHARING_VIOLATION')
-        _S_IFDIR = platform.ConstantInteger('_S_IFDIR')
-        _S_IFREG = platform.ConstantInteger('_S_IFREG')
-        _S_IFCHR = platform.ConstantInteger('_S_IFCHR')
-        _S_IFIFO = platform.ConstantInteger('_S_IFIFO')
-        FILE_TYPE_UNKNOWN = platform.ConstantInteger('FILE_TYPE_UNKNOWN')
-        FILE_TYPE_CHAR = platform.ConstantInteger('FILE_TYPE_CHAR')
-        FILE_TYPE_PIPE = platform.ConstantInteger('FILE_TYPE_PIPE')
-
-        FILE_WRITE_ATTRIBUTES = platform.ConstantInteger(
-            'FILE_WRITE_ATTRIBUTES')
-        OPEN_EXISTING = platform.ConstantInteger(
-            'OPEN_EXISTING')
-        FILE_FLAG_BACKUP_SEMANTICS = platform.ConstantInteger(
-            'FILE_FLAG_BACKUP_SEMANTICS')
-        VOLUME_NAME_DOS = platform.ConstantInteger('VOLUME_NAME_DOS')
-        VOLUME_NAME_NT = platform.ConstantInteger('VOLUME_NAME_NT')
-
-        WIN32_FILE_ATTRIBUTE_DATA = platform.Struct(
-            'WIN32_FILE_ATTRIBUTE_DATA',
-            [('dwFileAttributes', rwin32.DWORD),
-             ('nFileSizeHigh', rwin32.DWORD),
-             ('nFileSizeLow', rwin32.DWORD),
-             ('ftCreationTime', rwin32.FILETIME),
-             ('ftLastAccessTime', rwin32.FILETIME),
-             ('ftLastWriteTime', rwin32.FILETIME)])
-
-        BY_HANDLE_FILE_INFORMATION = platform.Struct(
-            'BY_HANDLE_FILE_INFORMATION',
-            [('dwFileAttributes', rwin32.DWORD),
-             ('ftCreationTime', rwin32.FILETIME),
-             ('ftLastAccessTime', rwin32.FILETIME),
-             ('ftLastWriteTime', rwin32.FILETIME),
-             ('dwVolumeSerialNumber', rwin32.DWORD),
-             ('nFileSizeHigh', rwin32.DWORD),
-             ('nFileSizeLow', rwin32.DWORD),
-             ('nNumberOfLinks', rwin32.DWORD),
-             ('nFileIndexHigh', rwin32.DWORD),
-             ('nFileIndexLow', rwin32.DWORD)])
-
-    config = platform.configure(CConfig)
-
-    def external(*args, **kwargs):
-        kwargs['compilation_info'] = CConfig._compilation_info_
-        llfunc = rffi.llexternal(calling_conv='win', *args, **kwargs)
-        return staticmethod(llfunc)
-
-    class Win32Traits:
-        apisuffix = suffix
-
-        for name in '''WIN32_FIND_DATA WIN32_FILE_ATTRIBUTE_DATA BY_HANDLE_FILE_INFORMATION
-                       GetFileExInfoStandard
-                       FILE_ATTRIBUTE_DIRECTORY FILE_ATTRIBUTE_READONLY
-                       INVALID_FILE_ATTRIBUTES
-                       _S_IFDIR _S_IFREG _S_IFCHR _S_IFIFO
-                       FILE_TYPE_UNKNOWN FILE_TYPE_CHAR FILE_TYPE_PIPE
-                       FILE_WRITE_ATTRIBUTES OPEN_EXISTING FILE_FLAG_BACKUP_SEMANTICS
-                       VOLUME_NAME_DOS VOLUME_NAME_NT
-                       ERROR_FILE_NOT_FOUND ERROR_NO_MORE_FILES
-                       ERROR_SHARING_VIOLATION
-                    '''.split():
-            locals()[name] = config[name]
-        LPWIN32_FIND_DATA    = lltype.Ptr(WIN32_FIND_DATA)
-        GET_FILEEX_INFO_LEVELS = rffi.ULONG # an enumeration
-
-        FindFirstFile = external('FindFirstFile' + suffix,
-                                 [traits.CCHARP, LPWIN32_FIND_DATA],
-                                 rwin32.HANDLE)
-        FindNextFile = external('FindNextFile' + suffix,
-                                [rwin32.HANDLE, LPWIN32_FIND_DATA],
-                                rwin32.BOOL)
-        FindClose = external('FindClose',
-                             [rwin32.HANDLE],
-                             rwin32.BOOL)
-
-        GetFileAttributes = external(
-            'GetFileAttributes' + suffix,
-            [traits.CCHARP],
-            rwin32.DWORD)
-
-        SetFileAttributes = external(
-            'SetFileAttributes' + suffix,
-            [traits.CCHARP, rwin32.DWORD],
-            rwin32.BOOL)
-
-        GetFileAttributesEx = external(
-            'GetFileAttributesEx' + suffix,
-            [traits.CCHARP, GET_FILEEX_INFO_LEVELS,
-             lltype.Ptr(WIN32_FILE_ATTRIBUTE_DATA)],
-            rwin32.BOOL)
-
-        GetFileInformationByHandle = external(
-            'GetFileInformationByHandle',
-            [rwin32.HANDLE, lltype.Ptr(BY_HANDLE_FILE_INFORMATION)],
-            rwin32.BOOL)
-
-        GetFileType = external(
-            'GetFileType',
-            [rwin32.HANDLE],
-            rwin32.DWORD)
-
-        LPSTRP = rffi.CArrayPtr(traits.CCHARP)
-
-        GetFullPathName = external(
-            'GetFullPathName' + suffix,
-            [traits.CCHARP, rwin32.DWORD,
-             traits.CCHARP, LPSTRP],
-            rwin32.DWORD)
-
-        GetCurrentDirectory = external(
-            'GetCurrentDirectory' + suffix,
-            [rwin32.DWORD, traits.CCHARP],
-            rwin32.DWORD)
-
-        SetCurrentDirectory = external(
-            'SetCurrentDirectory' + suffix,
-            [traits.CCHARP],
-            rwin32.BOOL)
-
-        CreateDirectory = external(
-            'CreateDirectory' + suffix,
-            [traits.CCHARP, rffi.VOIDP],
-            rwin32.BOOL)
-
-        SetEnvironmentVariable = external(
-            'SetEnvironmentVariable' + suffix,
-            [traits.CCHARP, traits.CCHARP],
-            rwin32.BOOL)
-
-        CreateFile = external(
-            'CreateFile' + apisuffix,
-            [traits.CCHARP, rwin32.DWORD, rwin32.DWORD,
-             rwin32.LPSECURITY_ATTRIBUTES, rwin32.DWORD, rwin32.DWORD,
-             rwin32.HANDLE],
-            rwin32.HANDLE)
-
-        DeleteFile = external(
-            'DeleteFile' + suffix,
-            [traits.CCHARP],
-            rwin32.BOOL)
-
-        MoveFile = external(
-            'MoveFile' + suffix,
-            [traits.CCHARP, traits.CCHARP],
-            rwin32.BOOL)
-
-    return Win32Traits
 
 #_______________________________________________________________
 # listdir
 
 def make_listdir_impl(traits):
     from rpython.rlib import rwin32
+    from rpython.rlib.rwin32file import make_win32_traits
+
     win32traits = make_win32_traits(traits)
 
     if traits.str is unicode:
@@ -256,6 +75,8 @@
 
 def make_chdir_impl(traits):
     from rpython.rlib import rwin32
+    from rpython.rlib.rwin32file import make_win32_traits
+
     win32traits = make_win32_traits(traits)
 
     if traits.str is unicode:
@@ -310,6 +131,8 @@
 
 def make_chmod_impl(traits):
     from rpython.rlib import rwin32
+    from rpython.rlib.rwin32file import make_win32_traits
+
     win32traits = make_win32_traits(traits)
 
     @func_renamer('chmod_llimpl_%s' % traits.str.__name__)
@@ -349,65 +172,3 @@
             lltype.free(lpBuffer, flavor='raw')
 
     return getfullpathname_llimpl
-
-def make_utime_impl(traits):
-    from rpython.rlib import rwin32
-    win32traits = make_win32_traits(traits)
-    from rpython.rtyper.module.ll_os_stat import time_t_to_FILE_TIME
-
-    GetSystemTime = rffi.llexternal(
-        'GetSystemTime',
-        [lltype.Ptr(rwin32.SYSTEMTIME)],
-        lltype.Void,
-        calling_conv='win')
-
-    SystemTimeToFileTime = rffi.llexternal(
-        'SystemTimeToFileTime',
-        [lltype.Ptr(rwin32.SYSTEMTIME),
-         lltype.Ptr(rwin32.FILETIME)],
-        rwin32.BOOL,
-        calling_conv='win')
-
-    SetFileTime = rffi.llexternal(
-        'SetFileTime',
-        [rwin32.HANDLE,
-         lltype.Ptr(rwin32.FILETIME),
-         lltype.Ptr(rwin32.FILETIME),
-         lltype.Ptr(rwin32.FILETIME)],
-        rwin32.BOOL,
-        calling_conv = 'win')
-
-    @specialize.argtype(1)
-    def os_utime_llimpl(path, tp):
-        hFile = win32traits.CreateFile(path,
-                           win32traits.FILE_WRITE_ATTRIBUTES, 0,
-                           None, win32traits.OPEN_EXISTING,
-                           win32traits.FILE_FLAG_BACKUP_SEMANTICS,
-                           rwin32.NULL_HANDLE)
-        if hFile == rwin32.INVALID_HANDLE_VALUE:
-            raise rwin32.lastWindowsError()
-        ctime = lltype.nullptr(rwin32.FILETIME)
-        atime = lltype.malloc(rwin32.FILETIME, flavor='raw')
-        mtime = lltype.malloc(rwin32.FILETIME, flavor='raw')
-        try:
-            if tp is None:
-                now = lltype.malloc(rwin32.SYSTEMTIME, flavor='raw')
-                try:
-                    GetSystemTime(now)
-                    if (not SystemTimeToFileTime(now, atime) or
-                        not SystemTimeToFileTime(now, mtime)):
-                        raise rwin32.lastWindowsError()
-                finally:
-                    lltype.free(now, flavor='raw')
-            else:
-                actime, modtime = tp
-                time_t_to_FILE_TIME(actime, atime)
-                time_t_to_FILE_TIME(modtime, mtime)
-            if not SetFileTime(hFile, ctime, atime, mtime):
-                raise rwin32.lastWindowsError()
-        finally:
-            rwin32.CloseHandle(hFile)
-            lltype.free(atime, flavor='raw')
-            lltype.free(mtime, flavor='raw')
-
-    return os_utime_llimpl

From noreply at buildbot.pypy.org  Fri Nov  7 14:52:19 2014
From: noreply at buildbot.pypy.org (amauryfa)
Date: Fri,  7 Nov 2014 14:52:19 +0100 (CET)
Subject: [pypy-commit] pypy more-rposix: Move os.times()
Message-ID: <20141107135219.886531C030E@cobra.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc 
Branch: more-rposix
Changeset: r74364:b159fdc7b679
Date: 2014-11-07 00:13 +0100
http://bitbucket.org/pypy/pypy/changeset/b159fdc7b679/

Log:	Move os.times()

diff --git a/rpython/rlib/rposix.py b/rpython/rlib/rposix.py
--- a/rpython/rlib/rposix.py
+++ b/rpython/rlib/rposix.py
@@ -138,7 +138,8 @@
 if _WIN32:
     includes = ['io.h', 'sys/utime.h', 'sys/types.h']
 else:
-    includes = ['unistd.h', 'utime.h', 'sys/time.h', 'sys/types.h']
+    includes = ['unistd.h',  'sys/types.h',
+                'utime.h', 'sys/time.h', 'sys/times.h']
 eci = ExternalCompilationInfo(
     includes=includes,
 )
@@ -149,6 +150,15 @@
     UTIMBUF = rffi_platform.Struct('struct %sutimbuf' % UNDERSCORE_ON_WIN32,
                                    [('actime', rffi.INT),
                                     ('modtime', rffi.INT)])
+    if not _WIN32:
+        CLOCK_T = rffi_platform.SimpleType('clock_t', rffi.INT)
+
+        TMS = rffi_platform.Struct(
+            'struct tms', [('tms_utime', rffi.INT),
+                           ('tms_stime', rffi.INT),
+                           ('tms_cutime', rffi.INT),
+                           ('tms_cstime', rffi.INT)])
+
 config = rffi_platform.configure(CConfig)
 globals().update(config)
 
@@ -422,7 +432,7 @@
     c_utimes = external('utimes', [rffi.CCHARP, TIMEVAL2P], rffi.INT)
 
 if _WIN32:
-    from rpython.rlib.rposix import rwin32
+    from rpython.rlib import rwin32
     GetSystemTime = external(
         'GetSystemTime',
         [lltype.Ptr(rwin32.SYSTEMTIME)],
@@ -517,3 +527,67 @@
             rwin32.CloseHandle(hFile)
             lltype.free(atime, flavor='raw')
             lltype.free(mtime, flavor='raw')
+
+if not _WIN32:
+    TMSP = lltype.Ptr(TMS)
+    os_times = external('times', [TMSP], CLOCK_T)
+
+    # Here is a random extra platform parameter which is important.
+    # Strictly speaking, this should probably be retrieved at runtime, not
+    # at translation time.
+    CLOCK_TICKS_PER_SECOND = float(os.sysconf('SC_CLK_TCK'))
+else:
+    GetCurrentProcess = external(
+        'GetCurrentProcess', [],
+        rwin32.HANDLE, calling_conv='win')
+    GetProcessTimes = external(
+        'GetProcessTimes', [
+            rwin32.HANDLE,
+            lltype.Ptr(rwin32.FILETIME), lltype.Ptr(rwin32.FILETIME),
+            lltype.Ptr(rwin32.FILETIME), lltype.Ptr(rwin32.FILETIME)],
+        rwin32.BOOL, calling_conv='win')
+
+ at register_replacement_for(os.times, sandboxed_name='ll_os.ll_times')
+def times():
+    if not _WIN32:
+        l_tmsbuf = lltype.malloc(TMSP.TO, flavor='raw')
+        try:
+            result = os_times(l_tmsbuf)
+            result = rffi.cast(lltype.Signed, result)
+            if result == -1:
+                raise OSError(get_errno(), "times failed")
+            return (
+                rffi.cast(lltype.Signed, l_tmsbuf.c_tms_utime)
+                                               / CLOCK_TICKS_PER_SECOND,
+                rffi.cast(lltype.Signed, l_tmsbuf.c_tms_stime)
+                                               / CLOCK_TICKS_PER_SECOND,
+                rffi.cast(lltype.Signed, l_tmsbuf.c_tms_cutime)
+                                               / CLOCK_TICKS_PER_SECOND,
+                rffi.cast(lltype.Signed, l_tmsbuf.c_tms_cstime)
+                                               / CLOCK_TICKS_PER_SECOND,
+                result / CLOCK_TICKS_PER_SECOND)
+        finally:
+            lltype.free(l_tmsbuf, flavor='raw')
+    else:
+        pcreate = lltype.malloc(rwin32.FILETIME, flavor='raw')
+        pexit   = lltype.malloc(rwin32.FILETIME, flavor='raw')
+        pkernel = lltype.malloc(rwin32.FILETIME, flavor='raw')
+        puser   = lltype.malloc(rwin32.FILETIME, flavor='raw')
+        try:
+            hProc = GetCurrentProcess()
+            GetProcessTimes(hProc, pcreate, pexit, pkernel, puser)
+            # The fields of a FILETIME structure are the hi and lo parts
+            # of a 64-bit value expressed in 100 nanosecond units
+            # (of course).
+            return (
+                rffi.cast(lltype.Signed, pkernel.c_dwHighDateTime) * 429.4967296 +
+                rffi.cast(lltype.Signed, pkernel.c_dwLowDateTime) * 1E-7,
+                rffi.cast(lltype.Signed, puser.c_dwHighDateTime) * 429.4967296 +
+                rffi.cast(lltype.Signed, puser.c_dwLowDateTime) * 1E-7,
+                0, 0, 0)
+        finally:
+            lltype.free(puser,   flavor='raw')
+            lltype.free(pkernel, flavor='raw')
+            lltype.free(pexit,   flavor='raw')
+            lltype.free(pcreate, flavor='raw')
+
diff --git a/rpython/rtyper/module/ll_os.py b/rpython/rtyper/module/ll_os.py
--- a/rpython/rtyper/module/ll_os.py
+++ b/rpython/rtyper/module/ll_os.py
@@ -247,75 +247,6 @@
         return extdef([int], int, llimpl=c_func_llimpl,
                       export_name='ll_os.ll_os_' + name)
 
-    @registering(os.times)
-    def register_os_times(self):
-        if sys.platform.startswith('win'):
-            from rpython.rlib import rwin32
-            GetCurrentProcess = self.llexternal('GetCurrentProcess', [],
-                                                rwin32.HANDLE)
-            GetProcessTimes = self.llexternal('GetProcessTimes',
-                                              [rwin32.HANDLE,
-                                               lltype.Ptr(rwin32.FILETIME),
-                                               lltype.Ptr(rwin32.FILETIME),
-                                               lltype.Ptr(rwin32.FILETIME),
-                                               lltype.Ptr(rwin32.FILETIME)],
-                                              rwin32.BOOL)
-
-            def times_lltypeimpl():
-                pcreate = lltype.malloc(rwin32.FILETIME, flavor='raw')
-                pexit   = lltype.malloc(rwin32.FILETIME, flavor='raw')
-                pkernel = lltype.malloc(rwin32.FILETIME, flavor='raw')
-                puser   = lltype.malloc(rwin32.FILETIME, flavor='raw')
-                hProc = GetCurrentProcess()
-                GetProcessTimes(hProc, pcreate, pexit, pkernel, puser)
-                # The fields of a FILETIME structure are the hi and lo parts
-                # of a 64-bit value expressed in 100 nanosecond units
-                # (of course).
-                result = (rffi.cast(lltype.Signed, pkernel.c_dwHighDateTime) * 429.4967296 +
-                          rffi.cast(lltype.Signed, pkernel.c_dwLowDateTime) * 1E-7,
-                          rffi.cast(lltype.Signed, puser.c_dwHighDateTime) * 429.4967296 +
-                          rffi.cast(lltype.Signed, puser.c_dwLowDateTime) * 1E-7,
-                          0, 0, 0)
-                lltype.free(puser,   flavor='raw')
-                lltype.free(pkernel, flavor='raw')
-                lltype.free(pexit,   flavor='raw')
-                lltype.free(pcreate, flavor='raw')
-                return result
-            self.register(os.times, [], (float, float, float, float, float),
-                          "ll_os.ll_times", llimpl=times_lltypeimpl)
-            return
-
-        TMSP = lltype.Ptr(self.TMS)
-        os_times = self.llexternal('times', [TMSP], self.CLOCK_T)
-
-        # Here is a random extra platform parameter which is important.
-        # Strictly speaking, this should probably be retrieved at runtime, not
-        # at translation time.
-        CLOCK_TICKS_PER_SECOND = float(os.sysconf('SC_CLK_TCK'))
-
-        def times_lltypeimpl():
-            l_tmsbuf = lltype.malloc(TMSP.TO, flavor='raw')
-            try:
-                result = os_times(l_tmsbuf)
-                result = rffi.cast(lltype.Signed, result)
-                if result == -1:
-                    raise OSError(rposix.get_errno(), "times failed")
-                return (
-                    rffi.cast(lltype.Signed, l_tmsbuf.c_tms_utime)
-                                                   / CLOCK_TICKS_PER_SECOND,
-                    rffi.cast(lltype.Signed, l_tmsbuf.c_tms_stime)
-                                                   / CLOCK_TICKS_PER_SECOND,
-                    rffi.cast(lltype.Signed, l_tmsbuf.c_tms_cutime)
-                                                   / CLOCK_TICKS_PER_SECOND,
-                    rffi.cast(lltype.Signed, l_tmsbuf.c_tms_cstime)
-                                                   / CLOCK_TICKS_PER_SECOND,
-                    result / CLOCK_TICKS_PER_SECOND)
-            finally:
-                lltype.free(l_tmsbuf, flavor='raw')
-        self.register(os.times, [], (float, float, float, float, float),
-                      "ll_os.ll_times", llimpl=times_lltypeimpl)
-
-
     @registering_if(os, 'setsid')
     def register_os_setsid(self):
         os_setsid = self.llexternal('setsid', [], rffi.PID_T)

From noreply at buildbot.pypy.org  Fri Nov  7 14:52:20 2014
From: noreply at buildbot.pypy.org (amauryfa)
Date: Fri,  7 Nov 2014 14:52:20 +0100 (CET)
Subject: [pypy-commit] pypy more-rposix: os.getsid(), os.setsid()
Message-ID: <20141107135220.A81551C030E@cobra.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc 
Branch: more-rposix
Changeset: r74365:9ffac25cc23f
Date: 2014-11-07 00:26 +0100
http://bitbucket.org/pypy/pypy/changeset/9ffac25cc23f/

Log:	os.getsid(), os.setsid()

diff --git a/rpython/rlib/rposix.py b/rpython/rlib/rposix.py
--- a/rpython/rlib/rposix.py
+++ b/rpython/rlib/rposix.py
@@ -416,7 +416,7 @@
     return rffi.charp2str(result)
 
 
-#_______________________________________________________________
+#___________________________________________________________________
 
 UTIMBUFP = lltype.Ptr(UTIMBUF)
 c_utime = external('utime', [rffi.CCHARP, UTIMBUFP], rffi.INT)
@@ -591,3 +591,23 @@
             lltype.free(pexit,   flavor='raw')
             lltype.free(pcreate, flavor='raw')
 
+#___________________________________________________________________
+
+ at specialize.arg(0)
+def handle_posix_error(name, result):
+    if result < 0:
+        raise OSError(get_errno(), '%s failed' % name)
+    return intmask(result)
+
+c_setsid = external('setsid', [], rffi.PID_T)
+c_getsid = external('getsid', [rffi.PID_T], rffi.PID_T)
+
+ at register_replacement_for(getattr(os, 'setsid', None),
+                          sandboxed_name='ll_os.ll_os_setsid')
+def setsid():
+    return handle_posix_error('os_setsid', c_setsid())
+
+ at register_replacement_for(getattr(os, 'getsid', None),
+                          sandboxed_name='ll_os.ll_os_getsid')
+def getsid(pid):
+    return handle_posix_error('os_setsid', c_getsid(pid))
diff --git a/rpython/rtyper/module/ll_os.py b/rpython/rtyper/module/ll_os.py
--- a/rpython/rtyper/module/ll_os.py
+++ b/rpython/rtyper/module/ll_os.py
@@ -247,18 +247,6 @@
         return extdef([int], int, llimpl=c_func_llimpl,
                       export_name='ll_os.ll_os_' + name)
 
-    @registering_if(os, 'setsid')
-    def register_os_setsid(self):
-        os_setsid = self.llexternal('setsid', [], rffi.PID_T)
-        def setsid_llimpl():
-            result = rffi.cast(lltype.Signed, os_setsid())
-            if result == -1:
-                raise OSError(rposix.get_errno(), "os_setsid failed")
-            return result
-
-        return extdef([], int, export_name="ll_os.ll_os_setsid",
-                      llimpl=setsid_llimpl)
-
     @registering_if(os, 'chroot')
     def register_os_chroot(self):
         os_chroot = self.llexternal('chroot', [rffi.CCHARP], rffi.INT)
@@ -552,14 +540,6 @@
     def register_os_setregid(self):
         return self.extdef_for_os_function_accepting_2int('setregid')
 
-    @registering_if(os, 'getsid')
-    def register_os_getsid(self):
-        return self.extdef_for_os_function_int_to_int('getsid')
-
-    @registering_if(os, 'setsid')
-    def register_os_setsid(self):
-        return self.extdef_for_os_function_returning_int('setsid')
-
     @registering_if(os, 'getresuid')
     def register_os_getresuid(self):
         c_getresuid = self.llexternal('getresuid', [rffi.INTP] * 3, rffi.INT)

From noreply at buildbot.pypy.org  Fri Nov  7 14:52:21 2014
From: noreply at buildbot.pypy.org (amauryfa)
Date: Fri,  7 Nov 2014 14:52:21 +0100 (CET)
Subject: [pypy-commit] pypy more-rposix: Some simplifications
Message-ID: <20141107135221.C5A391C030E@cobra.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc 
Branch: more-rposix
Changeset: r74366:42829da0669a
Date: 2014-11-07 00:44 +0100
http://bitbucket.org/pypy/pypy/changeset/42829da0669a/

Log:	Some simplifications

diff --git a/rpython/rlib/rposix.py b/rpython/rlib/rposix.py
--- a/rpython/rlib/rposix.py
+++ b/rpython/rlib/rposix.py
@@ -326,34 +326,37 @@
 # They usually check the return value and raise an (RPython) OSError
 # with errno.
 
- at register_replacement_for(os.dup, sandboxed_name='ll_os.ll_os_dup')
+def replace_os_function(name):
+    return register_replacement_for(
+        getattr(os, name, None),
+        sandboxed_name='ll_os.ll_os_%s' % name)
+
+ at specialize.arg(0)
+def handle_posix_error(name, result):
+    if result < 0:
+        raise OSError(get_errno(), '%s failed' % name)
+    return intmask(result)
+
+ at replace_os_function('dup')
 def dup(fd):
     validate_fd(fd)
-    newfd = c_dup(fd)
-    if newfd < 0:
-        raise OSError(get_errno(), "dup failed")
-    return intmask(newfd)
+    return handle_posix_error('dup', c_dup(fd))
 
- at register_replacement_for(os.dup2, sandboxed_name='ll_os.ll_os_dup2')
+ at replace_os_function('dup2')
 def dup2(fd, newfd):
     validate_fd(fd)
-    error = c_dup2(fd, newfd)
-    if error < 0:
-        raise OSError(get_errno(), "dup2 failed")
+    handle_posix_error('dup2', c_dup2(fd, newfd))
 
- at register_replacement_for(os.open, sandboxed_name='ll_os.ll_os_open')
+ at replace_os_function('open')
 @specialize.argtype(0)
 def open(path, flags, mode):
     if _prefer_unicode(path):
         fd = c_wopen(_as_unicode0(path), flags, mode)
     else:
         fd = c_open(_as_bytes0(path), flags, mode)
-    if fd < 0:
-        raise OSError(get_errno(), "open failed")
-    return intmask(fd)
+    return handle_posix_error('open', fd)
         
- at register_replacement_for(getattr(os, 'execv', None),
-                          sandboxed_name='ll_os.ll_os_execv')
+ at replace_os_function('execv')
 def execv(path, args):
     rstring.check_str0(path)
     # This list conversion already takes care of NUL bytes.
@@ -362,8 +365,7 @@
     rffi.free_charpp(l_args)
     raise OSError(get_errno(), "execv failed")
 
- at register_replacement_for(getattr(os, 'execve', None),
-                          sandboxed_name='ll_os.ll_os_execve')
+ at replace_os_function('execve')
 def execve(path, args, env):
     envstrs = []
     for item in env.iteritems():
@@ -380,19 +382,15 @@
     rffi.free_charpp(l_args)
     raise OSError(get_errno(), "execve failed")
 
- at register_replacement_for(getattr(os, 'spawnv', None),
-                          sandboxed_name='ll_os.ll_os_spawnv')
+ at replace_os_function('spawnv')
 def spawnv(mode, path, args):
     rstring.check_str0(path)
     l_args = rffi.ll_liststr2charpp(args)
     childpid = c_spawnv(mode, path, l_args)
     rffi.free_charpp(l_args)
-    if childpid < 0:
-        raise OSError(get_errno(), "os_spawnv failed")
-    return intmask(childpid)
+    return handle_posix_error('spawnv', childpid)
 
- at register_replacement_for(getattr(os, 'spawnve', None),
-                          sandboxed_name='ll_os.ll_os_spawnve')
+ at replace_os_function('spawnve')
 def spawnve(mode, path, args, env):
     envstrs = []
     for item in env.iteritems():
@@ -403,12 +401,9 @@
     childpid = c_spawnve(mode, path, l_args, l_env)
     rffi.free_charpp(l_env)
     rffi.free_charpp(l_args)
-    if childpid == -1:
-        raise OSError(get_errno(), "os_spawnve failed")
-    return intmask(childpid)
+    return handle_posix_error('spawnve', childpid)
 
- at register_replacement_for(getattr(os, 'getlogin', None),
-                          sandboxed_name='ll_os.ll_os_getlogin')
+ at replace_os_function('getlogin')
 def getlogin():
     result = c_getlogin()
     if not result:
@@ -456,8 +451,7 @@
         calling_conv='win')
 
 
- at register_replacement_for(getattr(os, 'utime', None),
-                          sandboxed_name='ll_os.ll_os_utime')
+ at replace_os_function('utime')
 @specialize.argtype(0, 1)
 def utime(path, times):
     if not _WIN32:
@@ -483,8 +477,7 @@
                 l_utimbuf.c_modtime = rffi.r_time_t(modtime)
                 error = c_utime(path, l_utimbuf)
                 lltype.free(l_utimbuf, flavor='raw')
-        if error < 0:
-            raise OSError(get_errno(), "os_utime failed")
+        handle_posix_error('utime', error)
     else:  # _WIN32 case
         from rpython.rlib.rwin32file import make_win32_traits
         if _prefer_unicode(path):
@@ -547,7 +540,7 @@
             lltype.Ptr(rwin32.FILETIME), lltype.Ptr(rwin32.FILETIME)],
         rwin32.BOOL, calling_conv='win')
 
- at register_replacement_for(os.times, sandboxed_name='ll_os.ll_times')
+ at replace_os_function('times')
 def times():
     if not _WIN32:
         l_tmsbuf = lltype.malloc(TMSP.TO, flavor='raw')
@@ -593,21 +586,13 @@
 
 #___________________________________________________________________
 
- at specialize.arg(0)
-def handle_posix_error(name, result):
-    if result < 0:
-        raise OSError(get_errno(), '%s failed' % name)
-    return intmask(result)
-
 c_setsid = external('setsid', [], rffi.PID_T)
 c_getsid = external('getsid', [rffi.PID_T], rffi.PID_T)
 
- at register_replacement_for(getattr(os, 'setsid', None),
-                          sandboxed_name='ll_os.ll_os_setsid')
+ at replace_os_function('setsid')
 def setsid():
     return handle_posix_error('os_setsid', c_setsid())
 
- at register_replacement_for(getattr(os, 'getsid', None),
-                          sandboxed_name='ll_os.ll_os_getsid')
+ at replace_os_function('getsid')
 def getsid(pid):
     return handle_posix_error('os_setsid', c_getsid(pid))

From noreply at buildbot.pypy.org  Fri Nov  7 14:52:23 2014
From: noreply at buildbot.pypy.org (amauryfa)
Date: Fri,  7 Nov 2014 14:52:23 +0100 (CET)
Subject: [pypy-commit] pypy more-rposix: Move uid/gid functions
Message-ID: <20141107135223.0B2361C030E@cobra.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc 
Branch: more-rposix
Changeset: r74367:048ec065ff5a
Date: 2014-11-07 00:54 +0100
http://bitbucket.org/pypy/pypy/changeset/048ec065ff5a/

Log:	Move uid/gid functions

diff --git a/rpython/rlib/rposix.py b/rpython/rlib/rposix.py
--- a/rpython/rlib/rposix.py
+++ b/rpython/rlib/rposix.py
@@ -588,11 +588,51 @@
 
 c_setsid = external('setsid', [], rffi.PID_T)
 c_getsid = external('getsid', [rffi.PID_T], rffi.PID_T)
+c_getuid = external('getuid', [], rffi.INT)
+c_geteuid = external('geteuid', [], rffi.INT)
+c_setuid = external('setuid', [rffi.INT], rffi.INT)
+c_seteuid = external('seteuid', [rffi.INT], rffi.INT)
+c_getgid = external('getgid', [], rffi.INT)
+c_getegid = external('getegid', [], rffi.INT)
+c_setgid = external('setgid', [rffi.INT], rffi.INT)
+c_setegid = external('setegid', [rffi.INT], rffi.INT)
 
 @replace_os_function('setsid')
 def setsid():
-    return handle_posix_error('os_setsid', c_setsid())
+    return handle_posix_error('setsid', c_setsid())
 
 @replace_os_function('getsid')
 def getsid(pid):
-    return handle_posix_error('os_setsid', c_getsid(pid))
+    return handle_posix_error('getsid', c_getsid(pid))
+
+ at replace_os_function('getuid')
+def getuid():
+    return handle_posix_error('getuid', c_getuid())
+
+ at replace_os_function('geteuid')
+def geteuid():
+    return handle_posix_error('geteuid', c_geteuid())
+
+ at replace_os_function('setuid')
+def setuid(uid):
+    return handle_posix_error('setuid', c_setuid(uid))
+
+ at replace_os_function('seteuid')
+def seteuid(uid):
+    return handle_posix_error('seteuid', c_seteuid(uid))
+
+ at replace_os_function('getgid')
+def getgid():
+    return handle_posix_error('getgid', c_getgid())
+
+ at replace_os_function('getegid')
+def getegid():
+    return handle_posix_error('getegid', c_getegid())
+
+ at replace_os_function('setgid')
+def setgid(gid):
+    return handle_posix_error('setgid', c_setgid(gid))
+
+ at replace_os_function('setegid')
+def setegid(gid):
+    return handle_posix_error('setegid', c_setegid(gid))
diff --git a/rpython/rtyper/module/ll_os.py b/rpython/rtyper/module/ll_os.py
--- a/rpython/rtyper/module/ll_os.py
+++ b/rpython/rtyper/module/ll_os.py
@@ -365,42 +365,10 @@
         return extdef([int], SomeString(can_be_None=True),
                       "ll_os.ll_confstr", llimpl=confstr_llimpl)
 
-    @registering_if(os, 'getuid')
-    def register_os_getuid(self):
-        return self.extdef_for_os_function_returning_int('getuid')
-
-    @registering_if(os, 'geteuid')
-    def register_os_geteuid(self):
-        return self.extdef_for_os_function_returning_int('geteuid')
-
-    @registering_if(os, 'setuid')
-    def register_os_setuid(self):
-        return self.extdef_for_os_function_accepting_int('setuid')
-
-    @registering_if(os, 'seteuid')
-    def register_os_seteuid(self):
-        return self.extdef_for_os_function_accepting_int('seteuid')
-
-    @registering_if(os, 'setgid')
-    def register_os_setgid(self):
-        return self.extdef_for_os_function_accepting_int('setgid')
-
-    @registering_if(os, 'setegid')
-    def register_os_setegid(self):
-        return self.extdef_for_os_function_accepting_int('setegid')
-
     @registering_if(os, 'getpid')
     def register_os_getpid(self):
         return self.extdef_for_os_function_returning_int('getpid', releasegil=False)
 
-    @registering_if(os, 'getgid')
-    def register_os_getgid(self):
-        return self.extdef_for_os_function_returning_int('getgid')
-
-    @registering_if(os, 'getegid')
-    def register_os_getegid(self):
-        return self.extdef_for_os_function_returning_int('getegid')
-
     @registering_if(os, 'getgroups')
     def register_os_getgroups(self):
         GP = rffi.CArrayPtr(rffi.PID_T)

From noreply at buildbot.pypy.org  Fri Nov  7 14:52:24 2014
From: noreply at buildbot.pypy.org (amauryfa)
Date: Fri,  7 Nov 2014 14:52:24 +0100 (CET)
Subject: [pypy-commit] pypy more-rposix: Move more functions: os.setresuid()
	&co
Message-ID: <20141107135224.415E01C030E@cobra.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc 
Branch: more-rposix
Changeset: r74368:f5baa8e21fdd
Date: 2014-11-07 08:42 +0100
http://bitbucket.org/pypy/pypy/changeset/f5baa8e21fdd/

Log:	Move more functions: os.setresuid() &co

diff --git a/rpython/rlib/rposix.py b/rpython/rlib/rposix.py
--- a/rpython/rlib/rposix.py
+++ b/rpython/rlib/rposix.py
@@ -615,11 +615,11 @@
 
 @replace_os_function('setuid')
 def setuid(uid):
-    return handle_posix_error('setuid', c_setuid(uid))
+    handle_posix_error('setuid', c_setuid(uid))
 
 @replace_os_function('seteuid')
 def seteuid(uid):
-    return handle_posix_error('seteuid', c_seteuid(uid))
+    handle_posix_error('seteuid', c_seteuid(uid))
 
 @replace_os_function('getgid')
 def getgid():
@@ -631,8 +631,57 @@
 
 @replace_os_function('setgid')
 def setgid(gid):
-    return handle_posix_error('setgid', c_setgid(gid))
+    handle_posix_error('setgid', c_setgid(gid))
 
 @replace_os_function('setegid')
 def setegid(gid):
-    return handle_posix_error('setegid', c_setegid(gid))
+    handle_posix_error('setegid', c_setegid(gid))
+
+c_setreuid = external('setreuid', [rffi.INT, rffi.INT], rffi.INT)
+c_setregid = external('setregid', [rffi.INT, rffi.INT], rffi.INT)
+
+ at replace_os_function('setreuid')
+def setreuid(ruid, euid):
+    handle_posix_error('setreuid', c_setreuid(ruid, euid))
+
+ at replace_os_function('setregid')
+def setregid(rgid, egid):
+    handle_posix_error('setregid', c_setregid(rgid, egid))
+
+c_getresuid = external('getresuid', [rffi.INTP] * 3, rffi.INT)
+c_getresgid = external('getresgid', [rffi.INTP] * 3, rffi.INT)
+c_setresuid = external('setresuid', [rffi.INT] * 3, rffi.INT)
+c_setresgid = external('setresgid', [rffi.INT] * 3, rffi.INT)
+
+ at replace_os_function('getresuid')
+def getresuid():
+    out = lltype.malloc(rffi.INTP.TO, 3, flavor='raw')
+    try:
+        handle_posix_error('getresuid',
+                           c_getresuid(rffi.ptradd(out, 0),
+                                       rffi.ptradd(out, 1),
+                                       rffi.ptradd(out, 2)))
+        return (intmask(out[0]), intmask(out[1]), intmask(out[2]))
+    finally:
+        lltype.free(out, flavor='raw')
+
+ at replace_os_function('getresgid')
+def getresgid():
+    out = lltype.malloc(rffi.INTP.TO, 3, flavor='raw')
+    try:
+        handle_posix_error('getresgid',
+                           c_getresgid(rffi.ptradd(out, 0),
+                                       rffi.ptradd(out, 1),
+                                       rffi.ptradd(out, 2)))
+        return (intmask(out[0]), intmask(out[1]), intmask(out[2]))
+    finally:
+        lltype.free(out, flavor='raw')
+
+ at replace_os_function('setresuid')
+def setresuid(ruid, euid, suid):
+    handle_posix_error('setresuid', c_setresuid(ruid, euid, suid))
+
+ at replace_os_function('setresgid')
+def setresgid(rgid, egid, sgid):
+    handle_posix_error('setresgid', c_setresgid(rgid, egid, sgid))
+
diff --git a/rpython/rtyper/module/ll_os.py b/rpython/rtyper/module/ll_os.py
--- a/rpython/rtyper/module/ll_os.py
+++ b/rpython/rtyper/module/ll_os.py
@@ -500,84 +500,6 @@
     def register_os_setpgid(self):
         return self.extdef_for_os_function_accepting_2int('setpgid')
 
-    @registering_if(os, 'setreuid')
-    def register_os_setreuid(self):
-        return self.extdef_for_os_function_accepting_2int('setreuid')
-
-    @registering_if(os, 'setregid')
-    def register_os_setregid(self):
-        return self.extdef_for_os_function_accepting_2int('setregid')
-
-    @registering_if(os, 'getresuid')
-    def register_os_getresuid(self):
-        c_getresuid = self.llexternal('getresuid', [rffi.INTP] * 3, rffi.INT)
-
-        def c_getresuid_llimpl():
-            out = lltype.malloc(rffi.INTP.TO, 3, flavor='raw')
-            try:
-                res = c_getresuid(rffi.ptradd(out, 0),
-                                  rffi.ptradd(out, 1),
-                                  rffi.ptradd(out, 2))
-                res = rffi.cast(lltype.Signed, res)
-                if res == -1:
-                    raise OSError(rposix.get_errno(), "getresuid failed")
-                return (rffi.cast(lltype.Signed, out[0]),
-                        rffi.cast(lltype.Signed, out[1]),
-                        rffi.cast(lltype.Signed, out[2]))
-            finally:
-                lltype.free(out, flavor='raw')
-
-        return extdef([], (int, int, int), llimpl=c_getresuid_llimpl,
-                      export_name='ll_os.ll_os_getresuid')
-
-    @registering_if(os, 'getresgid')
-    def register_os_getresgid(self):
-        c_getresgid = self.llexternal('getresgid', [rffi.INTP] * 3, rffi.INT)
-
-        def c_getresgid_llimpl():
-            out = lltype.malloc(rffi.INTP.TO, 3, flavor='raw')
-            try:
-                res = c_getresgid(rffi.ptradd(out, 0),
-                                  rffi.ptradd(out, 1),
-                                  rffi.ptradd(out, 2))
-                res = rffi.cast(lltype.Signed, res)
-                if res == -1:
-                    raise OSError(rposix.get_errno(), "getresgid failed")
-                return (rffi.cast(lltype.Signed, out[0]),
-                        rffi.cast(lltype.Signed, out[1]),
-                        rffi.cast(lltype.Signed, out[2]))
-            finally:
-                lltype.free(out, flavor='raw')
-
-        return extdef([], (int, int, int), llimpl=c_getresgid_llimpl,
-                      export_name='ll_os.ll_os_getresgid')
-
-    @registering_if(os, 'setresuid')
-    def register_os_setresuid(self):
-        c_setresuid = self.llexternal('setresuid', [rffi.INT] * 3, rffi.INT)
-
-        def c_setresuid_llimpl(ruid, euid, suid):
-            res = c_setresuid(ruid, euid, suid)
-            res = rffi.cast(lltype.Signed, res)
-            if res == -1:
-                raise OSError(rposix.get_errno(), "setresuid failed")
-
-        return extdef([int, int, int], None, llimpl=c_setresuid_llimpl,
-                      export_name='ll_os.ll_os_setresuid')
-
-    @registering_if(os, 'setresgid')
-    def register_os_setresgid(self):
-        c_setresgid = self.llexternal('setresgid', [rffi.INT] * 3, rffi.INT)
-
-        def c_setresgid_llimpl(rgid, egid, sgid):
-            res = c_setresgid(rgid, egid, sgid)
-            res = rffi.cast(lltype.Signed, res)
-            if res == -1:
-                raise OSError(rposix.get_errno(), "setresgid failed")
-
-        return extdef([int, int, int], None, llimpl=c_setresgid_llimpl,
-                      export_name='ll_os.ll_os_setresgid')
-
     @registering_if(os, 'getloadavg')
     def register_os_getloadavg(self):
         AP = rffi.CArrayPtr(lltype.Float)

From noreply at buildbot.pypy.org  Fri Nov  7 14:52:25 2014
From: noreply at buildbot.pypy.org (amauryfa)
Date: Fri,  7 Nov 2014 14:52:25 +0100 (CET)
Subject: [pypy-commit] pypy more-rposix: Port *pid() and *pgrp() functions
Message-ID: <20141107135225.634D91C030E@cobra.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc 
Branch: more-rposix
Changeset: r74369:ece292fe78e1
Date: 2014-11-07 09:12 +0100
http://bitbucket.org/pypy/pypy/changeset/ece292fe78e1/

Log:	Port *pid() and *pgrp() functions

diff --git a/rpython/rlib/rposix.py b/rpython/rlib/rposix.py
--- a/rpython/rlib/rposix.py
+++ b/rpython/rlib/rposix.py
@@ -159,6 +159,9 @@
                            ('tms_cutime', rffi.INT),
                            ('tms_cstime', rffi.INT)])
 
+    GETPGRP_HAVE_ARG = rffi_platform.Has("getpgrp(0)")
+    SETPGRP_HAVE_ARG = rffi_platform.Has("setpgrp(0, 0)")
+
 config = rffi_platform.configure(CConfig)
 globals().update(config)
 
@@ -586,8 +589,105 @@
 
 #___________________________________________________________________
 
+c_getpid = external('getpid', [], rffi.PID_T, releasegil=False)
+c_getppid = external('getppid', [], rffi.PID_T, releasegil=False)
 c_setsid = external('setsid', [], rffi.PID_T)
 c_getsid = external('getsid', [rffi.PID_T], rffi.PID_T)
+
+ at replace_os_function('getpid')
+def getpid():
+    return intmask(c_getpid())
+
+ at replace_os_function('getppid')
+def getppid():
+    return intmask(c_getppid())
+
+ at replace_os_function('setsid')
+def setsid():
+    return handle_posix_error('setsid', c_setsid())
+
+ at replace_os_function('getsid')
+def getsid(pid):
+    return handle_posix_error('getsid', c_getsid(pid))
+
+c_getpgid = external('getpid', [rffi.PID_T], rffi.PID_T)
+c_setpgid = external('setpid', [rffi.PID_T, rffi.PID_T], rffi.INT)
+
+ at replace_os_function('getpgid')
+def getpgid(pid):
+    return handle_posix_error('getpgid', c_getpgid(pid))
+
+ at replace_os_function('setpgid')
+def setpgid(pid, gid):
+    handle_posix_error('setpgid', c_setpgid(pid, gid))
+
+PID_GROUPS_T = rffi.CArrayPtr(rffi.PID_T)
+c_getgroups = external('getgroups', [rffi.INT, PID_GROUPS_T], rffi.INT)
+c_setgroups = external('setgroups', [rffi.SIZE_T, PID_GROUPS_T], rffi.INT)
+c_initgroups = external('initgroups', [rffi.CCHARP, rffi.PID_T], rffi.INT)
+
+ at replace_os_function('getgroups')
+def getgroups():
+    n = handle_posix_error('getgroups',
+                           c_getgroups(0, lltype.nullptr(PID_GROUPS_T.TO)))
+    groups = lltype.malloc(PID_GROUPS_T.TO, n, flavor='raw')
+    try:
+        n = handle_posix_error('getgroups', c_getgroups(n, groups))
+        return [intmask(groups[i]) for i in range(n)]
+    finally:
+        lltype.free(groups, flavor='raw')
+
+ at replace_os_function('setgroups')
+def setgroups(gids):
+    n = len(gids)
+    groups = lltype.malloc(PID_GROUPS_T.TO, n, flavor='raw')
+    try:
+        for i in range(n):
+            groups[i] = rffi.cast(rffi.PID_T, gids[i])
+        handle_posix_error('setgroups', c_setgroups(n, groups))
+    finally:
+        lltype.free(groups, flavor='raw')
+
+ at replace_os_function('initgroups')
+def initgroups(user, group):
+    handle_posix_error('initgroups', c_initgroups(user, group))
+
+if GETPGRP_HAVE_ARG:
+    c_getpgrp = external('getpgrp', [rffi.INT], rffi.INT)
+else:
+    c_getpgrp = external('getpgrp', [], rffi.INT)
+if SETPGRP_HAVE_ARG:
+    c_setpgrp = external('setpgrp', [rffi.INT, rffi.INT], rffi.INT)
+else:
+    c_setpgrp = external('setpgrp', [], rffi.INT)
+
+ at replace_os_function('getpgrp')
+def getpgrp():
+    if GETPGRP_HAVE_ARG:
+        return handle_posix_error('getpgrp', c_getpgrp(0))
+    else:
+        return handle_posix_error('getpgrp', c_getpgrp())
+
+ at replace_os_function('setpgrp')
+def setpgrp():
+    if SETPGRP_HAVE_ARG:
+        return handle_posix_error('setpgrp', c_setpgrp(0, 0))
+    else:
+        return handle_posix_error('setpgrp', c_setpgrp())
+
+c_tcgetpgrp = external('tcgetpgrp', [rffi.INT], rffi.PID_T)
+c_tcsetpgrp = external('tcsetpgrp', [rffi.INT, rffi.PID_T], rffi.INT)
+
+ at replace_os_function('tcgetpgrp')
+def tcgetpgrp(fd):
+    return handle_posix_error('tcgetpgrp', c_tcgetpgrp(fd))
+
+ at replace_os_function('tcsetpgrp')
+def tcsetpgrp(fd, pgrp):
+    return handle_posix_error('tcsetpgrp', c_tcsetpgrp(fd, pgrp))
+
+#___________________________________________________________________
+
 c_getuid = external('getuid', [], rffi.INT)
 c_geteuid = external('geteuid', [], rffi.INT)
 c_setuid = external('setuid', [rffi.INT], rffi.INT)
@@ -597,14 +697,6 @@
 c_setgid = external('setgid', [rffi.INT], rffi.INT)
 c_setegid = external('setegid', [rffi.INT], rffi.INT)
 
- at replace_os_function('setsid')
-def setsid():
-    return handle_posix_error('setsid', c_setsid())
-
- at replace_os_function('getsid')
-def getsid(pid):
-    return handle_posix_error('getsid', c_getsid(pid))
-
 @replace_os_function('getuid')
 def getuid():
     return handle_posix_error('getuid', c_getuid())
diff --git a/rpython/rtyper/module/ll_os.py b/rpython/rtyper/module/ll_os.py
--- a/rpython/rtyper/module/ll_os.py
+++ b/rpython/rtyper/module/ll_os.py
@@ -149,18 +149,6 @@
         if not _WIN32:
             assert self.OFF_T_SIZE == rffi.sizeof(rffi.LONGLONG)
 
-        if hasattr(os, 'getpgrp'):
-            self.GETPGRP_HAVE_ARG = platform.checkcompiles(
-                "getpgrp(0)",
-                '#include ',
-                [])
-
-        if hasattr(os, 'setpgrp'):
-            self.SETPGRP_HAVE_ARG = platform.checkcompiles(
-                "setpgrp(0,0)",
-                '#include ',
-                [])
-
         # we need an indirection via c functions to get macro calls working on llvm XXX still?
         if hasattr(os, 'WCOREDUMP'):
             decl_snippet = """
@@ -365,141 +353,6 @@
         return extdef([int], SomeString(can_be_None=True),
                       "ll_os.ll_confstr", llimpl=confstr_llimpl)
 
-    @registering_if(os, 'getpid')
-    def register_os_getpid(self):
-        return self.extdef_for_os_function_returning_int('getpid', releasegil=False)
-
-    @registering_if(os, 'getgroups')
-    def register_os_getgroups(self):
-        GP = rffi.CArrayPtr(rffi.PID_T)
-        c_getgroups = self.llexternal('getgroups', [rffi.INT, GP], rffi.INT)
-
-        def getgroups_llimpl():
-            n = c_getgroups(0, lltype.nullptr(GP.TO))
-            if n >= 0:
-                groups = lltype.malloc(GP.TO, n, flavor='raw')
-                try:
-                    n = c_getgroups(n, groups)
-                    result = [rffi.cast(lltype.Signed, groups[i])
-                              for i in range(n)]
-                finally:
-                    lltype.free(groups, flavor='raw')
-                if n >= 0:
-                    return result
-            raise OSError(rposix.get_errno(), "os_getgroups failed")
-
-        return extdef([], [int], llimpl=getgroups_llimpl,
-                      export_name="ll_os.ll_getgroups")
-
-    @registering_if(os, 'setgroups')
-    def register_os_setgroups(self):
-        GP = rffi.CArrayPtr(rffi.PID_T)
-        c_setgroups = self.llexternal('setgroups', [rffi.SIZE_T, GP], rffi.INT)
-
-        def setgroups_llimpl(list):
-            n = len(list)
-            groups = lltype.malloc(GP.TO, n, flavor='raw')
-            try:
-                for i in range(n):
-                    groups[i] = rffi.cast(rffi.PID_T, list[i])
-                n = c_setgroups(rffi.cast(rffi.SIZE_T, n), groups)
-            finally:
-                lltype.free(groups, flavor='raw')
-            if n != 0:
-                raise OSError(rposix.get_errno(), "os_setgroups failed")
-
-        return extdef([[int]], None, llimpl=setgroups_llimpl,
-                      export_name="ll_os.ll_setgroups")
-
-    @registering_if(os, 'initgroups')
-    def register_os_initgroups(self):
-        c_initgroups = self.llexternal('initgroups',
-                                       [rffi.CCHARP, rffi.PID_T], rffi.INT)
-
-        def initgroups_llimpl(user, group):
-            n = c_initgroups(user, rffi.cast(rffi.PID_T, group))
-            if n != 0:
-                raise OSError(rposix.get_errno(), "os_initgroups failed")
-
-        return extdef([str, int], None, llimpl=initgroups_llimpl,
-                      export_name="ll_os.ll_initgroups")
-
-    @registering_if(os, 'getpgrp')
-    def register_os_getpgrp(self):
-        name = 'getpgrp'
-        if self.GETPGRP_HAVE_ARG:
-            c_func = self.llexternal(name, [rffi.INT], rffi.INT)
-            def c_func_llimpl():
-                res = rffi.cast(rffi.SIGNED, c_func(0))
-                if res == -1:
-                    raise OSError(rposix.get_errno(), "%s failed" % name)
-                return res
-
-            c_func_llimpl.func_name = name + '_llimpl'
-
-            return extdef([], int, llimpl=c_func_llimpl,
-                          export_name='ll_os.ll_os_' + name)
-        else:
-            return self.extdef_for_os_function_returning_int('getpgrp')
-
-    @registering_if(os, 'setpgrp')
-    def register_os_setpgrp(self):
-        name = 'setpgrp'
-        if self.SETPGRP_HAVE_ARG:
-            c_func = self.llexternal(name, [rffi.INT, rffi.INT], rffi.INT)
-            def c_func_llimpl():
-                res = rffi.cast(rffi.SIGNED, c_func(0, 0))
-                if res == -1:
-                    raise OSError(rposix.get_errno(), "%s failed" % name)
-
-            c_func_llimpl.func_name = name + '_llimpl'
-
-            return extdef([], None, llimpl=c_func_llimpl,
-                          export_name='ll_os.ll_os_' + name)
-        else:
-            return self.extdef_for_os_function_accepting_0int(name)
-
-    @registering_if(os, 'tcgetpgrp')
-    def register_os_tcgetpgrp(self):
-        c_tcgetpgrp = self.llexternal('tcgetpgrp', [rffi.INT], rffi.PID_T)
-
-        def c_tcgetpgrp_llimpl(fd):
-            res = c_tcgetpgrp(rffi.cast(rffi.INT, fd))
-            res = rffi.cast(lltype.Signed, res)
-            if res == -1:
-                raise OSError(rposix.get_errno(), "tcgetpgrp failed")
-            return res
-
-        return extdef([int], int, llimpl=c_tcgetpgrp_llimpl,
-                      export_name='ll_os.ll_os_tcgetpgrp')
-
-    @registering_if(os, 'tcsetpgrp')
-    def register_os_tcsetpgrp(self):
-        c_tcsetpgrp = self.llexternal('tcsetpgrp', [rffi.INT, rffi.PID_T],
-                                      rffi.INT)
-
-        def c_tcsetpgrp_llimpl(fd, pgrp):
-            res = c_tcsetpgrp(rffi.cast(rffi.INT, fd),
-                              rffi.cast(rffi.PID_T, pgrp))
-            res = rffi.cast(lltype.Signed, res)
-            if res == -1:
-                raise OSError(rposix.get_errno(), "tcsetpgrp failed")
-
-        return extdef([int, int], None, llimpl=c_tcsetpgrp_llimpl,
-                      export_name='ll_os.ll_os_tcsetpgrp')
-
-    @registering_if(os, 'getppid')
-    def register_os_getppid(self):
-        return self.extdef_for_os_function_returning_int('getppid')
-
-    @registering_if(os, 'getpgid')
-    def register_os_getpgid(self):
-        return self.extdef_for_os_function_int_to_int('getpgid')
-
-    @registering_if(os, 'setpgid')
-    def register_os_setpgid(self):
-        return self.extdef_for_os_function_accepting_2int('setpgid')
-
     @registering_if(os, 'getloadavg')
     def register_os_getloadavg(self):
         AP = rffi.CArrayPtr(lltype.Float)

From noreply at buildbot.pypy.org  Fri Nov  7 14:52:26 2014
From: noreply at buildbot.pypy.org (amauryfa)
Date: Fri,  7 Nov 2014 14:52:26 +0100 (CET)
Subject: [pypy-commit] pypy more-rposix: fixes
Message-ID: <20141107135226.7E2771C030E@cobra.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc 
Branch: more-rposix
Changeset: r74370:c41704a70c05
Date: 2014-11-07 09:29 +0100
http://bitbucket.org/pypy/pypy/changeset/c41704a70c05/

Log:	fixes

diff --git a/rpython/rlib/rposix.py b/rpython/rlib/rposix.py
--- a/rpython/rlib/rposix.py
+++ b/rpython/rlib/rposix.py
@@ -139,7 +139,8 @@
     includes = ['io.h', 'sys/utime.h', 'sys/types.h']
 else:
     includes = ['unistd.h',  'sys/types.h',
-                'utime.h', 'sys/time.h', 'sys/times.h']
+                'utime.h', 'sys/time.h', 'sys/times.h',
+                'grp.h']
 eci = ExternalCompilationInfo(
     includes=includes,
 )
@@ -610,8 +611,8 @@
 def getsid(pid):
     return handle_posix_error('getsid', c_getsid(pid))
 
-c_getpgid = external('getpid', [rffi.PID_T], rffi.PID_T)
-c_setpgid = external('setpid', [rffi.PID_T, rffi.PID_T], rffi.INT)
+c_getpgid = external('getpgid', [rffi.PID_T], rffi.PID_T)
+c_setpgid = external('setpgid', [rffi.PID_T, rffi.PID_T], rffi.INT)
 
 @replace_os_function('getpgid')
 def getpgid(pid):
@@ -671,9 +672,9 @@
 @replace_os_function('setpgrp')
 def setpgrp():
     if SETPGRP_HAVE_ARG:
-        return handle_posix_error('setpgrp', c_setpgrp(0, 0))
+        handle_posix_error('setpgrp', c_setpgrp(0, 0))
     else:
-        return handle_posix_error('setpgrp', c_setpgrp())
+        handle_posix_error('setpgrp', c_setpgrp())
 
 c_tcgetpgrp = external('tcgetpgrp', [rffi.INT], rffi.PID_T)
 c_tcsetpgrp = external('tcsetpgrp', [rffi.INT, rffi.PID_T], rffi.INT)

From noreply at buildbot.pypy.org  Fri Nov  7 15:24:51 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Fri,  7 Nov 2014 15:24:51 +0100 (CET)
Subject: [pypy-commit] pypy default: Add some more RPY_EXPORTED_FOR_TESTS,
	for Windows
Message-ID: <20141107142451.6E4211D2900@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r74371:50f95a001c7c
Date: 2014-11-07 15:22 +0100
http://bitbucket.org/pypy/pypy/changeset/50f95a001c7c/

Log:	Add some more RPY_EXPORTED_FOR_TESTS, for Windows

diff --git a/rpython/translator/c/src/profiling.c b/rpython/translator/c/src/profiling.c
--- a/rpython/translator/c/src/profiling.c
+++ b/rpython/translator/c/src/profiling.c
@@ -10,7 +10,7 @@
 static int profiling_setup = 0;
 
 RPY_EXPORTED_FOR_TESTS
-void pypy_setup_profiling()
+void pypy_setup_profiling(void)
 {
   if (!profiling_setup) {
     cpu_set_t set;
@@ -23,7 +23,7 @@
 }
 
 RPY_EXPORTED_FOR_TESTS
-void pypy_teardown_profiling()
+void pypy_teardown_profiling(void)
 {
   if (profiling_setup) {
     sched_setaffinity(0, sizeof(cpu_set_t), &base_cpu_set);
@@ -40,7 +40,8 @@
 static DWORD_PTR base_affinity_mask;
 static int profiling_setup = 0;
 
-void pypy_setup_profiling() { 
+RPY_EXPORTED_FOR_TESTS
+void pypy_setup_profiling(void) {
     if (!profiling_setup) {
         DWORD_PTR affinity_mask, system_affinity_mask;
         GetProcessAffinityMask(GetCurrentProcess(),
@@ -55,7 +56,8 @@
     }
 }
 
-void pypy_teardown_profiling() {
+RPY_EXPORTED_FOR_TESTS
+void pypy_teardown_profiling(void) {
     if (profiling_setup) {
         SetProcessAffinityMask(GetCurrentProcess(), base_affinity_mask);
         profiling_setup = 0;
@@ -65,7 +67,7 @@
 #else
 
 /* Empty implementations for other platforms */
-void pypy_setup_profiling() { }
-void pypy_teardown_profiling() { }
+RPY_EXPORTED_FOR_TESTS void pypy_setup_profiling(void) { }
+RPY_EXPORTED_FOR_TESTS void pypy_teardown_profiling(void) { }
 
 #endif
diff --git a/rpython/translator/c/src/profiling.h b/rpython/translator/c/src/profiling.h
--- a/rpython/translator/c/src/profiling.h
+++ b/rpython/translator/c/src/profiling.h
@@ -1,7 +1,7 @@
 #ifndef _PYPY_PROFILING_H
 #define _PYPY_PROFILING_H
 
-void pypy_setup_profiling();
-void pypy_teardown_profiling();
+RPY_EXPORTED_FOR_TESTS void pypy_setup_profiling(void);
+RPY_EXPORTED_FOR_TESTS void pypy_teardown_profiling(void);
 
 #endif

From noreply at buildbot.pypy.org  Fri Nov  7 15:29:21 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Fri,  7 Nov 2014 15:29:21 +0100 (CET)
Subject: [pypy-commit] pypy default: pypy__tzset() returns Void.
Message-ID: <20141107142921.4B0A21D2900@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r74372:047540e17457
Date: 2014-11-07 15:27 +0100
http://bitbucket.org/pypy/pypy/changeset/047540e17457/

Log:	pypy__tzset() returns Void.

diff --git a/pypy/module/rctime/interp_time.py b/pypy/module/rctime/interp_time.py
--- a/pypy/module/rctime/interp_time.py
+++ b/pypy/module/rctime/interp_time.py
@@ -178,19 +178,19 @@
 if _WIN:
     win_eci = ExternalCompilationInfo(
         includes = ["time.h"],
-        post_include_bits = ["RPY_EXPORTED_FOR_TESTS\n"
+        post_include_bits = ["RPY_EXPORTED_FOR_TESTS "
                              "long pypy_get_timezone();\n"
-                             "RPY_EXPORTED_FOR_TESTS\n"
+                             "RPY_EXPORTED_FOR_TESTS "
                              "int pypy_get_daylight();\n"
-                             "RPY_EXPORTED_FOR_TESTS\n"
+                             "RPY_EXPORTED_FOR_TESTS "
                              "char** pypy_get_tzname();\n"
-                             "RPY_EXPORTED_FOR_TESTS\n"
-                             "void* pypy__tzset();"],
+                             "RPY_EXPORTED_FOR_TESTS "
+                             "void pypy__tzset();"],
         separate_module_sources = ["""
         long pypy_get_timezone() { return timezone; }
         int pypy_get_daylight() { return daylight; }
         char** pypy_get_tzname() { return tzname; }
-        void pypy__tzset() { return _tzset(); }
+        void pypy__tzset() { _tzset(); }
         """])
     # Ensure sure that we use _tzset() and timezone from the same C Runtime.
     c_tzset = external('pypy__tzset', [], lltype.Void, win_eci)

From noreply at buildbot.pypy.org  Fri Nov  7 15:39:20 2014
From: noreply at buildbot.pypy.org (mattip)
Date: Fri,  7 Nov 2014 15:39:20 +0100 (CET)
Subject: [pypy-commit] pypy default: add RPY_EXPORTED_FOR_TESTS for windows
Message-ID: <20141107143920.332E31C030E@cobra.cs.uni-duesseldorf.de>

Author: Matti Picus 
Branch: 
Changeset: r74373:f20de5f3f2b5
Date: 2014-11-07 15:37 -0600
http://bitbucket.org/pypy/pypy/changeset/f20de5f3f2b5/

Log:	add RPY_EXPORTED_FOR_TESTS for windows

diff --git a/rpython/translator/c/src/libffi_msvc/ffi.h b/rpython/translator/c/src/libffi_msvc/ffi.h
--- a/rpython/translator/c/src/libffi_msvc/ffi.h
+++ b/rpython/translator/c/src/libffi_msvc/ffi.h
@@ -60,6 +60,7 @@
 
 /* ---- System configuration information --------------------------------- */
 
+#include  /* for RPY_EXPORTED_FOR_TESTS */
 #include 
 
 #ifndef LIBFFI_ASM
@@ -264,12 +265,14 @@
 
 /* ---- Public interface definition -------------------------------------- */
 
+RPY_EXPORTED_FOR_TESTS
 ffi_status ffi_prep_cif(/*@out@*/ /*@partial@*/ ffi_cif *cif, 
 			ffi_abi abi,
 			unsigned int nargs, 
 			/*@dependent@*/ /*@out@*/ /*@partial@*/ ffi_type *rtype, 
 			/*@dependent@*/ ffi_type **atypes);
 
+RPY_EXPORTED_FOR_TESTS
 int
 ffi_call(/*@dependent@*/ ffi_cif *cif, 
 	 void (*fn)(), 

From noreply at buildbot.pypy.org  Fri Nov  7 15:51:43 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Fri,  7 Nov 2014 15:51:43 +0100 (CET)
Subject: [pypy-commit] pypy default: On Windows it's a compilation error to
 try to assign to "stdin". Fix by
Message-ID: <20141107145143.34E861D2900@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r74374:7bea28d4aab3
Date: 2014-11-07 15:48 +0100
http://bitbucket.org/pypy/pypy/changeset/7bea28d4aab3/

Log:	On Windows it's a compilation error to try to assign to "stdin".
	Fix by not generating the setter function if not required.

diff --git a/rpython/rlib/rfile.py b/rpython/rlib/rfile.py
--- a/rpython/rlib/rfile.py
+++ b/rpython/rlib/rfile.py
@@ -96,9 +96,12 @@
 c_ferror = llexternal('ferror', [FILEP], rffi.INT)
 c_clearerr = llexternal('clearerr', [FILEP], lltype.Void)
 
-c_stdin = rffi.CExternVariable(FILEP, 'stdin', eci, c_type='FILE*')[0]
-c_stdout = rffi.CExternVariable(FILEP, 'stdout', eci, c_type='FILE*')[0]
-c_stderr = rffi.CExternVariable(FILEP, 'stderr', eci, c_type='FILE*')[0]
+c_stdin = rffi.CExternVariable(FILEP, 'stdin', eci, c_type='FILE*',
+                               getter_only=True)
+c_stdout = rffi.CExternVariable(FILEP, 'stdout', eci, c_type='FILE*',
+                                getter_only=True)
+c_stderr = rffi.CExternVariable(FILEP, 'stderr', eci, c_type='FILE*',
+                                getter_only=True)
 
 
 def _error(ll_file):
diff --git a/rpython/rtyper/lltypesystem/rffi.py b/rpython/rtyper/lltypesystem/rffi.py
--- a/rpython/rtyper/lltypesystem/rffi.py
+++ b/rpython/rtyper/lltypesystem/rffi.py
@@ -605,7 +605,7 @@
 
 def CExternVariable(TYPE, name, eci, _CConstantClass=CConstant,
                     sandboxsafe=False, _nowrapper=False,
-                    c_type=None):
+                    c_type=None, getter_only=False):
     """Return a pair of functions - a getter and a setter - to access
     the given global C variable.
     """
@@ -638,19 +638,26 @@
     if sys.platform != 'win32':
         lines.append('extern %s %s;' % (c_type, name))
     lines.append(c_getter)
-    lines.append(c_setter)
+    if not getter_only:
+        lines.append(c_setter)
+    prototypes = [getter_prototype]
+    if not getter_only:
+        prototypes.append(setter_prototype)
     sources = ('\n'.join(lines),)
     new_eci = eci.merge(ExternalCompilationInfo(
         separate_module_sources = sources,
-        post_include_bits = [getter_prototype, setter_prototype],
+        post_include_bits = prototypes,
     ))
 
     getter = llexternal(getter_name, [], TYPE, compilation_info=new_eci,
                         sandboxsafe=sandboxsafe, _nowrapper=_nowrapper)
-    setter = llexternal(setter_name, [TYPE], lltype.Void,
-                        compilation_info=new_eci, sandboxsafe=sandboxsafe,
-                        _nowrapper=_nowrapper)
-    return getter, setter
+    if getter_only:
+        return getter
+    else:
+        setter = llexternal(setter_name, [TYPE], lltype.Void,
+                            compilation_info=new_eci, sandboxsafe=sandboxsafe,
+                            _nowrapper=_nowrapper)
+        return getter, setter
 
 # char, represented as a Python character
 # (use SIGNEDCHAR or UCHAR for the small integer types)

From noreply at buildbot.pypy.org  Fri Nov  7 15:51:44 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Fri,  7 Nov 2014 15:51:44 +0100 (CET)
Subject: [pypy-commit] pypy default: merge heads
Message-ID: <20141107145144.5C6D61D2900@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r74375:aff60e5fa2b2
Date: 2014-11-07 15:51 +0100
http://bitbucket.org/pypy/pypy/changeset/aff60e5fa2b2/

Log:	merge heads

diff --git a/rpython/translator/c/src/libffi_msvc/ffi.h b/rpython/translator/c/src/libffi_msvc/ffi.h
--- a/rpython/translator/c/src/libffi_msvc/ffi.h
+++ b/rpython/translator/c/src/libffi_msvc/ffi.h
@@ -60,6 +60,7 @@
 
 /* ---- System configuration information --------------------------------- */
 
+#include  /* for RPY_EXPORTED_FOR_TESTS */
 #include 
 
 #ifndef LIBFFI_ASM
@@ -264,12 +265,14 @@
 
 /* ---- Public interface definition -------------------------------------- */
 
+RPY_EXPORTED_FOR_TESTS
 ffi_status ffi_prep_cif(/*@out@*/ /*@partial@*/ ffi_cif *cif, 
 			ffi_abi abi,
 			unsigned int nargs, 
 			/*@dependent@*/ /*@out@*/ /*@partial@*/ ffi_type *rtype, 
 			/*@dependent@*/ ffi_type **atypes);
 
+RPY_EXPORTED_FOR_TESTS
 int
 ffi_call(/*@dependent@*/ ffi_cif *cif, 
 	 void (*fn)(), 

From noreply at buildbot.pypy.org  Fri Nov  7 17:41:01 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Fri,  7 Nov 2014 17:41:01 +0100 (CET)
Subject: [pypy-commit] pypy default: Issue 1914: test and fix
Message-ID: <20141107164101.476301D28DC@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r74376:a6902fd7e9a7
Date: 2014-11-07 17:40 +0100
http://bitbucket.org/pypy/pypy/changeset/a6902fd7e9a7/

Log:	Issue 1914: test and fix

diff --git a/rpython/jit/metainterp/test/test_rawmem.py b/rpython/jit/metainterp/test/test_rawmem.py
--- a/rpython/jit/metainterp/test/test_rawmem.py
+++ b/rpython/jit/metainterp/test/test_rawmem.py
@@ -89,6 +89,16 @@
                                        'finish': 1})
         self.metainterp.staticdata.stats.check_resops({'finish': 1}, omit_finish=False)
 
+    def test_scoped_alloc_buffer(self):
+        def f():
+            with rffi.scoped_alloc_buffer(42) as p:
+                p.raw[0] = 'X'
+                s = p.str(1)
+            return ord(s[0])
+
+        res = self.interp_operations(f, [])
+        assert res == ord('X')
+
 
 class TestRawMem(RawMemTests, LLJitMixin):
 
diff --git a/rpython/rtyper/lltypesystem/rffi.py b/rpython/rtyper/lltypesystem/rffi.py
--- a/rpython/rtyper/lltypesystem/rffi.py
+++ b/rpython/rtyper/lltypesystem/rffi.py
@@ -822,6 +822,8 @@
     free_nonmovingbuffer._annenforceargs_ = [strtype, None, bool, bool]
 
     # int -> (char*, str, int)
+    # Can't inline this because of the raw address manipulation.
+    @jit.dont_look_inside
     def alloc_buffer(count):
         """
         Returns a (raw_buffer, gc_buffer, case_num) triple,

From noreply at buildbot.pypy.org  Fri Nov  7 18:22:53 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Fri,  7 Nov 2014 18:22:53 +0100 (CET)
Subject: [pypy-commit] pypy stmgc-c7: Fix
Message-ID: <20141107172253.0A81D1D28DC@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: stmgc-c7
Changeset: r74377:2da4dcd634f8
Date: 2014-11-07 18:22 +0100
http://bitbucket.org/pypy/pypy/changeset/2da4dcd634f8/

Log:	Fix

diff --git a/pypy/stm/print_stm_log.py b/pypy/stm/print_stm_log.py
--- a/pypy/stm/print_stm_log.py
+++ b/pypy/stm/print_stm_log.py
@@ -66,7 +66,7 @@
             sec, nsec, threadnum, otherthreadnum, event, len1, len2 = \
                   struct.unpack("IIIIBBB", packet)
             if event >= _STM_EVENT_N:
-                raise ValueError("the file %r appears corrupted")
+                raise ValueError("the file %r appears corrupted" % (filename,))
             m1 = f.read(len1)
             m2 = f.read(len2)
             result.append(LogEntry(sec + 0.000000001 * nsec,

From noreply at buildbot.pypy.org  Fri Nov  7 18:42:38 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Fri,  7 Nov 2014 18:42:38 +0100 (CET)
Subject: [pypy-commit] stmgc hashtable: Add a passing test
Message-ID: <20141107174238.0E8C11D28F2@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: hashtable
Changeset: r1495:9633e4f3f503
Date: 2014-11-07 18:42 +0100
http://bitbucket.org/pypy/stmgc/changeset/9633e4f3f503/

Log:	Add a passing test

diff --git a/c7/test/test_finalizer.py b/c7/test/test_finalizer.py
--- a/c7/test/test_finalizer.py
+++ b/c7/test/test_finalizer.py
@@ -139,12 +139,17 @@
 
 
 class TestRegularFinalizer(BaseTest):
+    expect_content_character = None
 
     def setup_method(self, meth):
         BaseTest.setup_method(self, meth)
         #
         @ffi.callback("void(object_t *)")
         def finalizer(obj):
+            print "finalizing!", obj
+            assert stm_get_obj_size(obj) in [16, 32, 48, 56]
+            if self.expect_content_character is not None:
+                assert stm_get_char(obj) == self.expect_content_character
             self.finalizers_called.append(obj)
         self.finalizers_called = []
         lib.stmcb_finalizer = finalizer
@@ -175,6 +180,21 @@
         stm_major_collect()
         self.expect_finalized([lp1, lp2, lp3])
 
+    def test_finalizer_from_other_thread(self):
+        self.start_transaction()
+        lp1 = stm_allocate_with_finalizer(48)
+        stm_set_char(lp1, 'H')
+        self.expect_content_character = 'H'
+        print lp1
+        #
+        self.switch(1)
+        self.start_transaction()
+        stm_major_collect()
+        self.expect_finalized([])      # marked as dead, but wrong thread
+        #
+        self.switch(0)
+        self.expect_finalized([lp1])   # now it has been finalized
+
     def test_finalizer_ordering(self):
         self.start_transaction()
         lp1 = stm_allocate_with_finalizer_refs(1)

From noreply at buildbot.pypy.org  Fri Nov  7 18:53:58 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Fri,  7 Nov 2014 18:53:58 +0100 (CET)
Subject: [pypy-commit] stmgc hashtable: A crashing test
Message-ID: <20141107175358.988061C030E@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: hashtable
Changeset: r1496:926d81d98692
Date: 2014-11-07 18:54 +0100
http://bitbucket.org/pypy/stmgc/changeset/926d81d98692/

Log:	A crashing test

diff --git a/c7/test/test_finalizer.py b/c7/test/test_finalizer.py
--- a/c7/test/test_finalizer.py
+++ b/c7/test/test_finalizer.py
@@ -140,6 +140,7 @@
 
 class TestRegularFinalizer(BaseTest):
     expect_content_character = None
+    run_major_collect_in_finalizer = False
 
     def setup_method(self, meth):
         BaseTest.setup_method(self, meth)
@@ -151,6 +152,8 @@
             if self.expect_content_character is not None:
                 assert stm_get_char(obj) == self.expect_content_character
             self.finalizers_called.append(obj)
+            if self.run_major_collect_in_finalizer:
+                stm_major_collect()
         self.finalizers_called = []
         lib.stmcb_finalizer = finalizer
         self._finalizer_keepalive = finalizer
@@ -206,7 +209,7 @@
         stm_major_collect()
         self.expect_finalized([lp3])
 
-    def test_finalizer_extra_transation(self):
+    def test_finalizer_extra_transaction(self):
         self.start_transaction()
         lp1 = stm_allocate_with_finalizer(32)
         print lp1
@@ -240,3 +243,12 @@
         stm_major_collect()
         self.switch(0)
         self.expect_finalized([lp2, lp1])
+
+    def test_run_major_collect_in_finalizer(self):
+        self.run_major_collect_in_finalizer = True
+        self.start_transaction()
+        lp1 = stm_allocate_with_finalizer(32)
+        lp2 = stm_allocate_with_finalizer(32)
+        lp3 = stm_allocate_with_finalizer(32)
+        print lp1, lp2, lp3
+        stm_major_collect()

From noreply at buildbot.pypy.org  Fri Nov  7 19:01:33 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Fri,  7 Nov 2014 19:01:33 +0100 (CET)
Subject: [pypy-commit] stmgc hashtable: fix 926d81d98692
Message-ID: <20141107180133.B89DA1D28F2@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: hashtable
Changeset: r1497:fcfed3deb79d
Date: 2014-11-07 19:01 +0100
http://bitbucket.org/pypy/stmgc/changeset/fcfed3deb79d/

Log:	fix 926d81d98692

diff --git a/c7/stm/finalizer.c b/c7/stm/finalizer.c
--- a/c7/stm/finalizer.c
+++ b/c7/stm/finalizer.c
@@ -390,6 +390,24 @@
     LIST_FREE(_finalizer_emptystack);
 }
 
+static void mark_visit_from_finalizer1(char *base, struct finalizers_s *f)
+{
+    if (f != NULL && f->run_finalizers != NULL) {
+        LIST_FOREACH_R(f->run_finalizers, object_t * /*item*/,
+                       mark_visit_object(item, base));
+    }
+}
+
+static void mark_visit_from_finalizer_pending(void)
+{
+    long j;
+    for (j = 1; j <= NB_SEGMENTS; j++) {
+        struct stm_priv_segment_info_s *pseg = get_priv_segment(j);
+        mark_visit_from_finalizer1(pseg->pub.segment_base, pseg->finalizers);
+    }
+    mark_visit_from_finalizer1(stm_object_pages, &g_finalizers);
+}
+
 static void _execute_finalizers(struct finalizers_s *f)
 {
     if (f->run_finalizers == NULL)
diff --git a/c7/stm/finalizer.h b/c7/stm/finalizer.h
--- a/c7/stm/finalizer.h
+++ b/c7/stm/finalizer.h
@@ -6,6 +6,7 @@
     uintptr_t *running_next;
 };
 
+static void mark_visit_from_finalizer_pending(void);
 static void deal_with_young_objects_with_finalizers(void);
 static void deal_with_old_objects_with_finalizers(void);
 static void deal_with_objects_with_finalizers(void);
diff --git a/c7/stm/gcpage.c b/c7/stm/gcpage.c
--- a/c7/stm/gcpage.c
+++ b/c7/stm/gcpage.c
@@ -631,6 +631,7 @@
     mark_visit_from_modified_objects();
     mark_visit_from_markers();
     mark_visit_from_roots();
+    mark_visit_from_finalizer_pending();
     LIST_FREE(mark_objects_to_trace);
 
     /* finalizer support: will mark as WL_VISITED all objects with a

From noreply at buildbot.pypy.org  Fri Nov  7 19:06:06 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Fri,  7 Nov 2014 19:06:06 +0100 (CET)
Subject: [pypy-commit] stmgc hashtable: Remove this include,
	which is outdated
Message-ID: <20141107180606.257331D28F2@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: hashtable
Changeset: r1498:ece6317ba186
Date: 2014-11-07 19:06 +0100
http://bitbucket.org/pypy/stmgc/changeset/ece6317ba186/

Log:	Remove this include, which is outdated

diff --git a/c7/stm/hashtable.c b/c7/stm/hashtable.c
--- a/c7/stm/hashtable.c
+++ b/c7/stm/hashtable.c
@@ -1,5 +1,3 @@
-#include "stmgc.h"
-
 /*
 Design of stmgc's "hashtable" objects
 =====================================

From noreply at buildbot.pypy.org  Fri Nov  7 19:10:01 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Fri,  7 Nov 2014 19:10:01 +0100 (CET)
Subject: [pypy-commit] pypy stmgc-c7: import stmgc/ece6317ba186 [hashtable]
Message-ID: <20141107181001.019861D28DC@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: stmgc-c7
Changeset: r74378:8bb7d5b1d1a9
Date: 2014-11-07 19:07 +0100
http://bitbucket.org/pypy/pypy/changeset/8bb7d5b1d1a9/

Log:	import stmgc/ece6317ba186 [hashtable]

diff --git a/rpython/translator/stm/src_stm/revision b/rpython/translator/stm/src_stm/revision
--- a/rpython/translator/stm/src_stm/revision
+++ b/rpython/translator/stm/src_stm/revision
@@ -1,1 +1,1 @@
-3a8ef5f741ab
+ece6317ba186
diff --git a/rpython/translator/stm/src_stm/stm/core.c b/rpython/translator/stm/src_stm/stm/core.c
--- a/rpython/translator/stm/src_stm/stm/core.c
+++ b/rpython/translator/stm/src_stm/stm/core.c
@@ -373,6 +373,7 @@
     assert(tree_is_cleared(STM_PSEGMENT->nursery_objects_shadows));
     assert(tree_is_cleared(STM_PSEGMENT->callbacks_on_commit_and_abort[0]));
     assert(tree_is_cleared(STM_PSEGMENT->callbacks_on_commit_and_abort[1]));
+    assert(list_is_empty(STM_PSEGMENT->young_objects_with_light_finalizers));
     assert(STM_PSEGMENT->objects_pointing_to_nursery == NULL);
     assert(STM_PSEGMENT->large_overflow_objects == NULL);
     assert(STM_PSEGMENT->finalizers == NULL);
@@ -973,6 +974,8 @@
                        (int)pseg->transaction_state);
     }
 
+    abort_finalizers(pseg);
+
     /* throw away the content of the nursery */
     long bytes_in_nursery = throw_away_nursery(pseg);
 
@@ -1061,8 +1064,6 @@
     /* invoke the callbacks */
     invoke_and_clear_user_callbacks(1);   /* for abort */
 
-    abort_finalizers();
-
     if (is_abort(STM_SEGMENT->nursery_end)) {
         /* done aborting */
         STM_SEGMENT->nursery_end = pause_signalled ? NSE_SIGPAUSE
diff --git a/rpython/translator/stm/src_stm/stm/finalizer.c b/rpython/translator/stm/src_stm/stm/finalizer.c
--- a/rpython/translator/stm/src_stm/stm/finalizer.c
+++ b/rpython/translator/stm/src_stm/stm/finalizer.c
@@ -59,28 +59,73 @@
     STM_PSEGMENT->finalizers = NULL;
 }
 
-static void _abort_finalizers(void)
+static void abort_finalizers(struct stm_priv_segment_info_s *pseg)
 {
     /* like _commit_finalizers(), but forget everything from the
        current transaction */
-    if (STM_PSEGMENT->finalizers->run_finalizers != NULL) {
-        if (STM_PSEGMENT->finalizers->running_next != NULL) {
-            *STM_PSEGMENT->finalizers->running_next = (uintptr_t)-1;
+    if (pseg->finalizers != NULL) {
+        if (pseg->finalizers->run_finalizers != NULL) {
+            if (pseg->finalizers->running_next != NULL) {
+                *pseg->finalizers->running_next = (uintptr_t)-1;
+            }
+            list_free(pseg->finalizers->run_finalizers);
         }
-        list_free(STM_PSEGMENT->finalizers->run_finalizers);
+        list_free(pseg->finalizers->objects_with_finalizers);
+        free(pseg->finalizers);
+        pseg->finalizers = NULL;
     }
-    list_free(STM_PSEGMENT->finalizers->objects_with_finalizers);
-    free(STM_PSEGMENT->finalizers);
-    STM_PSEGMENT->finalizers = NULL;
+
+    /* call the light finalizers for objects that are about to
+       be forgotten from the current transaction */
+    char *old_gs_register = STM_SEGMENT->segment_base;
+    bool must_fix_gs = old_gs_register != pseg->pub.segment_base;
+
+    struct list_s *lst = pseg->young_objects_with_light_finalizers;
+    long i, count = list_count(lst);
+    if (lst > 0) {
+        for (i = 0; i < count; i++) {
+            object_t *obj = (object_t *)list_item(lst, i);
+            assert(_is_young(obj));
+            if (must_fix_gs) {
+                set_gs_register(pseg->pub.segment_base);
+                must_fix_gs = false;
+            }
+            stmcb_light_finalizer(obj);
+        }
+        list_clear(lst);
+    }
+
+    /* also deals with overflow objects: they are at the tail of
+       old_objects_with_light_finalizers (this list is kept in order
+       and we cannot add any already-committed object) */
+    lst = pseg->old_objects_with_light_finalizers;
+    count = list_count(lst);
+    while (count > 0) {
+        object_t *obj = (object_t *)list_item(lst, --count);
+        if (!IS_OVERFLOW_OBJ(pseg, obj))
+            break;
+        lst->count = count;
+        if (must_fix_gs) {
+            set_gs_register(pseg->pub.segment_base);
+            must_fix_gs = false;
+        }
+        stmcb_light_finalizer(obj);
+    }
+
+    if (STM_SEGMENT->segment_base != old_gs_register)
+        set_gs_register(old_gs_register);
 }
 
 
 void stm_enable_light_finalizer(object_t *obj)
 {
-    if (_is_young(obj))
+    if (_is_young(obj)) {
         LIST_APPEND(STM_PSEGMENT->young_objects_with_light_finalizers, obj);
-    else
+    }
+    else {
+        assert(_is_from_same_transaction(obj));
         LIST_APPEND(STM_PSEGMENT->old_objects_with_light_finalizers, obj);
+    }
 }
 
 object_t *stm_allocate_with_finalizer(ssize_t size_rounded_up)
@@ -109,7 +154,7 @@
     struct list_s *lst = STM_PSEGMENT->young_objects_with_light_finalizers;
     long i, count = list_count(lst);
     for (i = 0; i < count; i++) {
-        object_t* obj = (object_t *)list_item(lst, i);
+        object_t *obj = (object_t *)list_item(lst, i);
         assert(_is_young(obj));
 
         object_t *TLPREFIX *pforwarded_array = (object_t *TLPREFIX *)obj;
@@ -139,7 +184,7 @@
         long i, count = list_count(lst);
         lst->count = 0;
         for (i = 0; i < count; i++) {
-            object_t* obj = (object_t *)list_item(lst, i);
+            object_t *obj = (object_t *)list_item(lst, i);
             if (!mark_visited_test(obj)) {
                 /* not marked: object dies */
                 /* we're calling the light finalizer in the same
@@ -346,6 +391,24 @@
     LIST_FREE(_finalizer_emptystack);
 }
 
+static void mark_visit_from_finalizer1(char *base, struct finalizers_s *f)
+{
+    if (f != NULL && f->run_finalizers != NULL) {
+        LIST_FOREACH_R(f->run_finalizers, object_t * /*item*/,
+                       mark_visit_object(item, base));
+    }
+}
+
+static void mark_visit_from_finalizer_pending(void)
+{
+    long j;
+    for (j = 1; j <= NB_SEGMENTS; j++) {
+        struct stm_priv_segment_info_s *pseg = get_priv_segment(j);
+        mark_visit_from_finalizer1(pseg->pub.segment_base, pseg->finalizers);
+    }
+    mark_visit_from_finalizer1(stm_object_pages, &g_finalizers);
+}
+
 static void _execute_finalizers(struct finalizers_s *f)
 {
     if (f->run_finalizers == NULL)
diff --git a/rpython/translator/stm/src_stm/stm/finalizer.h b/rpython/translator/stm/src_stm/stm/finalizer.h
--- a/rpython/translator/stm/src_stm/stm/finalizer.h
+++ b/rpython/translator/stm/src_stm/stm/finalizer.h
@@ -7,6 +7,7 @@
     uintptr_t *running_next;
 };
 
+static void mark_visit_from_finalizer_pending(void);
 static void deal_with_young_objects_with_finalizers(void);
 static void deal_with_old_objects_with_finalizers(void);
 static void deal_with_objects_with_finalizers(void);
@@ -15,18 +16,13 @@
 static void teardown_finalizer(void);
 
 static void _commit_finalizers(void);
-static void _abort_finalizers(void);
+static void abort_finalizers(struct stm_priv_segment_info_s *);
 
 #define commit_finalizers()   do {              \
     if (STM_PSEGMENT->finalizers != NULL)       \
         _commit_finalizers();                   \
 } while (0)
 
-#define abort_finalizers()   do {               \
-    if (STM_PSEGMENT->finalizers != NULL)       \
-        _abort_finalizers();                    \
-} while (0)
-
 
 /* regular finalizers (objs from already-committed transactions) */
 static struct finalizers_s g_finalizers;
diff --git a/rpython/translator/stm/src_stm/stm/gcpage.c b/rpython/translator/stm/src_stm/stm/gcpage.c
--- a/rpython/translator/stm/src_stm/stm/gcpage.c
+++ b/rpython/translator/stm/src_stm/stm/gcpage.c
@@ -345,6 +345,8 @@
     LIST_APPEND(mark_objects_to_trace, obj);
 }
 
+#define TRACE_FOR_MAJOR_COLLECTION  (&mark_record_trace)
+
 static void mark_trace(object_t *obj, char *segment_base)
 {
     assert(list_is_empty(mark_objects_to_trace));
@@ -353,7 +355,7 @@
         /* trace into the object (the version from 'segment_base') */
         struct object_s *realobj =
             (struct object_s *)REAL_ADDRESS(segment_base, obj);
-        stmcb_trace(realobj, &mark_record_trace);
+        stmcb_trace(realobj, TRACE_FOR_MAJOR_COLLECTION);
 
         if (list_is_empty(mark_objects_to_trace))
             break;
@@ -630,6 +632,7 @@
     mark_visit_from_modified_objects();
     mark_visit_from_markers();
     mark_visit_from_roots();
+    mark_visit_from_finalizer_pending();
     LIST_FREE(mark_objects_to_trace);
 
     /* finalizer support: will mark as WL_VISITED all objects with a
diff --git a/rpython/translator/stm/src_stm/stm/nursery.c b/rpython/translator/stm/src_stm/stm/nursery.c
--- a/rpython/translator/stm/src_stm/stm/nursery.c
+++ b/rpython/translator/stm/src_stm/stm/nursery.c
@@ -45,6 +45,10 @@
         tree_contains(STM_PSEGMENT->young_outside_nursery, (uintptr_t)obj));
 }
 
+static inline bool _is_from_same_transaction(object_t *obj) {
+    return _is_young(obj) || IS_OVERFLOW_OBJ(STM_PSEGMENT, obj);
+}
+
 long stm_can_move(object_t *obj)
 {
     /* 'long' return value to avoid using 'bool' in the public interface */
@@ -330,6 +334,7 @@
 }
 
 
+#define TRACE_FOR_MINOR_COLLECTION  (&minor_trace_if_young)
 
 static inline void _collect_now(object_t *obj)
 {
@@ -343,7 +348,7 @@
            outside the nursery, possibly forcing nursery objects out and
            adding them to 'objects_pointing_to_nursery' as well. */
         char *realobj = REAL_ADDRESS(STM_SEGMENT->segment_base, obj);
-        stmcb_trace((struct object_s *)realobj, &minor_trace_if_young);
+        stmcb_trace((struct object_s *)realobj, TRACE_FOR_MINOR_COLLECTION);
 
         obj->stm_flags |= GCFLAG_WRITE_BARRIER;
     }
diff --git a/rpython/translator/stm/src_stm/stmgc.c b/rpython/translator/stm/src_stm/stmgc.c
--- a/rpython/translator/stm/src_stm/stmgc.c
+++ b/rpython/translator/stm/src_stm/stmgc.c
@@ -18,6 +18,7 @@
 #include "stm/marker.h"
 #include "stm/prof.h"
 #include "stm/finalizer.h"
+#include "stm/hashtable.h"
 
 #include "stm/misc.c"
 #include "stm/list.c"
@@ -40,3 +41,4 @@
 #include "stm/prof.c"
 #include "stm/rewind_setjmp.c"
 #include "stm/finalizer.c"
+#include "stm/hashtable.c"
diff --git a/rpython/translator/stm/src_stm/stmgc.h b/rpython/translator/stm/src_stm/stmgc.h
--- a/rpython/translator/stm/src_stm/stmgc.h
+++ b/rpython/translator/stm/src_stm/stmgc.h
@@ -509,7 +509,7 @@
 /* Support for light finalizers.  This is a simple version of
    finalizers that guarantees not to do anything fancy, like not
    resurrecting objects. */
-void (*stmcb_light_finalizer)(object_t *);
+extern void (*stmcb_light_finalizer)(object_t *);
 void stm_enable_light_finalizer(object_t *);
 
 /* Support for regular finalizers.  Unreachable objects with
@@ -526,9 +526,30 @@
    transaction.  For older objects, the finalizer is called from a
    random thread between regular transactions, in a new custom
    transaction. */
-void (*stmcb_finalizer)(object_t *);
+extern void (*stmcb_finalizer)(object_t *);
 object_t *stm_allocate_with_finalizer(ssize_t size_rounded_up);
 
+/* Hashtables.  Keys are 64-bit unsigned integers, values are
+   'object_t *'.  Note that the type 'stm_hashtable_t' is not an
+   object type at all; you need to allocate and free it explicitly.
+   If you want to embed the hashtable inside an 'object_t' you
+   probably need a light finalizer to do the freeing. */
+typedef struct stm_hashtable_s stm_hashtable_t;
+stm_hashtable_t *stm_hashtable_create(void);
+void stm_hashtable_free(stm_hashtable_t *);
+object_t *stm_hashtable_read(object_t *, stm_hashtable_t *, uintptr_t key);
+void stm_hashtable_write(object_t *, stm_hashtable_t *, uintptr_t key,
+                         object_t *nvalue, stm_thread_local_t *);
+extern uint32_t stm_hashtable_entry_userdata;
+void stm_hashtable_tracefn(stm_hashtable_t *, void (object_t **));
+
+struct stm_hashtable_entry_s {
+    struct object_s header;
+    uint32_t userdata;
+    uintptr_t index;
+    object_t *object;
+};
+
 /* ==================== END ==================== */
 
 #endif

From noreply at buildbot.pypy.org  Fri Nov  7 20:02:47 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Fri,  7 Nov 2014 20:02:47 +0100 (CET)
Subject: [pypy-commit] stmgc hashtable: Need to fflush the stm.log file
	before a fork().
Message-ID: <20141107190247.EC8B11D3847@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: hashtable
Changeset: r1499:b27bcdd6bc87
Date: 2014-11-07 20:02 +0100
http://bitbucket.org/pypy/stmgc/changeset/b27bcdd6bc87/

Log:	Need to fflush the stm.log file before a fork().

diff --git a/c7/stm/forksupport.c b/c7/stm/forksupport.c
--- a/c7/stm/forksupport.c
+++ b/c7/stm/forksupport.c
@@ -201,9 +201,6 @@
        just release these locks early */
     s_mutex_unlock();
 
-    /* Open a new profiling file, if any */
-    forksupport_open_new_profiling_file();
-
     /* Move the copy of the mmap over the old one, overwriting it
        and thus freeing the old mapping in this process
     */
diff --git a/c7/stm/prof.c b/c7/stm/prof.c
--- a/c7/stm/prof.c
+++ b/c7/stm/prof.c
@@ -74,7 +74,13 @@
     return false;
 }
 
-static void forksupport_open_new_profiling_file(void)
+static void prof_forksupport_prepare(void)
+{
+    if (profiling_file != NULL)
+        fflush(profiling_file);
+}
+
+static void prof_forksupport_child(void)
 {
     if (close_timing_log() && profiling_basefn != NULL) {
         char filename[1024];
@@ -98,6 +104,15 @@
         expand_marker = default_expand_marker;
     profiling_expand_marker = expand_marker;
 
+    static bool fork_support_ready = false;
+    if (!fork_support_ready) {
+        int res = pthread_atfork(prof_forksupport_prepare,
+                                 NULL, prof_forksupport_child);
+        if (res != 0)
+            stm_fatalerror("pthread_atfork() failed: %m");
+        fork_support_ready = true;
+    }
+
     if (!open_timing_log(profiling_file_name))
         return -1;
 
diff --git a/c7/stm/prof.h b/c7/stm/prof.h
deleted file mode 100644
--- a/c7/stm/prof.h
+++ /dev/null
@@ -1,2 +0,0 @@
-
-static void forksupport_open_new_profiling_file(void);
diff --git a/c7/stmgc.c b/c7/stmgc.c
--- a/c7/stmgc.c
+++ b/c7/stmgc.c
@@ -15,7 +15,6 @@
 #include "stm/fprintcolor.h"
 #include "stm/weakref.h"
 #include "stm/marker.h"
-#include "stm/prof.h"
 #include "stm/finalizer.h"
 #include "stm/hashtable.h"
 

From noreply at buildbot.pypy.org  Fri Nov  7 20:07:51 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Fri,  7 Nov 2014 20:07:51 +0100 (CET)
Subject: [pypy-commit] pypy stmgc-c7: import stmgc/b27bcdd6bc87 [hashtable]
Message-ID: <20141107190751.357E51D3847@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: stmgc-c7
Changeset: r74379:38c9afbd253c
Date: 2014-11-07 20:04 +0100
http://bitbucket.org/pypy/pypy/changeset/38c9afbd253c/

Log:	import stmgc/b27bcdd6bc87 [hashtable]

diff --git a/rpython/translator/stm/src_stm/revision b/rpython/translator/stm/src_stm/revision
--- a/rpython/translator/stm/src_stm/revision
+++ b/rpython/translator/stm/src_stm/revision
@@ -1,1 +1,1 @@
-ece6317ba186
+b27bcdd6bc87
diff --git a/rpython/translator/stm/src_stm/stm/forksupport.c b/rpython/translator/stm/src_stm/stm/forksupport.c
--- a/rpython/translator/stm/src_stm/stm/forksupport.c
+++ b/rpython/translator/stm/src_stm/stm/forksupport.c
@@ -202,9 +202,6 @@
        just release these locks early */
     s_mutex_unlock();
 
-    /* Open a new profiling file, if any */
-    forksupport_open_new_profiling_file();
-
     /* Move the copy of the mmap over the old one, overwriting it
        and thus freeing the old mapping in this process
     */
diff --git a/rpython/translator/stm/src_stm/stm/prof.c b/rpython/translator/stm/src_stm/stm/prof.c
--- a/rpython/translator/stm/src_stm/stm/prof.c
+++ b/rpython/translator/stm/src_stm/stm/prof.c
@@ -75,7 +75,13 @@
     return false;
 }
 
-static void forksupport_open_new_profiling_file(void)
+static void prof_forksupport_prepare(void)
+{
+    if (profiling_file != NULL)
+        fflush(profiling_file);
+}
+
+static void prof_forksupport_child(void)
 {
     if (close_timing_log() && profiling_basefn != NULL) {
         char filename[1024];
@@ -99,6 +105,15 @@
         expand_marker = default_expand_marker;
     profiling_expand_marker = expand_marker;
 
+    static bool fork_support_ready = false;
+    if (!fork_support_ready) {
+        int res = pthread_atfork(prof_forksupport_prepare,
+                                 NULL, prof_forksupport_child);
+        if (res != 0)
+            stm_fatalerror("pthread_atfork() failed: %m");
+        fork_support_ready = true;
+    }
+
     if (!open_timing_log(profiling_file_name))
         return -1;
 
diff --git a/rpython/translator/stm/src_stm/stmgc.c b/rpython/translator/stm/src_stm/stmgc.c
--- a/rpython/translator/stm/src_stm/stmgc.c
+++ b/rpython/translator/stm/src_stm/stmgc.c
@@ -16,7 +16,6 @@
 #include "stm/fprintcolor.h"
 #include "stm/weakref.h"
 #include "stm/marker.h"
-#include "stm/prof.h"
 #include "stm/finalizer.h"
 #include "stm/hashtable.h"
 

From noreply at buildbot.pypy.org  Fri Nov  7 20:09:22 2014
From: noreply at buildbot.pypy.org (amauryfa)
Date: Fri,  7 Nov 2014 20:09:22 +0100 (CET)
Subject: [pypy-commit] pypy more-rposix: Temporary workaround,
	should be done differently.
Message-ID: <20141107190922.468051D28DC@cobra.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc 
Branch: more-rposix
Changeset: r74380:3600a9d62002
Date: 2014-11-07 17:17 +0100
http://bitbucket.org/pypy/pypy/changeset/3600a9d62002/

Log:	Temporary workaround, should be done differently.

diff --git a/rpython/rlib/objectmodel.py b/rpython/rlib/objectmodel.py
--- a/rpython/rlib/objectmodel.py
+++ b/rpython/rlib/objectmodel.py
@@ -301,6 +301,8 @@
                 return ctx.appcall(func, *args_w)
         if sandboxed_name:
             func._sandbox_external_name = sandboxed_name
+            # XXX THIS IS NOT CORRECT. Only do this when config.sandbox.
+            func._dont_inline_ = True
         return func
     return wrap
 

From noreply at buildbot.pypy.org  Fri Nov  7 20:09:23 2014
From: noreply at buildbot.pypy.org (amauryfa)
Date: Fri,  7 Nov 2014 20:09:23 +0100 (CET)
Subject: [pypy-commit] pypy more-rposix: Port more functions.
Message-ID: <20141107190923.71D571D28DC@cobra.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc 
Branch: more-rposix
Changeset: r74381:868a31be4ea7
Date: 2014-11-07 17:18 +0100
http://bitbucket.org/pypy/pypy/changeset/868a31be4ea7/

Log:	Port more functions.

diff --git a/rpython/rlib/rposix.py b/rpython/rlib/rposix.py
--- a/rpython/rlib/rposix.py
+++ b/rpython/rlib/rposix.py
@@ -166,24 +166,16 @@
 config = rffi_platform.configure(CConfig)
 globals().update(config)
 
-def external(name, args, result, **kwds):
-    return rffi.llexternal(name, args, result, compilation_info=eci, **kwds)
+def external(name, args, result, compilation_info=eci, **kwds):
+    return rffi.llexternal(name, args, result,
+                           compilation_info=compilation_info, **kwds)
 
 c_dup = external(UNDERSCORE_ON_WIN32 + 'dup', [rffi.INT], rffi.INT)
 c_dup2 = external(UNDERSCORE_ON_WIN32 + 'dup2', [rffi.INT, rffi.INT], rffi.INT)
 c_open = external(UNDERSCORE_ON_WIN32 + 'open',
                   [rffi.CCHARP, rffi.INT, rffi.MODE_T], rffi.INT)
-c_execv = external('execv', [rffi.CCHARP, rffi.CCHARPP], rffi.INT)
-c_execve = external('execve',
-                    [rffi.CCHARP, rffi.CCHARPP, rffi.CCHARPP], rffi.INT)
 c_getlogin = external('getlogin', [], rffi.CCHARP, releasegil=False)
 
-# Win32 specific functions
-c_spawnv = external('spawnv',
-                    [rffi.INT, rffi.CCHARP, rffi.CCHARPP], rffi.INT)
-c_spawnve = external('spawnve',
-                    [rffi.INT, rffi.CCHARP, rffi.CCHARPP, rffi.CCHARP],
-                     rffi.INT)
 # Win32 Unicode functions
 c_wopen = external(UNDERSCORE_ON_WIN32 + 'wopen',
                    [rffi.CWCHARP, rffi.INT, rffi.MODE_T], rffi.INT)
@@ -360,6 +352,15 @@
         fd = c_open(_as_bytes0(path), flags, mode)
     return handle_posix_error('open', fd)
         
+c_execv = external('execv', [rffi.CCHARP, rffi.CCHARPP], rffi.INT)
+c_execve = external('execve',
+                    [rffi.CCHARP, rffi.CCHARPP, rffi.CCHARPP], rffi.INT)
+c_spawnv = external('spawnv',
+                    [rffi.INT, rffi.CCHARP, rffi.CCHARPP], rffi.INT)
+c_spawnve = external('spawnve',
+                    [rffi.INT, rffi.CCHARP, rffi.CCHARPP, rffi.CCHARP],
+                     rffi.INT)
+
 @replace_os_function('execv')
 def execv(path, args):
     rstring.check_str0(path)
@@ -778,3 +779,61 @@
 def setresgid(rgid, egid, sgid):
     handle_posix_error('setresgid', c_setresgid(rgid, egid, sgid))
 
+#___________________________________________________________________
+
+c_chroot = external('chroot', [rffi.CCHARP], rffi.INT)
+
+ at replace_os_function('chroot')
+def chroot(path):
+    handle_posix_error('chroot', os_chroot(_as_bytes0(path)))
+
+CHARARRAY1 = lltype.FixedSizeArray(lltype.Char, 1)
+class CConfig:
+    _compilation_info_ = ExternalCompilationInfo(
+        includes = ['sys/utsname.h']
+    )
+    UTSNAME = rffi_platform.Struct('struct utsname', [
+        ('sysname',  CHARARRAY1),
+        ('nodename', CHARARRAY1),
+        ('release',  CHARARRAY1),
+        ('version',  CHARARRAY1),
+        ('machine',  CHARARRAY1)])
+config = rffi_platform.configure(CConfig)
+UTSNAMEP = lltype.Ptr(config['UTSNAME'])
+
+c_uname = external('uname', [UTSNAMEP], rffi.INT,
+                    compilation_info=CConfig._compilation_info_)
+
+ at replace_os_function('uname')
+def uname():
+    l_utsbuf = lltype.malloc(UTSNAMEP.TO, flavor='raw')
+    try:
+        handle_posix_error('uname', c_uname(l_utsbuf))
+        return (
+            rffi.charp2str(rffi.cast(rffi.CCHARP, l_utsbuf.c_sysname)),
+            rffi.charp2str(rffi.cast(rffi.CCHARP, l_utsbuf.c_nodename)),
+            rffi.charp2str(rffi.cast(rffi.CCHARP, l_utsbuf.c_release)),
+            rffi.charp2str(rffi.cast(rffi.CCHARP, l_utsbuf.c_version)),
+            rffi.charp2str(rffi.cast(rffi.CCHARP, l_utsbuf.c_machine)),
+        )
+    finally:
+        lltype.free(l_utsbuf, flavor='raw')
+
+c_makedev = external('makedev', [rffi.INT, rffi.INT], rffi.INT)
+c_major = external('major', [rffi.INT], rffi.INT)
+c_minor = external('minor', [rffi.INT], rffi.INT)
+
+ at replace_os_function('makedev')
+def makedev(maj, min):
+    return c_makedev(maj, min)
+
+ at replace_os_function('major')
+def major(dev):
+    return c_major(dev)
+
+ at replace_os_function('minor')
+def minor(dev):
+    return c_minor(dev)
+
+#___________________________________________________________________
+
diff --git a/rpython/rtyper/module/ll_os.py b/rpython/rtyper/module/ll_os.py
--- a/rpython/rtyper/module/ll_os.py
+++ b/rpython/rtyper/module/ll_os.py
@@ -235,54 +235,6 @@
         return extdef([int], int, llimpl=c_func_llimpl,
                       export_name='ll_os.ll_os_' + name)
 
-    @registering_if(os, 'chroot')
-    def register_os_chroot(self):
-        os_chroot = self.llexternal('chroot', [rffi.CCHARP], rffi.INT)
-        def chroot_llimpl(arg):
-            result = os_chroot(arg)
-            if result == -1:
-                raise OSError(rposix.get_errno(), "os_chroot failed")
-
-        return extdef([str0], None, export_name="ll_os.ll_os_chroot",
-                      llimpl=chroot_llimpl)
-
-    @registering_if(os, 'uname')
-    def register_os_uname(self):
-        CHARARRAY = lltype.FixedSizeArray(lltype.Char, 1)
-        class CConfig:
-            _compilation_info_ = ExternalCompilationInfo(
-                includes = ['sys/utsname.h']
-            )
-            UTSNAME = platform.Struct('struct utsname', [
-                ('sysname',  CHARARRAY),
-                ('nodename', CHARARRAY),
-                ('release',  CHARARRAY),
-                ('version',  CHARARRAY),
-                ('machine',  CHARARRAY)])
-        config = platform.configure(CConfig)
-        UTSNAMEP = lltype.Ptr(config['UTSNAME'])
-
-        os_uname = self.llexternal('uname', [UTSNAMEP], rffi.INT,
-                                   compilation_info=CConfig._compilation_info_)
-
-        def uname_llimpl():
-            l_utsbuf = lltype.malloc(UTSNAMEP.TO, flavor='raw')
-            result = os_uname(l_utsbuf)
-            if result == -1:
-                raise OSError(rposix.get_errno(), "os_uname failed")
-            retval = (
-                rffi.charp2str(rffi.cast(rffi.CCHARP, l_utsbuf.c_sysname)),
-                rffi.charp2str(rffi.cast(rffi.CCHARP, l_utsbuf.c_nodename)),
-                rffi.charp2str(rffi.cast(rffi.CCHARP, l_utsbuf.c_release)),
-                rffi.charp2str(rffi.cast(rffi.CCHARP, l_utsbuf.c_version)),
-                rffi.charp2str(rffi.cast(rffi.CCHARP, l_utsbuf.c_machine)),
-                )
-            lltype.free(l_utsbuf, flavor='raw')
-            return retval
-
-        return extdef([], (str, str, str, str, str),
-                      "ll_os.ll_uname", llimpl=uname_llimpl)
-
     @registering_if(os, 'sysconf')
     def register_os_sysconf(self):
         c_sysconf = self.llexternal('sysconf', [rffi.INT], rffi.LONG)
@@ -369,30 +321,6 @@
         return extdef([], (float, float, float),
                       "ll_os.ll_getloadavg", llimpl=getloadavg_llimpl)
 
-    @registering_if(os, 'makedev')
-    def register_os_makedev(self):
-        c_makedev = self.llexternal('makedev', [rffi.INT, rffi.INT], rffi.INT)
-        def makedev_llimpl(maj, min):
-            return c_makedev(maj, min)
-        return extdef([int, int], int,
-                      "ll_os.ll_makedev", llimpl=makedev_llimpl)
-
-    @registering_if(os, 'major')
-    def register_os_major(self):
-        c_major = self.llexternal('major', [rffi.INT], rffi.INT)
-        def major_llimpl(dev):
-            return c_major(dev)
-        return extdef([int], int,
-                      "ll_os.ll_major", llimpl=major_llimpl)
-
-    @registering_if(os, 'minor')
-    def register_os_minor(self):
-        c_minor = self.llexternal('minor', [rffi.INT], rffi.INT)
-        def minor_llimpl(dev):
-            return c_minor(dev)
-        return extdef([int], int,
-                      "ll_os.ll_minor", llimpl=minor_llimpl)
-
 # ------------------------------- os.read -------------------------------
 
     @registering(os.read)

From noreply at buildbot.pypy.org  Fri Nov  7 20:09:24 2014
From: noreply at buildbot.pypy.org (amauryfa)
Date: Fri,  7 Nov 2014 20:09:24 +0100 (CET)
Subject: [pypy-commit] pypy more-rposix: Port os.confstr &co
Message-ID: <20141107190924.96B531D28DC@cobra.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc 
Branch: more-rposix
Changeset: r74382:cd2c5d61bf3a
Date: 2014-11-07 17:52 +0100
http://bitbucket.org/pypy/pypy/changeset/cd2c5d61bf3a/

Log:	Port os.confstr &co

diff --git a/rpython/rlib/rposix.py b/rpython/rlib/rposix.py
--- a/rpython/rlib/rposix.py
+++ b/rpython/rlib/rposix.py
@@ -837,3 +837,56 @@
 
 #___________________________________________________________________
 
+c_sysconf = external('sysconf', [rffi.INT], rffi.LONG)
+c_fpathconf = external('fpathconf', [rffi.INT, rffi.INT], rffi.LONG)
+c_pathconf = external('pathconf', [rffi.CCHARP, rffi.INT], rffi.LONG)
+c_confstr = external('confstr',
+                     [rffi.INT, rffi.CCHARP, rffi.SIZE_T], rffi.SIZE_T)
+
+ at replace_os_function('sysconf')
+def sysconf(value):
+    set_errno(0)
+    res = c_sysconf(i)
+    if res == -1:
+        errno = get_errno()
+        if errno != 0:
+            raise OSError(errno, "sysconf failed")
+    return res
+
+ at replace_os_functions('fpathconf')
+def fpathconf(fd, value):
+    rposix.set_errno(0)
+    res = c_fpathconf(fd, value)
+    if res == -1:
+        errno = rposix.get_errno()
+        if errno != 0:
+            raise OSError(errno, "fpathconf failed")
+    return res
+
+ at replace_os_function('pathconf')
+def pathconf(path, value):
+    rposix.set_errno(0)
+    res = c_pathconf(_as_bytes0(path), value)
+    if res == -1:
+        errno = rposix.get_errno()
+        if errno != 0:
+            raise OSError(errno, "pathconf failed")
+    return res
+
+ at replace_os_function('confstr')
+def confstr(value):
+    rposix.set_errno(0)
+    n = c_confstr(value, lltype.nullptr(rffi.CCHARP.TO), 0)
+    if n > 0:
+        buf = lltype.malloc(rffi.CCHARP.TO, n, flavor='raw')
+        try:
+            c_confstr(value, buf, n)
+            return rffi.charp2strn(buf, n)
+        finally:
+            lltype.free(buf, flavor='raw')
+    else:
+        errno = rposix.get_errno()
+        if errno != 0:
+            raise OSError(errno, "confstr failed")
+        return None
+
diff --git a/rpython/rtyper/module/ll_os.py b/rpython/rtyper/module/ll_os.py
--- a/rpython/rtyper/module/ll_os.py
+++ b/rpython/rtyper/module/ll_os.py
@@ -235,76 +235,6 @@
         return extdef([int], int, llimpl=c_func_llimpl,
                       export_name='ll_os.ll_os_' + name)
 
-    @registering_if(os, 'sysconf')
-    def register_os_sysconf(self):
-        c_sysconf = self.llexternal('sysconf', [rffi.INT], rffi.LONG)
-
-        def sysconf_llimpl(i):
-            rposix.set_errno(0)
-            res = c_sysconf(i)
-            if res == -1:
-                errno = rposix.get_errno()
-                if errno != 0:
-                    raise OSError(errno, "sysconf failed")
-            return res
-        return extdef([int], int, "ll_os.ll_sysconf", llimpl=sysconf_llimpl)
-
-    @registering_if(os, 'fpathconf')
-    def register_os_fpathconf(self):
-        c_fpathconf = self.llexternal('fpathconf',
-                                      [rffi.INT, rffi.INT], rffi.LONG)
-
-        def fpathconf_llimpl(fd, i):
-            rposix.set_errno(0)
-            res = c_fpathconf(fd, i)
-            if res == -1:
-                errno = rposix.get_errno()
-                if errno != 0:
-                    raise OSError(errno, "fpathconf failed")
-            return res
-        return extdef([int, int], int, "ll_os.ll_fpathconf",
-                      llimpl=fpathconf_llimpl)
-
-    @registering_if(os, 'pathconf')
-    def register_os_pathconf(self):
-        c_pathconf = self.llexternal('pathconf',
-                                     [rffi.CCHARP, rffi.INT], rffi.LONG)
-
-        def pathconf_llimpl(path, i):
-            rposix.set_errno(0)
-            res = c_pathconf(path, i)
-            if res == -1:
-                errno = rposix.get_errno()
-                if errno != 0:
-                    raise OSError(errno, "pathconf failed")
-            return res
-        return extdef([str0, int], int, "ll_os.ll_pathconf",
-                      llimpl=pathconf_llimpl)
-
-    @registering_if(os, 'confstr')
-    def register_os_confstr(self):
-        c_confstr = self.llexternal('confstr', [rffi.INT, rffi.CCHARP,
-                                                rffi.SIZE_T], rffi.SIZE_T)
-
-        def confstr_llimpl(i):
-            rposix.set_errno(0)
-            n = c_confstr(i, lltype.nullptr(rffi.CCHARP.TO), 0)
-            n = rffi.cast(lltype.Signed, n)
-            if n > 0:
-                buf = lltype.malloc(rffi.CCHARP.TO, n, flavor='raw')
-                try:
-                    c_confstr(i, buf, n)
-                    return rffi.charp2strn(buf, n)
-                finally:
-                    lltype.free(buf, flavor='raw')
-            else:
-                errno = rposix.get_errno()
-                if errno != 0:
-                    raise OSError(errno, "confstr failed")
-                return None
-        return extdef([int], SomeString(can_be_None=True),
-                      "ll_os.ll_confstr", llimpl=confstr_llimpl)
-
     @registering_if(os, 'getloadavg')
     def register_os_getloadavg(self):
         AP = rffi.CArrayPtr(lltype.Float)

From noreply at buildbot.pypy.org  Fri Nov  7 20:09:25 2014
From: noreply at buildbot.pypy.org (amauryfa)
Date: Fri,  7 Nov 2014 20:09:25 +0100 (CET)
Subject: [pypy-commit] pypy more-rposix: Port fork() and forkpty()
Message-ID: <20141107190925.B6B941D28DC@cobra.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc 
Branch: more-rposix
Changeset: r74383:d7099109e3cc
Date: 2014-11-07 19:32 +0100
http://bitbucket.org/pypy/pypy/changeset/d7099109e3cc/

Log:	Port fork() and forkpty()

diff --git a/rpython/rlib/rposix.py b/rpython/rlib/rposix.py
--- a/rpython/rlib/rposix.py
+++ b/rpython/rlib/rposix.py
@@ -10,6 +10,7 @@
 from rpython.rlib import jit
 from rpython.translator.platform import platform
 from rpython.rlib import rstring
+from rpython.rlib import debug, rthread
 
 _WIN32 = sys.platform.startswith('win')
 UNDERSCORE_ON_WIN32 = '_' if _WIN32 else ''
@@ -137,12 +138,15 @@
 
 if _WIN32:
     includes = ['io.h', 'sys/utime.h', 'sys/types.h']
+    libraries = []
 else:
     includes = ['unistd.h',  'sys/types.h',
                 'utime.h', 'sys/time.h', 'sys/times.h',
                 'grp.h']
+    libraries = ['util']
 eci = ExternalCompilationInfo(
     includes=includes,
+    libraries=libraries,
 )
 
 class CConfig:
@@ -329,7 +333,7 @@
 
 @specialize.arg(0)
 def handle_posix_error(name, result):
-    if result < 0:
+    if intmask(result) < 0:
         raise OSError(get_errno(), '%s failed' % name)
     return intmask(result)
 
@@ -408,6 +412,54 @@
     rffi.free_charpp(l_args)
     return handle_posix_error('spawnve', childpid)
 
+c_fork = external('fork', [], rffi.PID_T, _nowrapper = True)
+c_openpty = external('openpty',
+                     [rffi.INTP, rffi.INTP, rffi.VOIDP, rffi.VOIDP, rffi.VOIDP],
+                     rffi.INT)
+c_forkpty = external('forkpty',
+                     [rffi.INTP, rffi.VOIDP, rffi.VOIDP, rffi.VOIDP],
+                     rffi.PID_T)
+
+ at replace_os_function('fork')
+def fork():
+    # NB. keep forkpty() up-to-date, too
+    ofs = debug.debug_offset()
+    opaqueaddr = rthread.gc_thread_before_fork()
+    childpid = c_fork()
+    rthread.gc_thread_after_fork(childpid, opaqueaddr)
+    childpid = handle_posix_error('fork', childpid)
+    if childpid == 0:
+        debug.debug_forked(ofs)
+    return childpid
+
+ at replace_os_function('openpty')
+def openpty():
+    master_p = lltype.malloc(rffi.INTP.TO, 1, flavor='raw')
+    slave_p = lltype.malloc(rffi.INTP.TO, 1, flavor='raw')
+    try:
+        handle_posix_error(
+            'openpty', c_openpty(master_p, slave_p, None, None, None))
+        return (intmask(master_p[0]), intmask(slave_p[0]))
+    finally:
+        lltype.free(master_p, flavor='raw')
+        lltype.free(slave_p, flavor='raw')
+
+ at replace_os_function('forkpty')
+def forkpty():
+    master_p = lltype.malloc(rffi.INTP.TO, 1, flavor='raw')
+    master_p[0] = rffi.cast(rffi.INT, -1)
+    try:
+        ofs = debug.debug_offset()
+        opaqueaddr = rthread.gc_thread_before_fork()
+        childpid = os_forkpty(master_p, None, None, None)
+        rthread.gc_thread_after_fork(childpid, opaqueaddr)
+        childpid = handle_posix_error('forkpty', childpid)
+        if childpid == 0:
+            debug.debug_forked(ofs)
+        return (childpid, master_p[0])
+    finally:
+        lltype.free(master_p, flavor='raw')
+
 @replace_os_function('getlogin')
 def getlogin():
     result = c_getlogin()
@@ -853,7 +905,7 @@
             raise OSError(errno, "sysconf failed")
     return res
 
- at replace_os_functions('fpathconf')
+ at replace_os_function('fpathconf')
 def fpathconf(fd, value):
     rposix.set_errno(0)
     res = c_fpathconf(fd, value)
diff --git a/rpython/rtyper/module/ll_os.py b/rpython/rtyper/module/ll_os.py
--- a/rpython/rtyper/module/ll_os.py
+++ b/rpython/rtyper/module/ll_os.py
@@ -980,78 +980,6 @@
         return extdef([str0, str0], s_None, llimpl=symlink_llimpl,
                       export_name="ll_os.ll_os_symlink")
 
-    @registering_if(os, 'fork')
-    def register_os_fork(self):
-        from rpython.rlib import debug, rthread
-        os_fork = self.llexternal('fork', [], rffi.PID_T,
-                                  _nowrapper = True)
-
-        def fork_llimpl():
-            # NB. keep forkpty() up-to-date, too
-            ofs = debug.debug_offset()
-            opaqueaddr = rthread.gc_thread_before_fork()
-            childpid = rffi.cast(lltype.Signed, os_fork())
-            rthread.gc_thread_after_fork(childpid, opaqueaddr)
-            if childpid == -1:
-                raise OSError(rposix.get_errno(), "os_fork failed")
-            if childpid == 0:
-                debug.debug_forked(ofs)
-            return rffi.cast(lltype.Signed, childpid)
-
-        return extdef([], int, llimpl=fork_llimpl,
-                      export_name="ll_os.ll_os_fork")
-
-    @registering_if(os, 'openpty')
-    def register_os_openpty(self):
-        os_openpty = self.llexternal(
-            'openpty',
-            [rffi.INTP, rffi.INTP, rffi.VOIDP, rffi.VOIDP, rffi.VOIDP],
-            rffi.INT,
-            compilation_info=ExternalCompilationInfo(libraries=['util']))
-        def openpty_llimpl():
-            master_p = lltype.malloc(rffi.INTP.TO, 1, flavor='raw')
-            slave_p = lltype.malloc(rffi.INTP.TO, 1, flavor='raw')
-            result = os_openpty(master_p, slave_p, None, None, None)
-            master_fd = master_p[0]
-            slave_fd = slave_p[0]
-            lltype.free(master_p, flavor='raw')
-            lltype.free(slave_p, flavor='raw')
-            if result == -1:
-                raise OSError(rposix.get_errno(), "os_openpty failed")
-            return (rffi.cast(lltype.Signed, master_fd),
-                    rffi.cast(lltype.Signed, slave_fd))
-
-        return extdef([], (int, int), "ll_os.ll_os_openpty",
-                      llimpl=openpty_llimpl)
-
-    @registering_if(os, 'forkpty')
-    def register_os_forkpty(self):
-        from rpython.rlib import debug, rthread
-        os_forkpty = self.llexternal(
-            'forkpty',
-            [rffi.INTP, rffi.VOIDP, rffi.VOIDP, rffi.VOIDP],
-            rffi.PID_T,
-            compilation_info=ExternalCompilationInfo(libraries=['util']))
-        def forkpty_llimpl():
-            master_p = lltype.malloc(rffi.INTP.TO, 1, flavor='raw')
-            master_p[0] = rffi.cast(rffi.INT, -1)
-            ofs = debug.debug_offset()
-            opaqueaddr = rthread.gc_thread_before_fork()
-            childpid = rffi.cast(lltype.Signed,
-                                 os_forkpty(master_p, None, None, None))
-            rthread.gc_thread_after_fork(childpid, opaqueaddr)
-            master_fd = master_p[0]
-            lltype.free(master_p, flavor='raw')
-            if childpid == -1:
-                raise OSError(rposix.get_errno(), "os_forkpty failed")
-            if childpid == 0:
-                debug.debug_forked(ofs)
-            return (rffi.cast(lltype.Signed, childpid),
-                    rffi.cast(lltype.Signed, master_fd))
-
-        return extdef([], (int, int), "ll_os.ll_os_forkpty",
-                      llimpl=forkpty_llimpl)
-
     @registering(os._exit)
     def register_os__exit(self):
         from rpython.rlib import debug

From noreply at buildbot.pypy.org  Fri Nov  7 20:09:27 2014
From: noreply at buildbot.pypy.org (amauryfa)
Date: Fri,  7 Nov 2014 20:09:27 +0100 (CET)
Subject: [pypy-commit] pypy more-rposix: os.waitpid(). Not tested on Windows
Message-ID: <20141107190927.0686C1D28DC@cobra.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc 
Branch: more-rposix
Changeset: r74384:fe529528d3fa
Date: 2014-11-07 19:52 +0100
http://bitbucket.org/pypy/pypy/changeset/fe529528d3fa/

Log:	os.waitpid(). Not tested on Windows

diff --git a/rpython/rlib/rposix.py b/rpython/rlib/rposix.py
--- a/rpython/rlib/rposix.py
+++ b/rpython/rlib/rposix.py
@@ -13,6 +13,7 @@
 from rpython.rlib import debug, rthread
 
 _WIN32 = sys.platform.startswith('win')
+_CYGWIN = sys.platform == 'cygwin'
 UNDERSCORE_ON_WIN32 = '_' if _WIN32 else ''
 
 class CConstantErrno(CConstant):
@@ -178,7 +179,6 @@
 c_dup2 = external(UNDERSCORE_ON_WIN32 + 'dup2', [rffi.INT, rffi.INT], rffi.INT)
 c_open = external(UNDERSCORE_ON_WIN32 + 'open',
                   [rffi.CCHARP, rffi.INT, rffi.MODE_T], rffi.INT)
-c_getlogin = external('getlogin', [], rffi.CCHARP, releasegil=False)
 
 # Win32 Unicode functions
 c_wopen = external(UNDERSCORE_ON_WIN32 + 'wopen',
@@ -460,6 +460,39 @@
     finally:
         lltype.free(master_p, flavor='raw')
 
+if _WIN32:
+    # emulate waitpid() with the _cwait() of Microsoft's compiler
+    c__cwait = external('_cwait',
+                        [rffi.INTP, rffi.PID_T, rffi.INT], rffi.PID_T)
+    def c_waitpid(pid, status_p, options):
+        result = c__cwait(status_p, pid, options)
+        # shift the status left a byte so this is more
+        # like the POSIX waitpid
+        status_p[0] = rffi.cast(rffi.INT, intmask(status_p[0]) << 8)
+        return result
+elif _CYGWIN:
+    c_waitpid = external('cygwin_waitpid',
+                         [rffi.PID_T, rffi.INTP, rffi.INT], rffi.PID_T)
+else:
+    c_waitpid = external('waitpid',
+                         [rffi.PID_T, rffi.INTP, rffi.INT], rffi.PID_T)
+
+ at replace_os_function('waitpid')
+def waitpid(pid, options):
+    status_p = lltype.malloc(rffi.INTP.TO, 1, flavor='raw')
+    status_p[0] = rffi.cast(rffi.INT, 0)
+    try:
+        result = handle_posix_error('waitpid',
+                                    c_waitpid(pid, status_p, options))
+        status = intmask(status_p[0])
+        return (result, status)
+    finally:
+        lltype.free(status_p, flavor='raw')
+
+c_getlogin = external('getlogin', [], rffi.CCHARP, releasegil=False)
+c_getloadavg = external('getloadavg', 
+                        [rffi.CArrayPtr(lltype.Float), rffi.INT], rffi.INT)
+
 @replace_os_function('getlogin')
 def getlogin():
     result = c_getlogin()
@@ -467,6 +500,16 @@
         raise OSError(get_errno(), "getlogin failed")
     return rffi.charp2str(result)
 
+ at replace_os_function('getloadavg')
+def getloadavg():
+    load = lltype.malloc(rffi.CArrayPtr(lltype.Float).TO, 3, flavor='raw')
+    try:
+        r = c_getloadavg(load, 3)
+        if r != 3:
+            raise OSError
+        return (load[0], load[1], load[2])
+    finally:
+        lltype.free(load, flavor='raw')
 
 #___________________________________________________________________
 
diff --git a/rpython/rtyper/module/ll_os.py b/rpython/rtyper/module/ll_os.py
--- a/rpython/rtyper/module/ll_os.py
+++ b/rpython/rtyper/module/ll_os.py
@@ -235,22 +235,6 @@
         return extdef([int], int, llimpl=c_func_llimpl,
                       export_name='ll_os.ll_os_' + name)
 
-    @registering_if(os, 'getloadavg')
-    def register_os_getloadavg(self):
-        AP = rffi.CArrayPtr(lltype.Float)
-        c_getloadavg = self.llexternal('getloadavg', [AP, rffi.INT], rffi.INT)
-
-        def getloadavg_llimpl():
-            load = lltype.malloc(AP.TO, 3, flavor='raw')
-            r = c_getloadavg(load, 3)
-            result_tuple = load[0], load[1], load[2]
-            lltype.free(load, flavor='raw')
-            if r != 3:
-                raise OSError
-            return result_tuple
-        return extdef([], (float, float, float),
-                      "ll_os.ll_getloadavg", llimpl=getloadavg_llimpl)
-
 # ------------------------------- os.read -------------------------------
 
     @registering(os.read)
@@ -678,50 +662,6 @@
                       "ll_os.ll_os_readlink",
                       llimpl=os_readlink_llimpl)
 
-    @registering(os.waitpid)
-    def register_os_waitpid(self):
-        if sys.platform.startswith('win'):
-            # emulate waitpid() with the _cwait() of Microsoft's compiler
-            os__cwait = self.llexternal('_cwait',
-                                        [rffi.INTP, rffi.PID_T, rffi.INT],
-                                        rffi.PID_T)
-            def os_waitpid(pid, status_p, options):
-                result = os__cwait(status_p, pid, options)
-                # shift the status left a byte so this is more
-                # like the POSIX waitpid
-                tmp = rffi.cast(rffi.SIGNED, status_p[0])
-                tmp <<= 8
-                status_p[0] = rffi.cast(rffi.INT, tmp)
-                return result
-        else:
-            # Posix
-            if _CYGWIN:
-                os_waitpid = self.llexternal('cygwin_waitpid',
-                                             [rffi.PID_T, rffi.INTP, rffi.INT],
-                                             rffi.PID_T)
-            else:
-                os_waitpid = self.llexternal('waitpid',
-                                             [rffi.PID_T, rffi.INTP, rffi.INT],
-                                             rffi.PID_T)
-
-        def os_waitpid_llimpl(pid, options):
-            status_p = lltype.malloc(rffi.INTP.TO, 1, flavor='raw')
-            status_p[0] = rffi.cast(rffi.INT, 0)
-            result = os_waitpid(rffi.cast(rffi.PID_T, pid),
-                                status_p,
-                                rffi.cast(rffi.INT, options))
-            result = rffi.cast(lltype.Signed, result)
-            status = status_p[0]
-            lltype.free(status_p, flavor='raw')
-            if result == -1:
-                raise OSError(rposix.get_errno(), "os_waitpid failed")
-            return (rffi.cast(lltype.Signed, result),
-                    rffi.cast(lltype.Signed, status))
-
-        return extdef([int, int], (int, int),
-                      "ll_os.ll_os_waitpid",
-                      llimpl=os_waitpid_llimpl)
-
     @registering(os.isatty)
     def register_os_isatty(self):
         os_isatty = self.llexternal(UNDERSCORE_ON_WIN32 + 'isatty',

From noreply at buildbot.pypy.org  Fri Nov  7 20:09:28 2014
From: noreply at buildbot.pypy.org (amauryfa)
Date: Fri,  7 Nov 2014 20:09:28 +0100 (CET)
Subject: [pypy-commit] pypy more-rposix: Port os.read(), write(), close()
Message-ID: <20141107190928.270C81D28DC@cobra.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc 
Branch: more-rposix
Changeset: r74385:5804e9b2ef43
Date: 2014-11-07 20:08 +0100
http://bitbucket.org/pypy/pypy/changeset/5804e9b2ef43/

Log:	Port os.read(), write(), close() + fixes

diff --git a/rpython/rlib/rposix.py b/rpython/rlib/rposix.py
--- a/rpython/rlib/rposix.py
+++ b/rpython/rlib/rposix.py
@@ -1,5 +1,6 @@
 import os
 import sys
+import errno
 from rpython.rtyper.lltypesystem.rffi import CConstant, CExternVariable, INT
 from rpython.rtyper.lltypesystem import lltype, ll2ctypes, rffi
 from rpython.rtyper.tool import rffi_platform
@@ -347,6 +348,8 @@
     validate_fd(fd)
     handle_posix_error('dup2', c_dup2(fd, newfd))
 
+#___________________________________________________________________
+
 @replace_os_function('open')
 @specialize.argtype(0)
 def open(path, flags, mode):
@@ -355,7 +358,38 @@
     else:
         fd = c_open(_as_bytes0(path), flags, mode)
     return handle_posix_error('open', fd)
+
+c_read = external(UNDERSCORE_ON_WIN32 + 'read',
+                  [rffi.INT, rffi.VOIDP, rffi.SIZE_T], rffi.SIZE_T)
+c_write = external(UNDERSCORE_ON_WIN32 + 'write',
+                   [rffi.INT, rffi.VOIDP, rffi.SIZE_T], rffi.SIZE_T)
+c_close = external(UNDERSCORE_ON_WIN32 + 'close', [rffi.INT], rffi.INT,
+                   releasegil=False)
+
+ at replace_os_function('read')
+def read(fd, count):
+    if count < 0:
+        raise OSError(errno.EINVAL, None)
+    validate_fd(fd)
+    with rffi.scoped_alloc_buffer(count) as buf:
+        void_buf = rffi.cast(rffi.VOIDP, buf.raw)
+        got = handle_posix_error('read', c_read(fd, void_buf, count))
+        return buf.str(got)
+
+ at replace_os_function('write')
+def write(fd, data):
+    count = len(data)
+    validate_fd(fd)
+    with rffi.scoped_nonmovingbuffer(data) as buf:
+        return handle_posix_error('write', c_write(fd, buf, count))
+
+ at replace_os_function('close')
+def close(fd):
+    validate_fd(fd)
+    handle_posix_error('close', c_close(fd))
         
+#___________________________________________________________________
+
 c_execv = external('execv', [rffi.CCHARP, rffi.CCHARPP], rffi.INT)
 c_execve = external('execve',
                     [rffi.CCHARP, rffi.CCHARPP, rffi.CCHARPP], rffi.INT)
@@ -451,7 +485,7 @@
     try:
         ofs = debug.debug_offset()
         opaqueaddr = rthread.gc_thread_before_fork()
-        childpid = os_forkpty(master_p, None, None, None)
+        childpid = c_forkpty(master_p, None, None, None)
         rthread.gc_thread_after_fork(childpid, opaqueaddr)
         childpid = handle_posix_error('forkpty', childpid)
         if childpid == 0:
@@ -880,7 +914,7 @@
 
 @replace_os_function('chroot')
 def chroot(path):
-    handle_posix_error('chroot', os_chroot(_as_bytes0(path)))
+    handle_posix_error('chroot', c_chroot(_as_bytes0(path)))
 
 CHARARRAY1 = lltype.FixedSizeArray(lltype.Char, 1)
 class CConfig:
@@ -941,7 +975,7 @@
 @replace_os_function('sysconf')
 def sysconf(value):
     set_errno(0)
-    res = c_sysconf(i)
+    res = c_sysconf(value)
     if res == -1:
         errno = get_errno()
         if errno != 0:
@@ -950,28 +984,28 @@
 
 @replace_os_function('fpathconf')
 def fpathconf(fd, value):
-    rposix.set_errno(0)
+    set_errno(0)
     res = c_fpathconf(fd, value)
     if res == -1:
-        errno = rposix.get_errno()
+        errno = get_errno()
         if errno != 0:
             raise OSError(errno, "fpathconf failed")
     return res
 
 @replace_os_function('pathconf')
 def pathconf(path, value):
-    rposix.set_errno(0)
+    set_errno(0)
     res = c_pathconf(_as_bytes0(path), value)
     if res == -1:
-        errno = rposix.get_errno()
+        errno = get_errno()
         if errno != 0:
             raise OSError(errno, "pathconf failed")
     return res
 
 @replace_os_function('confstr')
 def confstr(value):
-    rposix.set_errno(0)
-    n = c_confstr(value, lltype.nullptr(rffi.CCHARP.TO), 0)
+    set_errno(0)
+    n = intmask(c_confstr(value, lltype.nullptr(rffi.CCHARP.TO), 0))
     if n > 0:
         buf = lltype.malloc(rffi.CCHARP.TO, n, flavor='raw')
         try:
@@ -980,7 +1014,7 @@
         finally:
             lltype.free(buf, flavor='raw')
     else:
-        errno = rposix.get_errno()
+        errno = get_errno()
         if errno != 0:
             raise OSError(errno, "confstr failed")
         return None
diff --git a/rpython/rtyper/module/ll_os.py b/rpython/rtyper/module/ll_os.py
--- a/rpython/rtyper/module/ll_os.py
+++ b/rpython/rtyper/module/ll_os.py
@@ -237,60 +237,6 @@
 
 # ------------------------------- os.read -------------------------------
 
-    @registering(os.read)
-    def register_os_read(self):
-        os_read = self.llexternal(UNDERSCORE_ON_WIN32 + 'read',
-                                  [rffi.INT, rffi.VOIDP, rffi.SIZE_T],
-                                  rffi.SIZE_T)
-
-        def os_read_llimpl(fd, count):
-            if count < 0:
-                raise OSError(errno.EINVAL, None)
-            rposix.validate_fd(fd)
-            with rffi.scoped_alloc_buffer(count) as buf:
-                void_buf = rffi.cast(rffi.VOIDP, buf.raw)
-                got = rffi.cast(lltype.Signed, os_read(fd, void_buf, count))
-                if got < 0:
-                    raise OSError(rposix.get_errno(), "os_read failed")
-                return buf.str(got)
-
-        return extdef([int, int], SomeString(can_be_None=True),
-                      "ll_os.ll_os_read", llimpl=os_read_llimpl)
-
-    @registering(os.write)
-    def register_os_write(self):
-        os_write = self.llexternal(UNDERSCORE_ON_WIN32 + 'write',
-                                   [rffi.INT, rffi.VOIDP, rffi.SIZE_T],
-                                   rffi.SIZE_T)
-
-        def os_write_llimpl(fd, data):
-            count = len(data)
-            rposix.validate_fd(fd)
-            with rffi.scoped_nonmovingbuffer(data) as buf:
-                written = rffi.cast(lltype.Signed, os_write(
-                    rffi.cast(rffi.INT, fd),
-                    buf, rffi.cast(rffi.SIZE_T, count)))
-                if written < 0:
-                    raise OSError(rposix.get_errno(), "os_write failed")
-            return written
-
-        return extdef([int, str], SomeInteger(nonneg=True),
-                      "ll_os.ll_os_write", llimpl=os_write_llimpl)
-
-    @registering(os.close)
-    def register_os_close(self):
-        os_close = self.llexternal(UNDERSCORE_ON_WIN32 + 'close', [rffi.INT],
-                                   rffi.INT, releasegil=False)
-
-        def close_llimpl(fd):
-            rposix.validate_fd(fd)
-            error = rffi.cast(lltype.Signed, os_close(rffi.cast(rffi.INT, fd)))
-            if error == -1:
-                raise OSError(rposix.get_errno(), "close failed")
-
-        return extdef([int], s_None, llimpl=close_llimpl,
-                      export_name="ll_os.ll_os_close")
-
     @registering(os.lseek)
     def register_os_lseek(self):
         if sys.platform.startswith('win'):

From noreply at buildbot.pypy.org  Fri Nov  7 22:37:13 2014
From: noreply at buildbot.pypy.org (amauryfa)
Date: Fri,  7 Nov 2014 22:37:13 +0100 (CET)
Subject: [pypy-commit] pypy more-rposix: os.lseek(),
 use widen() instead of intmask(), fix some signatures.
Message-ID: <20141107213713.4F9F01D28DC@cobra.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc 
Branch: more-rposix
Changeset: r74386:4d3191787699
Date: 2014-11-07 21:13 +0100
http://bitbucket.org/pypy/pypy/changeset/4d3191787699/

Log:	os.lseek(), use widen() instead of intmask(), fix some signatures.

diff --git a/rpython/rlib/rposix.py b/rpython/rlib/rposix.py
--- a/rpython/rlib/rposix.py
+++ b/rpython/rlib/rposix.py
@@ -5,7 +5,7 @@
 from rpython.rtyper.lltypesystem import lltype, ll2ctypes, rffi
 from rpython.rtyper.tool import rffi_platform
 from rpython.translator.tool.cbuild import ExternalCompilationInfo
-from rpython.rlib.rarithmetic import intmask
+from rpython.rlib.rarithmetic import intmask, widen
 from rpython.rlib.objectmodel import (
     specialize, enforceargs, register_replacement_for)
 from rpython.rlib import jit
@@ -153,6 +153,10 @@
 
 class CConfig:
     _compilation_info_ = eci
+    SEEK_SET = rffi_platform.DefinedConstantInteger('SEEK_SET')
+    SEEK_CUR = rffi_platform.DefinedConstantInteger('SEEK_CUR')
+    SEEK_END = rffi_platform.DefinedConstantInteger('SEEK_END')
+
     HAVE_UTIMES = rffi_platform.Has('utimes')
     UTIMBUF = rffi_platform.Struct('struct %sutimbuf' % UNDERSCORE_ON_WIN32,
                                    [('actime', rffi.INT),
@@ -334,9 +338,10 @@
 
 @specialize.arg(0)
 def handle_posix_error(name, result):
-    if intmask(result) < 0:
+    result = widen(result)
+    if result < 0:
         raise OSError(get_errno(), '%s failed' % name)
-    return intmask(result)
+    return result
 
 @replace_os_function('dup')
 def dup(fd):
@@ -360,9 +365,9 @@
     return handle_posix_error('open', fd)
 
 c_read = external(UNDERSCORE_ON_WIN32 + 'read',
-                  [rffi.INT, rffi.VOIDP, rffi.SIZE_T], rffi.SIZE_T)
+                  [rffi.INT, rffi.VOIDP, rffi.SIZE_T], rffi.SSIZE_T)
 c_write = external(UNDERSCORE_ON_WIN32 + 'write',
-                   [rffi.INT, rffi.VOIDP, rffi.SIZE_T], rffi.SIZE_T)
+                   [rffi.INT, rffi.VOIDP, rffi.SIZE_T], rffi.SSIZE_T)
 c_close = external(UNDERSCORE_ON_WIN32 + 'close', [rffi.INT], rffi.INT,
                    releasegil=False)
 
@@ -387,7 +392,23 @@
 def close(fd):
     validate_fd(fd)
     handle_posix_error('close', c_close(fd))
-        
+
+c_lseek = external('_lseeki64' if _WIN32 else 'lseek',
+                   [rffi.INT, rffi.LONGLONG, rffi.INT], rffi.LONGLONG,
+                   macro=True)
+
+ at replace_os_function('lseek')
+def lseek_llimpl(fd, pos, how):
+    validate_fd(fd)
+    if SEEK_SET is not None:
+        if how == 0:
+            how = SEEK_SET
+        elif how == 1: 
+            how = SEEK_CUR
+        elif how == 2:
+            how = SEEK_END
+    return handle_posix_error('lseek', c_lseek(fd, pos, how))
+
 #___________________________________________________________________
 
 c_execv = external('execv', [rffi.CCHARP, rffi.CCHARPP], rffi.INT)
@@ -473,7 +494,7 @@
     try:
         handle_posix_error(
             'openpty', c_openpty(master_p, slave_p, None, None, None))
-        return (intmask(master_p[0]), intmask(slave_p[0]))
+        return (widen(master_p[0]), widen(slave_p[0]))
     finally:
         lltype.free(master_p, flavor='raw')
         lltype.free(slave_p, flavor='raw')
@@ -502,7 +523,7 @@
         result = c__cwait(status_p, pid, options)
         # shift the status left a byte so this is more
         # like the POSIX waitpid
-        status_p[0] = rffi.cast(rffi.INT, intmask(status_p[0]) << 8)
+        status_p[0] = rffi.cast(rffi.INT, widen(status_p[0]) << 8)
         return result
 elif _CYGWIN:
     c_waitpid = external('cygwin_waitpid',
@@ -518,7 +539,7 @@
     try:
         result = handle_posix_error('waitpid',
                                     c_waitpid(pid, status_p, options))
-        status = intmask(status_p[0])
+        status = widen(status_p[0])
         return (result, status)
     finally:
         lltype.free(status_p, flavor='raw')
@@ -727,11 +748,11 @@
 
 @replace_os_function('getpid')
 def getpid():
-    return intmask(c_getpid())
+    return widen(c_getpid())
 
 @replace_os_function('getppid')
 def getppid():
-    return intmask(c_getppid())
+    return widen(c_getppid())
 
 @replace_os_function('setsid')
 def setsid():
@@ -764,7 +785,7 @@
     groups = lltype.malloc(PID_GROUPS_T.TO, n, flavor='raw')
     try:
         n = handle_posix_error('getgroups', c_getgroups(n, groups))
-        return [intmask(groups[i]) for i in range(n)]
+        return [widen(groups[i]) for i in range(n)]
     finally:
         lltype.free(groups, flavor='raw')
 
@@ -884,7 +905,7 @@
                            c_getresuid(rffi.ptradd(out, 0),
                                        rffi.ptradd(out, 1),
                                        rffi.ptradd(out, 2)))
-        return (intmask(out[0]), intmask(out[1]), intmask(out[2]))
+        return (widen(out[0]), widen(out[1]), widen(out[2]))
     finally:
         lltype.free(out, flavor='raw')
 
@@ -896,7 +917,7 @@
                            c_getresgid(rffi.ptradd(out, 0),
                                        rffi.ptradd(out, 1),
                                        rffi.ptradd(out, 2)))
-        return (intmask(out[0]), intmask(out[1]), intmask(out[2]))
+        return (widen(out[0]), widen(out[1]), widen(out[2]))
     finally:
         lltype.free(out, flavor='raw')
 
diff --git a/rpython/rtyper/module/ll_os.py b/rpython/rtyper/module/ll_os.py
--- a/rpython/rtyper/module/ll_os.py
+++ b/rpython/rtyper/module/ll_os.py
@@ -237,49 +237,6 @@
 
 # ------------------------------- os.read -------------------------------
 
-    @registering(os.lseek)
-    def register_os_lseek(self):
-        if sys.platform.startswith('win'):
-            funcname = '_lseeki64'
-        else:
-            funcname = 'lseek'
-        if self.SEEK_SET is not None:
-            SEEK_SET = self.SEEK_SET
-            SEEK_CUR = self.SEEK_CUR
-            SEEK_END = self.SEEK_END
-        else:
-            SEEK_SET, SEEK_CUR, SEEK_END = 0, 1, 2
-        if (SEEK_SET, SEEK_CUR, SEEK_END) != (0, 1, 2):
-            # Turn 0, 1, 2 into SEEK_{SET,CUR,END}
-            def fix_seek_arg(n):
-                if n == 0: return SEEK_SET
-                if n == 1: return SEEK_CUR
-                if n == 2: return SEEK_END
-                return n
-        else:
-            def fix_seek_arg(n):
-                return n
-
-        os_lseek = self.llexternal(funcname,
-                                   [rffi.INT, rffi.LONGLONG, rffi.INT],
-                                   rffi.LONGLONG, macro=True)
-
-        def lseek_llimpl(fd, pos, how):
-            rposix.validate_fd(fd)
-            how = fix_seek_arg(how)
-            res = os_lseek(rffi.cast(rffi.INT,      fd),
-                           rffi.cast(rffi.LONGLONG, pos),
-                           rffi.cast(rffi.INT,      how))
-            res = rffi.cast(lltype.SignedLongLong, res)
-            if res < 0:
-                raise OSError(rposix.get_errno(), "os_lseek failed")
-            return res
-
-        return extdef([int, r_longlong, int],
-                      r_longlong,
-                      llimpl = lseek_llimpl,
-                      export_name = "ll_os.ll_os_lseek")
-
     @registering_if(os, 'ftruncate')
     def register_os_ftruncate(self):
         os_ftruncate = self.llexternal('ftruncate',

From noreply at buildbot.pypy.org  Fri Nov  7 22:37:14 2014
From: noreply at buildbot.pypy.org (amauryfa)
Date: Fri,  7 Nov 2014 22:37:14 +0100 (CET)
Subject: [pypy-commit] pypy more-rposix: os.ftruncate(), os.fsync()
Message-ID: <20141107213714.8C4301D28DC@cobra.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc 
Branch: more-rposix
Changeset: r74387:aaf7d91a3993
Date: 2014-11-07 21:26 +0100
http://bitbucket.org/pypy/pypy/changeset/aaf7d91a3993/

Log:	os.ftruncate(), os.fsync()

diff --git a/rpython/rlib/rposix.py b/rpython/rlib/rposix.py
--- a/rpython/rlib/rposix.py
+++ b/rpython/rlib/rposix.py
@@ -409,6 +409,26 @@
             how = SEEK_END
     return handle_posix_error('lseek', c_lseek(fd, pos, how))
 
+c_ftruncate = external('ftruncate', [rffi.INT, rffi.LONGLONG], rffi.INT,
+                       macro=True)
+c_fsync = external('fsync' if not _WIN32 else '_commit', [rffi.INT], rffi.INT)
+c_fdatasync = external('fdatasync', [rffi.INT], rffi.INT)
+
+ at replace_os_function('ftruncate')
+def ftruncate(fd, length):
+    validate_fd(fd)
+    handle_posix_error('ftruncate', c_ftruncate(fd, length))
+
+ at replace_os_function('fsync')
+def fsync(fd):
+    validate_fd(fd)
+    handle_posix_error('fsync', c_fsync(fd))
+
+ at replace_os_function('fdatasync')
+def fdatasync(fd):
+    rposix.validate_fd(fd)
+    handle_posix_error('fdatasync', c_fdatasync(fd))
+
 #___________________________________________________________________
 
 c_execv = external('execv', [rffi.CCHARP, rffi.CCHARPP], rffi.INT)
diff --git a/rpython/rtyper/module/ll_os.py b/rpython/rtyper/module/ll_os.py
--- a/rpython/rtyper/module/ll_os.py
+++ b/rpython/rtyper/module/ll_os.py
@@ -173,116 +173,6 @@
                 separate_module_sources = ["\n".join(defs)]
             ))
 
-    # a simple, yet useful factory
-    def extdef_for_os_function_returning_int(self, name, **kwds):
-        c_func = self.llexternal(name, [], rffi.INT, **kwds)
-        def c_func_llimpl():
-            res = rffi.cast(rffi.SIGNED, c_func())
-            if res == -1:
-                raise OSError(rposix.get_errno(), "%s failed" % name)
-            return res
-        c_func_llimpl.func_name = name + '_llimpl'
-
-        return extdef([], int, llimpl=c_func_llimpl,
-                      export_name='ll_os.ll_os_' + name)
-
-    def extdef_for_os_function_accepting_int(self, name, **kwds):
-        c_func = self.llexternal(name, [rffi.INT], rffi.INT, **kwds)
-        def c_func_llimpl(arg):
-            res = rffi.cast(rffi.SIGNED, c_func(arg))
-            if res == -1:
-                raise OSError(rposix.get_errno(), "%s failed" % name)
-
-        c_func_llimpl.func_name = name + '_llimpl'
-
-        return extdef([int], None, llimpl=c_func_llimpl,
-                      export_name='ll_os.ll_os_' + name)
-
-    def extdef_for_os_function_accepting_2int(self, name, **kwds):
-        c_func = self.llexternal(name, [rffi.INT, rffi.INT], rffi.INT, **kwds)
-        def c_func_llimpl(arg, arg2):
-            res = rffi.cast(rffi.SIGNED, c_func(arg, arg2))
-            if res == -1:
-                raise OSError(rposix.get_errno(), "%s failed" % name)
-
-        c_func_llimpl.func_name = name + '_llimpl'
-
-        return extdef([int, int], None, llimpl=c_func_llimpl,
-                      export_name='ll_os.ll_os_' + name)
-
-    def extdef_for_os_function_accepting_0int(self, name, **kwds):
-        c_func = self.llexternal(name, [], rffi.INT, **kwds)
-        def c_func_llimpl():
-            res = rffi.cast(rffi.SIGNED, c_func())
-            if res == -1:
-                raise OSError(rposix.get_errno(), "%s failed" % name)
-
-        c_func_llimpl.func_name = name + '_llimpl'
-
-        return extdef([], None, llimpl=c_func_llimpl,
-                      export_name='ll_os.ll_os_' + name)
-
-    def extdef_for_os_function_int_to_int(self, name, **kwds):
-        c_func = self.llexternal(name, [rffi.INT], rffi.INT, **kwds)
-        def c_func_llimpl(arg):
-            res = rffi.cast(rffi.SIGNED, c_func(arg))
-            if res == -1:
-                raise OSError(rposix.get_errno(), "%s failed" % name)
-            return res
-
-        c_func_llimpl.func_name = name + '_llimpl'
-
-        return extdef([int], int, llimpl=c_func_llimpl,
-                      export_name='ll_os.ll_os_' + name)
-
-# ------------------------------- os.read -------------------------------
-
-    @registering_if(os, 'ftruncate')
-    def register_os_ftruncate(self):
-        os_ftruncate = self.llexternal('ftruncate',
-                                       [rffi.INT, rffi.LONGLONG], rffi.INT, macro=True)
-
-        def ftruncate_llimpl(fd, length):
-            rposix.validate_fd(fd)
-            res = rffi.cast(rffi.LONG,
-                            os_ftruncate(rffi.cast(rffi.INT, fd),
-                                         rffi.cast(rffi.LONGLONG, length)))
-            if res < 0:
-                raise OSError(rposix.get_errno(), "os_ftruncate failed")
-
-        return extdef([int, r_longlong], s_None,
-                      llimpl = ftruncate_llimpl,
-                      export_name = "ll_os.ll_os_ftruncate")
-
-    @registering_if(os, 'fsync')
-    def register_os_fsync(self):
-        if not _WIN32:
-            os_fsync = self.llexternal('fsync', [rffi.INT], rffi.INT)
-        else:
-            os_fsync = self.llexternal('_commit', [rffi.INT], rffi.INT)
-
-        def fsync_llimpl(fd):
-            rposix.validate_fd(fd)
-            res = rffi.cast(rffi.SIGNED, os_fsync(rffi.cast(rffi.INT, fd)))
-            if res < 0:
-                raise OSError(rposix.get_errno(), "fsync failed")
-        return extdef([int], s_None,
-                      llimpl=fsync_llimpl,
-                      export_name="ll_os.ll_os_fsync")
-
-    @registering_if(os, 'fdatasync')
-    def register_os_fdatasync(self):
-        os_fdatasync = self.llexternal('fdatasync', [rffi.INT], rffi.INT)
-
-        def fdatasync_llimpl(fd):
-            rposix.validate_fd(fd)
-            res = rffi.cast(rffi.SIGNED, os_fdatasync(rffi.cast(rffi.INT, fd)))
-            if res < 0:
-                raise OSError(rposix.get_errno(), "fdatasync failed")
-        return extdef([int], s_None,
-                      llimpl=fdatasync_llimpl,
-                      export_name="ll_os.ll_os_fdatasync")
-
     @registering_if(os, 'fchdir')
     def register_os_fchdir(self):
         os_fchdir = self.llexternal('fchdir', [rffi.INT], rffi.INT)

From noreply at buildbot.pypy.org  Fri Nov  7 22:37:15 2014
From: noreply at buildbot.pypy.org (amauryfa)
Date: Fri,  7 Nov 2014 22:37:15 +0100 (CET)
Subject: [pypy-commit] pypy more-rposix: os.access(), os.fchdir()
Message-ID: <20141107213715.C45641D28DC@cobra.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc 
Branch: more-rposix
Changeset: r74388:ae00e9e0b19b
Date: 2014-11-07 21:41 +0100
http://bitbucket.org/pypy/pypy/changeset/ae00e9e0b19b/

Log:	os.access(), os.fchdir()

diff --git a/rpython/rlib/rposix.py b/rpython/rlib/rposix.py
--- a/rpython/rlib/rposix.py
+++ b/rpython/rlib/rposix.py
@@ -398,7 +398,7 @@
                    macro=True)
 
 @replace_os_function('lseek')
-def lseek_llimpl(fd, pos, how):
+def lseek(fd, pos, how):
     validate_fd(fd)
     if SEEK_SET is not None:
         if how == 0:
@@ -426,11 +426,36 @@
 
 @replace_os_function('fdatasync')
 def fdatasync(fd):
-    rposix.validate_fd(fd)
+    validate_fd(fd)
     handle_posix_error('fdatasync', c_fdatasync(fd))
 
 #___________________________________________________________________
 
+c_fchdir = external('fchdir', [rffi.INT], rffi.INT)
+c_access = external(UNDERSCORE_ON_WIN32 + 'access',
+                    [rffi.CCHARP, rffi.INT], rffi.INT)
+c_waccess = external(UNDERSCORE_ON_WIN32 + 'waccess',
+                     [rffi.CWCHARP, rffi.INT], rffi.INT)
+
+ at replace_os_function('fchdir')
+def fchdir(fd):
+    validate_fd(fd)
+    handle_posix_error('fchdir', c_fchdir(fd))
+
+ at replace_os_function('access')
+ at specialize.argtype(0)
+def access(path, mode):
+    if _WIN32:
+        # All files are executable on Windows
+        mode = mode & ~os.X_OK
+    if _prefer_unicode(path):
+        error = c_waccess(_as_unicode0(path), mode)
+    else:
+        error = c_access(_as_bytes0(path), mode)
+    return error == 0
+
+#___________________________________________________________________
+
 c_execv = external('execv', [rffi.CCHARP, rffi.CCHARPP], rffi.INT)
 c_execve = external('execve',
                     [rffi.CCHARP, rffi.CCHARPP, rffi.CCHARPP], rffi.INT)
diff --git a/rpython/rtyper/module/ll_os.py b/rpython/rtyper/module/ll_os.py
--- a/rpython/rtyper/module/ll_os.py
+++ b/rpython/rtyper/module/ll_os.py
@@ -173,39 +173,6 @@
                 separate_module_sources = ["\n".join(defs)]
             ))
 
-    @registering_if(os, 'fchdir')
-    def register_os_fchdir(self):
-        os_fchdir = self.llexternal('fchdir', [rffi.INT], rffi.INT)
-
-        def fchdir_llimpl(fd):
-            rposix.validate_fd(fd)
-            res = rffi.cast(rffi.SIGNED, os_fchdir(rffi.cast(rffi.INT, fd)))
-            if res < 0:
-                raise OSError(rposix.get_errno(), "fchdir failed")
-        return extdef([int], s_None,
-                      llimpl=fchdir_llimpl,
-                      export_name="ll_os.ll_os_fchdir")
-
-    @registering_str_unicode(os.access)
-    def register_os_access(self, traits):
-        os_access = self.llexternal(traits.posix_function_name('access'),
-                                    [traits.CCHARP, rffi.INT],
-                                    rffi.INT)
-
-        if sys.platform.startswith('win'):
-            # All files are executable on Windows
-            def access_llimpl(path, mode):
-                mode = mode & ~os.X_OK
-                error = rffi.cast(lltype.Signed, os_access(path, mode))
-                return error == 0
-        else:
-            def access_llimpl(path, mode):
-                error = rffi.cast(lltype.Signed, os_access(path, mode))
-                return error == 0
-
-        return extdef([traits.str0, int], s_Bool, llimpl=access_llimpl,
-                      export_name=traits.ll_os_name("access"))
-
     @registering_str_unicode(getattr(posix, '_getfullpathname', None),
                              condition=sys.platform=='win32')
     def register_posix__getfullpathname(self, traits):
diff --git a/rpython/rtyper/module/test/test_ll_os.py b/rpython/rtyper/module/test/test_ll_os.py
--- a/rpython/rtyper/module/test/test_ll_os.py
+++ b/rpython/rtyper/module/test/test_ll_os.py
@@ -21,7 +21,7 @@
     fd.close()
 
     for mode in os.R_OK, os.W_OK, os.X_OK, os.R_OK | os.W_OK | os.X_OK:
-        result = getllimpl(os.access)(filename, mode)
+        result = rposix.access(filename, mode)
         assert result == os.access(filename, mode)
 
 
@@ -217,62 +217,54 @@
     fname = str(udir.join('os_test.txt'))
     fd = os.open(fname, os.O_WRONLY|os.O_CREAT, 0777)
     assert fd >= 0
-    f = getllimpl(os.write)
-    f(fd, 'Hello world')
+    rposix.write(fd, 'Hello world')
     os.close(fd)
     with open(fname) as fid:
         assert fid.read() == "Hello world"
     fd = os.open(fname, os.O_WRONLY|os.O_CREAT, 0777)
     os.close(fd)
-    py.test.raises(OSError, f, fd, 'Hello world')
+    py.test.raises(OSError, rposix.write, fd, 'Hello world')
 
 def test_os_close():
     fname = str(udir.join('os_test.txt'))
     fd = os.open(fname, os.O_WRONLY|os.O_CREAT, 0777)
     assert fd >= 0
     os.write(fd, 'Hello world')
-    f = getllimpl(os.close)
-    f(fd)
-    py.test.raises(OSError, f, fd)
+    rposix.close(fd)
+    py.test.raises(OSError, rposix.close, fd)
 
 def test_os_lseek():
     fname = str(udir.join('os_test.txt'))
     fd = os.open(fname, os.O_RDWR|os.O_CREAT, 0777)
     assert fd >= 0
     os.write(fd, 'Hello world')
-    f = getllimpl(os.lseek)
-    f(fd,0,0)
+    rposix.lseek(fd,0,0)
     assert os.read(fd, 11) == 'Hello world'
     os.close(fd)
-    py.test.raises(OSError, f, fd, 0, 0)
+    py.test.raises(OSError, rposix.lseek, fd, 0, 0)
 
 def test_os_fsync():
     fname = str(udir.join('os_test.txt'))
     fd = os.open(fname, os.O_WRONLY|os.O_CREAT, 0777)
     assert fd >= 0
     os.write(fd, 'Hello world')
-    f = getllimpl(os.fsync)
-    f(fd)
+    rposix.fsync(fd)
     os.close(fd)
     fid = open(fname)
     assert fid.read() == 'Hello world'
     fid.close()
-    py.test.raises(OSError, f, fd)
+    py.test.raises(OSError, rposix.fsync, fd)
 
 def test_os_fdatasync():
-    try:
-        f = getllimpl(os.fdatasync)
-    except:
-        py.test.skip('No fdatasync in os')
     fname = str(udir.join('os_test.txt'))
     fd = os.open(fname, os.O_WRONLY|os.O_CREAT, 0777)
     assert fd >= 0
     os.write(fd, 'Hello world')
-    f(fd)
+    rposix.fdatasync(fd)
     fid = open(fname)
     assert fid.read() == 'Hello world'
     os.close(fd)
-    py.test.raises(OSError, f, fd)
+    py.test.raises(OSError, rposix.fdatasync, fd)
 
 
 def test_os_kill():

From noreply at buildbot.pypy.org  Fri Nov  7 22:37:16 2014
From: noreply at buildbot.pypy.org (amauryfa)
Date: Fri,  7 Nov 2014 22:37:16 +0100 (CET)
Subject: [pypy-commit] pypy more-rposix: os.getcwd() and os.getcwdu()
Message-ID: <20141107213716.F1DDB1D28DC@cobra.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc 
Branch: more-rposix
Changeset: r74389:abe22db32a60
Date: 2014-11-07 22:08 +0100
http://bitbucket.org/pypy/pypy/changeset/abe22db32a60/

Log:	os.getcwd() and os.getcwdu()

diff --git a/rpython/rlib/rposix.py b/rpython/rlib/rposix.py
--- a/rpython/rlib/rposix.py
+++ b/rpython/rlib/rposix.py
@@ -243,11 +243,24 @@
             return True
         else:
             return path.is_unicode
+
+    @specialize.argtype(0)
+    def _preferred_traits(path):
+        from rpython.rtyper.module.support import StringTraits, UnicodeTraits
+        if _prefer_unicode(path):
+            return UnicodeTraits()
+        else:
+            return StringTraits()
 else:
     @specialize.argtype(0)
     def _prefer_unicode(path):
         return False
 
+    @specialize.argtype(0)
+    def _preferred_traits(path):
+        from rpython.rtyper.module.support import StringTraits
+        return StringTraits()
+    
 @specialize.argtype(0)
 def stat(path):
     return os.stat(_as_bytes(path))
@@ -454,6 +467,65 @@
         error = c_access(_as_bytes0(path), mode)
     return error == 0
 
+# This Win32 function is not exposed via os, but needed to get a
+# correct implementation of os.path.abspath.
+ at specialize.argtype(0)
+def getfullpathname(path):
+    length = rwin32.MAX_PATH + 1
+    traits = _get_preferred_traits(path)
+    with traits.scoped_alloc_buffer(count) as buf:
+        res = win32traits.GetFullPathName(
+            path, rffi.cast(rwin32.DWORD, length),
+            buf.raw, lltype.nullptr(win32traits.LPSTRP.TO))
+        if res == 0:
+            raise rwin32.lastWindowsError("_getfullpathname failed")
+        return buf.str()
+
+c_getcwd = external(UNDERSCORE_ON_WIN32 + 'getcwd',
+                    [rffi.CCHARP, rffi.SIZE_T], rffi.CCHARP)
+c_wgetcwd = external(UNDERSCORE_ON_WIN32 + 'wgetcwd',
+                     [rffi.CWCHARP, rffi.SIZE_T], rffi.CWCHARP)
+
+ at replace_os_function('getcwd')
+def getcwd():
+    bufsize = 256
+    while True:
+        buf = lltype.malloc(rffi.CCHARP.TO, bufsize, flavor='raw')
+        res = c_getcwd(buf, bufsize)
+        if res:
+            break   # ok
+        error = get_errno()
+        lltype.free(buf, flavor='raw')
+        if error != errno.ERANGE:
+            raise OSError(error, "getcwd failed")
+        # else try again with a larger buffer, up to some sane limit
+        bufsize *= 4
+        if bufsize > 1024*1024:  # xxx hard-coded upper limit
+            raise OSError(error, "getcwd result too large")
+    result = rffi.charp2str(res)
+    lltype.free(buf, flavor='raw')
+    return result
+
+ at replace_os_function('getcwdu')
+def getcwdu():
+    bufsize = 256
+    while True:
+        buf = lltype.malloc(rffi.CWCHARP.TO, bufsize, flavor='raw')
+        res = c_wgetcwd(buf, bufsize)
+        if res:
+            break   # ok
+        error = rposix.get_errno()
+        lltype.free(buf, flavor='raw')
+        if error != errno.ERANGE:
+            raise OSError(error, "getcwd failed")
+        # else try again with a larger buffer, up to some sane limit
+        bufsize *= 4
+        if bufsize > 1024*1024:  # xxx hard-coded upper limit
+            raise OSError(error, "getcwd result too large")
+    result = rffi.wcharp2unicode(res)
+    lltype.free(buf, flavor='raw')
+    return result
+
 #___________________________________________________________________
 
 c_execv = external('execv', [rffi.CCHARP, rffi.CCHARPP], rffi.INT)
@@ -680,16 +752,9 @@
         handle_posix_error('utime', error)
     else:  # _WIN32 case
         from rpython.rlib.rwin32file import make_win32_traits
-        if _prefer_unicode(path):
-            # XXX remove dependency on rtyper.module.  The "traits"
-            # are just used for CreateFile anyway.
-            from rpython.rtyper.module.support import UnicodeTraits
-            win32traits = make_win32_traits(UnicodeTraits())
-            path = _as_unicode0(path)
-        else:
-            from rpython.rtyper.module.support import StringTraits
-            win32traits = make_win32_traits(StringTraits())
-            path = _as_bytes0(path)
+        traits = _preferred_traits(path)
+        win32traits = make_win32_traits(traits)
+        path = traits.as_str0(path)
         hFile = win32traits.CreateFile(path,
                            win32traits.FILE_WRITE_ATTRIBUTES, 0,
                            None, win32traits.OPEN_EXISTING,
diff --git a/rpython/rtyper/module/ll_os.py b/rpython/rtyper/module/ll_os.py
--- a/rpython/rtyper/module/ll_os.py
+++ b/rpython/rtyper/module/ll_os.py
@@ -173,75 +173,6 @@
                 separate_module_sources = ["\n".join(defs)]
             ))
 
-    @registering_str_unicode(getattr(posix, '_getfullpathname', None),
-                             condition=sys.platform=='win32')
-    def register_posix__getfullpathname(self, traits):
-        # this nt function is not exposed via os, but needed
-        # to get a correct implementation of os.path.abspath
-        from rpython.rtyper.module.ll_win32file import make_getfullpathname_impl
-        getfullpathname_llimpl = make_getfullpathname_impl(traits)
-
-        return extdef([traits.str0],  # a single argument which is a str
-                      traits.str0,    # returns a string
-                      traits.ll_os_name('_getfullpathname'),
-                      llimpl=getfullpathname_llimpl)
-
-    @registering(os.getcwd)
-    def register_os_getcwd(self):
-        os_getcwd = self.llexternal(UNDERSCORE_ON_WIN32 + 'getcwd',
-                                    [rffi.CCHARP, rffi.SIZE_T],
-                                    rffi.CCHARP)
-
-        def os_getcwd_llimpl():
-            bufsize = 256
-            while True:
-                buf = lltype.malloc(rffi.CCHARP.TO, bufsize, flavor='raw')
-                res = os_getcwd(buf, rffi.cast(rffi.SIZE_T, bufsize))
-                if res:
-                    break   # ok
-                error = rposix.get_errno()
-                lltype.free(buf, flavor='raw')
-                if error != errno.ERANGE:
-                    raise OSError(error, "getcwd failed")
-                # else try again with a larger buffer, up to some sane limit
-                bufsize *= 4
-                if bufsize > 1024*1024:  # xxx hard-coded upper limit
-                    raise OSError(error, "getcwd result too large")
-            result = rffi.charp2str(res)
-            lltype.free(buf, flavor='raw')
-            return result
-
-        return extdef([], str0,
-                      "ll_os.ll_os_getcwd", llimpl=os_getcwd_llimpl)
-
-    @registering(os.getcwdu, condition=sys.platform=='win32')
-    def register_os_getcwdu(self):
-        os_wgetcwd = self.llexternal(UNDERSCORE_ON_WIN32 + 'wgetcwd',
-                                     [rffi.CWCHARP, rffi.SIZE_T],
-                                     rffi.CWCHARP)
-
-        def os_getcwd_llimpl():
-            bufsize = 256
-            while True:
-                buf = lltype.malloc(rffi.CWCHARP.TO, bufsize, flavor='raw')
-                res = os_wgetcwd(buf, rffi.cast(rffi.SIZE_T, bufsize))
-                if res:
-                    break   # ok
-                error = rposix.get_errno()
-                lltype.free(buf, flavor='raw')
-                if error != errno.ERANGE:
-                    raise OSError(error, "getcwd failed")
-                # else try again with a larger buffer, up to some sane limit
-                bufsize *= 4
-                if bufsize > 1024*1024:  # xxx hard-coded upper limit
-                    raise OSError(error, "getcwd result too large")
-            result = rffi.wcharp2unicode(res)
-            lltype.free(buf, flavor='raw')
-            return result
-
-        return extdef([], unicode,
-                      "ll_os.ll_os_wgetcwd", llimpl=os_getcwd_llimpl)
-
     @registering_str_unicode(os.listdir)
     def register_os_listdir(self, traits):
         # we need a different approach on Windows and on Posix
diff --git a/rpython/rtyper/module/support.py b/rpython/rtyper/module/support.py
--- a/rpython/rtyper/module/support.py
+++ b/rpython/rtyper/module/support.py
@@ -2,6 +2,7 @@
 
 from rpython.annotator import model as annmodel
 from rpython.rtyper.lltypesystem import lltype, rffi
+from rpython.rlib.objectmodel import specialize
 
 _WIN32 = sys.platform.startswith('win')
 UNDERSCORE_ON_WIN32 = '_' if _WIN32 else ''
@@ -63,6 +64,22 @@
     def ll_os_name(name):
         return 'll_os.ll_os_' + name
 
+    @classmethod
+    @specialize.argtype(1)
+    def as_str(cls, path):
+        assert path is not None
+        if isinstance(path, unicode):
+            return path
+        else:
+            return path.as_unicode()
+    
+    @classmethod
+    @specialize.argtype(1)
+    def as_str0(cls, path):
+        res = cls.as_str(path)
+        rstring.check_str0(res)
+        return res
+
 class UnicodeTraits:
     str = unicode
     str0 = annmodel.s_Unicode0
@@ -83,6 +100,24 @@
     def ll_os_name(name):
         return 'll_os.ll_os_w' + name
 
+    @classmethod
+    def as_str(cls, path):
+        assert path is not None
+        if isinstance(path, str):
+            return path
+        elif isinstance(path, unicode):
+            # This never happens in PyPy's Python interpreter!
+            # Only in raw RPython code that uses unicode strings.
+            # We implement python2 behavior: silently convert to ascii.
+            return path.encode('ascii')
+        else:
+            return path.as_bytes()
+    
+    @classmethod
+    def as_str0(cls, path):
+        res = cls.as_str(path)
+        rstring.check_str0(res)
+        return res
 
 def ll_strcpy(dst_s, src_s, n):
     dstchars = dst_s.chars
diff --git a/rpython/rtyper/module/test/test_ll_os.py b/rpython/rtyper/module/test/test_ll_os.py
--- a/rpython/rtyper/module/test/test_ll_os.py
+++ b/rpython/rtyper/module/test/test_ll_os.py
@@ -95,8 +95,7 @@
     assert not data.endswith(stuff)
 
 def test_getcwd():
-    data = getllimpl(os.getcwd)()
-    assert data == os.getcwd()
+    assert rposix.getcwd() == os.getcwd()
 
 def test_chdir():
     def check_special_envvar():

From noreply at buildbot.pypy.org  Fri Nov  7 22:37:18 2014
From: noreply at buildbot.pypy.org (amauryfa)
Date: Fri,  7 Nov 2014 22:37:18 +0100 (CET)
Subject: [pypy-commit] pypy more-rposix: os.listdir()
Message-ID: <20141107213718.32E991D28DC@cobra.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc 
Branch: more-rposix
Changeset: r74390:8d7118f6770e
Date: 2014-11-07 22:36 +0100
http://bitbucket.org/pypy/pypy/changeset/8d7118f6770e/

Log:	os.listdir()

diff --git a/rpython/rlib/rposix.py b/rpython/rlib/rposix.py
--- a/rpython/rlib/rposix.py
+++ b/rpython/rlib/rposix.py
@@ -144,7 +144,7 @@
 else:
     includes = ['unistd.h',  'sys/types.h',
                 'utime.h', 'sys/time.h', 'sys/times.h',
-                'grp.h']
+                'grp.h', 'dirent.h']
     libraries = ['util']
 eci = ExternalCompilationInfo(
     includes=includes,
@@ -284,14 +284,6 @@
     return os.rename(_as_bytes(path1), _as_bytes(path2))
 
 @specialize.argtype(0)
-def listdir(dirname):
-    return os.listdir(_as_bytes(dirname))
-
- at specialize.argtype(0)
-def access(path, mode):
-    return os.access(_as_bytes(path), mode)
-
- at specialize.argtype(0)
 def chmod(path, mode):
     return os.chmod(_as_bytes(path), mode)
 
@@ -526,6 +518,92 @@
     lltype.free(buf, flavor='raw')
     return result
 
+if not _WIN32:
+    class CConfig:
+        _compilation_info_ = eci
+        DIRENT = rffi_platform.Struct('struct dirent',
+            [('d_name', lltype.FixedSizeArray(rffi.CHAR, 1))])
+
+    DIRP = rffi.COpaquePtr('DIR')
+    config = rffi_platform.configure(CConfig)
+    DIRENT = config['DIRENT']
+    DIRENTP = lltype.Ptr(DIRENT)
+    c_opendir = external('opendir', [rffi.CCHARP], DIRP)
+    # XXX macro=True is hack to make sure we get the correct kind of
+    # dirent struct (which depends on defines)
+    c_readdir = external('readdir', [DIRP], DIRENTP, macro=True)
+    c_closedir = external('closedir', [DIRP], rffi.INT)
+
+ at replace_os_function('listdir')
+ at specialize.argtype(0)
+def listdir(path):
+    if not _WIN32:
+        path = _as_bytes0(path)
+        dirp = c_opendir(path)
+        if not dirp:
+            raise OSError(get_errno(), "opendir failed")
+        result = []
+        while True:
+            set_errno(0)
+            direntp = c_readdir(dirp)
+            if not direntp:
+                error = get_errno()
+                break
+            namep = rffi.cast(rffi.CCHARP, direntp.c_d_name)
+            name = rffi.charp2str(namep)
+            if name != '.' and name != '..':
+                result.append(name)
+        c_closedir(dirp)
+        if error:
+            raise OSError(error, "readdir failed")
+        return result
+    else:  # _WIN32 case
+        from rpython.rlib.rwin32file import make_win32_traits
+        traits = _preferred_traits(path)
+        win32traits = make_win32_traits(traits)
+        path = traits.as_str0(path)
+
+        if traits.str is unicode:
+            if path and path[-1] not in (u'/', u'\\', u':'):
+                path += u'/'
+            mask = path + u'*.*'
+        else:
+            if path and path[-1] not in ('/', '\\', ':'):
+                path += '/'
+            mask = path + '*.*'
+
+        filedata = lltype.malloc(win32traits.WIN32_FIND_DATA, flavor='raw')
+        try:
+            result = []
+            hFindFile = win32traits.FindFirstFile(mask, filedata)
+            if hFindFile == rwin32.INVALID_HANDLE_VALUE:
+                error = rwin32.GetLastError()
+                if error == win32traits.ERROR_FILE_NOT_FOUND:
+                    return result
+                else:
+                    raise WindowsError(error,  "FindFirstFile failed")
+            while True:
+                name = traits.charp2str(rffi.cast(traits.CCHARP,
+                                                  filedata.c_cFileName))
+                if traits.str is unicode:
+                    if not (name == u"." or name == u".."):
+                        result.append(name)
+                else:
+                    if not (name == "." or name == ".."):
+                        result.append(name)
+                if not win32traits.FindNextFile(hFindFile, filedata):
+                    break
+            # FindNextFile sets error to ERROR_NO_MORE_FILES if
+            # it got to the end of the directory
+            error = rwin32.GetLastError()
+            win32traits.FindClose(hFindFile)
+            if error == win32traits.ERROR_NO_MORE_FILES:
+                return result
+            else:
+                raise WindowsError(error,  "FindNextFile failed")
+        finally:
+            lltype.free(filedata, flavor='raw')
+
 #___________________________________________________________________
 
 c_execv = external('execv', [rffi.CCHARP, rffi.CCHARPP], rffi.INT)
diff --git a/rpython/rtyper/module/ll_os.py b/rpython/rtyper/module/ll_os.py
--- a/rpython/rtyper/module/ll_os.py
+++ b/rpython/rtyper/module/ll_os.py
@@ -173,61 +173,6 @@
                 separate_module_sources = ["\n".join(defs)]
             ))
 
-    @registering_str_unicode(os.listdir)
-    def register_os_listdir(self, traits):
-        # we need a different approach on Windows and on Posix
-        if sys.platform.startswith('win'):
-            from rpython.rtyper.module.ll_win32file import make_listdir_impl
-            os_listdir_llimpl = make_listdir_impl(traits)
-        else:
-            assert traits.str is str
-            compilation_info = ExternalCompilationInfo(
-                includes = ['sys/types.h', 'dirent.h']
-            )
-            class CConfig:
-                _compilation_info_ = compilation_info
-                DIRENT = platform.Struct('struct dirent',
-                    [('d_name', lltype.FixedSizeArray(rffi.CHAR, 1))])
-
-            DIRP = rffi.COpaquePtr('DIR')
-            config = platform.configure(CConfig)
-            DIRENT = config['DIRENT']
-            DIRENTP = lltype.Ptr(DIRENT)
-            os_opendir = self.llexternal('opendir', [rffi.CCHARP], DIRP,
-                                         compilation_info=compilation_info)
-            # XXX macro=True is hack to make sure we get the correct kind of
-            # dirent struct (which depends on defines)
-            os_readdir = self.llexternal('readdir', [DIRP], DIRENTP,
-                                         compilation_info=compilation_info,
-                                         macro=True)
-            os_closedir = self.llexternal('closedir', [DIRP], rffi.INT,
-                                          compilation_info=compilation_info)
-
-            def os_listdir_llimpl(path):
-                dirp = os_opendir(path)
-                if not dirp:
-                    raise OSError(rposix.get_errno(), "os_opendir failed")
-                result = []
-                while True:
-                    rposix.set_errno(0)
-                    direntp = os_readdir(dirp)
-                    if not direntp:
-                        error = rposix.get_errno()
-                        break
-                    namep = rffi.cast(rffi.CCHARP, direntp.c_d_name)
-                    name = rffi.charp2str(namep)
-                    if name != '.' and name != '..':
-                        result.append(name)
-                os_closedir(dirp)
-                if error:
-                    raise OSError(error, "os_readdir failed")
-                return result
-
-        return extdef([traits.str0],  # a single argument which is a str
-                      [traits.str0],  # returns a list of strings
-                      traits.ll_os_name('listdir'),
-                      llimpl=os_listdir_llimpl)
-
     @registering(os.pipe)
     def register_os_pipe(self):
         # we need a different approach on Windows and on Posix
diff --git a/rpython/rtyper/module/ll_win32file.py b/rpython/rtyper/module/ll_win32file.py
--- a/rpython/rtyper/module/ll_win32file.py
+++ b/rpython/rtyper/module/ll_win32file.py
@@ -12,65 +12,6 @@
 
 
 #_______________________________________________________________
-# listdir
-
-def make_listdir_impl(traits):
-    from rpython.rlib import rwin32
-    from rpython.rlib.rwin32file import make_win32_traits
-
-    win32traits = make_win32_traits(traits)
-
-    if traits.str is unicode:
-        def make_listdir_mask(path):
-            if path and path[-1] not in (u'/', u'\\', u':'):
-                path += u'/'
-            return path + u'*.*'
-
-        def skip_listdir(name):
-            return name == u"." or name == u".."
-    else:
-        def make_listdir_mask(path):
-            if path and path[-1] not in ('/', '\\', ':'):
-                path += '/'
-            return path + '*.*'
-
-        def skip_listdir(name):
-            return name == "." or name == ".."
-
-    @func_renamer('listdir_llimpl_%s' % traits.str.__name__)
-    def listdir_llimpl(path):
-        mask = make_listdir_mask(path)
-        filedata = lltype.malloc(win32traits.WIN32_FIND_DATA, flavor='raw')
-        try:
-            result = []
-            hFindFile = win32traits.FindFirstFile(mask, filedata)
-            if hFindFile == rwin32.INVALID_HANDLE_VALUE:
-                error = rwin32.GetLastError()
-                if error == win32traits.ERROR_FILE_NOT_FOUND:
-                    return result
-                else:
-                    raise WindowsError(error,  "FindFirstFile failed")
-            while True:
-                name = traits.charp2str(rffi.cast(traits.CCHARP,
-                                                  filedata.c_cFileName))
-                if not skip_listdir(name):
-                    result.append(name)
-                if not win32traits.FindNextFile(hFindFile, filedata):
-                    break
-            # FindNextFile sets error to ERROR_NO_MORE_FILES if
-            # it got to the end of the directory
-            error = rwin32.GetLastError()
-            win32traits.FindClose(hFindFile)
-            if error == win32traits.ERROR_NO_MORE_FILES:
-                return result
-            else:
-                raise WindowsError(error,  "FindNextFile failed")
-        finally:
-            lltype.free(filedata, flavor='raw')
-
-    return listdir_llimpl
-
-#_______________________________________________________________
 # chdir
 
 def make_chdir_impl(traits):

From noreply at buildbot.pypy.org  Fri Nov  7 23:13:45 2014
From: noreply at buildbot.pypy.org (amintos)
Date: Fri,  7 Nov 2014 23:13:45 +0100 (CET)
Subject: [pypy-commit] pypy float-opt: optimize float division with
	reciprocal multiplication
Message-ID: <20141107221345.D7AFC1D28F2@cobra.cs.uni-duesseldorf.de>

Author: Toni Mattis 
Branch: float-opt
Changeset: r74391:937254cbc554
Date: 2014-11-06 16:56 +0100
http://bitbucket.org/pypy/pypy/changeset/937254cbc554/

Log:	optimize float division with reciprocal multiplication

diff --git a/rpython/jit/metainterp/optimizeopt/rewrite.py b/rpython/jit/metainterp/optimizeopt/rewrite.py
--- a/rpython/jit/metainterp/optimizeopt/rewrite.py
+++ b/rpython/jit/metainterp/optimizeopt/rewrite.py
@@ -1,7 +1,7 @@
 from rpython.jit.codewriter.effectinfo import EffectInfo
 from rpython.jit.metainterp import compile
 from rpython.jit.metainterp.history import (Const, ConstInt, BoxInt, BoxFloat,
-    BoxPtr, make_hashable_int)
+    BoxPtr, make_hashable_int, ConstFloat)
 from rpython.jit.metainterp.optimize import InvalidLoop
 from rpython.jit.metainterp.optimizeopt.intutils import IntBound
 from rpython.jit.metainterp.optimizeopt.optimizer import (Optimization, REMOVED,
@@ -10,7 +10,7 @@
 from rpython.jit.metainterp.resoperation import (opboolinvers, opboolreflex, rop,
     ResOperation)
 from rpython.rlib.rarithmetic import highest_bit
-
+from rpython.rlib.longlong2float import float2longlong
 
 class OptRewrite(Optimization):
     """Rewrite operations into equivalent, cheaper operations.
@@ -231,6 +231,25 @@
         self.emit_operation(op)
         self.pure(rop.FLOAT_MUL, [arg2, arg1], op.result)
 
+    def optimize_FLOAT_TRUEDIV(self, op):
+        arg1 = op.getarg(0)
+        arg2 = op.getarg(1)
+        v2 = self.getvalue(arg2)
+
+        # replace "x / const" by "x * (1/const)" if possible
+        if v2.is_constant():
+            bits = float2longlong(v2.box.getfloat())
+            fraction = bits & ((1 << 52) - 1)
+            exponent = (bits >> 52) & 0x7ff
+
+            # This optimization is valid for powers of two (fraction == 0)
+            # but not for zeroes and some subnormals (exponent == 0).
+            # Checking for zero-fraction also avoids NaNs:
+            if fraction == 0 and exponent != 0:
+                reciprocal = ConstFloat(1.0 / v2.box.getfloat())
+                op = op.copy_and_change(rop.FLOAT_MUL, args=[arg1, reciprocal])
+        self.emit_operation(op)
+
     def optimize_FLOAT_NEG(self, op):
         v1 = op.getarg(0)
         self.emit_operation(op)
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py
@@ -2364,6 +2364,28 @@
         """
         self.optimize_loop(ops, expected)
 
+    def test_float_division_by_multiplication(self):
+        ops = """
+        [f0]
+        f1 = float_truediv(f0, 2.0)
+        f2 = float_truediv(f1, 3.0)
+        f3 = float_truediv(f2, -0.25)
+        f4 = float_truediv(f3, 0.0)
+        f5 = escape(f4)
+        jump(f5)
+        """
+
+        expected = """
+        [f0]
+        f1 = float_mul(f0, 0.5)
+        f2 = float_truediv(f1, 3.0)
+        f3 = float_mul(f2, -4.0)
+        f4 = float_truediv(f3, 0.0)
+        f5 = escape(f4)
+        jump(f5)
+        """
+        self.optimize_loop(ops, expected)
+
     # ----------
 
     def _verify_fail_args(self, boxes, oparse, text):

From noreply at buildbot.pypy.org  Fri Nov  7 23:13:47 2014
From: noreply at buildbot.pypy.org (amintos)
Date: Fri,  7 Nov 2014 23:13:47 +0100 (CET)
Subject: [pypy-commit] pypy float-opt: float division optimization using
	frexp now
Message-ID: <20141107221347.202691D28F2@cobra.cs.uni-duesseldorf.de>

Author: Toni Mattis 
Branch: float-opt
Changeset: r74392:f30efb9a8e54
Date: 2014-11-06 22:22 +0100
http://bitbucket.org/pypy/pypy/changeset/f30efb9a8e54/

Log:	float division optimization using frexp now

diff --git a/rpython/jit/metainterp/optimizeopt/rewrite.py b/rpython/jit/metainterp/optimizeopt/rewrite.py
--- a/rpython/jit/metainterp/optimizeopt/rewrite.py
+++ b/rpython/jit/metainterp/optimizeopt/rewrite.py
@@ -10,7 +10,7 @@
 from rpython.jit.metainterp.resoperation import (opboolinvers, opboolreflex, rop,
     ResOperation)
 from rpython.rlib.rarithmetic import highest_bit
-from rpython.rlib.longlong2float import float2longlong
+import math
 
 class OptRewrite(Optimization):
     """Rewrite operations into equivalent, cheaper operations.
@@ -238,16 +238,16 @@
 
         # replace "x / const" by "x * (1/const)" if possible
         if v2.is_constant():
-            bits = float2longlong(v2.box.getfloat())
-            fraction = bits & ((1 << 52) - 1)
-            exponent = (bits >> 52) & 0x7ff
-
-            # This optimization is valid for powers of two (fraction == 0)
-            # but not for zeroes and some subnormals (exponent == 0).
-            # Checking for zero-fraction also avoids NaNs:
-            if fraction == 0 and exponent != 0:
-                reciprocal = ConstFloat(1.0 / v2.box.getfloat())
-                op = op.copy_and_change(rop.FLOAT_MUL, args=[arg1, reciprocal])
+            divisor = v2.box.getfloat()
+            fraction = math.frexp(divisor)[0]
+            # This optimization is valid for powers of two
+            # but not for zeroes, some denormals and NaN:
+            if fraction == 0.5 or fraction == -0.5:
+                reciprocal = 1.0 / divisor
+                rfraction = math.frexp(reciprocal)[0]
+                if rfraction == 0.5 or rfraction == -0.5:
+                    op = op.copy_and_change(rop.FLOAT_MUL,
+                                            args=[arg1, ConstFloat(reciprocal)])
         self.emit_operation(op)
 
     def optimize_FLOAT_NEG(self, op):

From noreply at buildbot.pypy.org  Fri Nov  7 23:13:48 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Fri,  7 Nov 2014 23:13:48 +0100 (CET)
Subject: [pypy-commit] pypy float-opt: Ready to merge
Message-ID: <20141107221348.500EB1D28F2@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: float-opt
Changeset: r74393:36ff85b41418
Date: 2014-11-07 23:11 +0100
http://bitbucket.org/pypy/pypy/changeset/36ff85b41418/

Log:	Ready to merge


From noreply at buildbot.pypy.org  Fri Nov  7 23:13:49 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Fri,  7 Nov 2014 23:13:49 +0100 (CET)
Subject: [pypy-commit] pypy default: Merge float-opt by Toni Mattis: turn
 float divisions by a constant power of two into float multiplication
Message-ID: <20141107221349.8F4CA1D28F2@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r74394:6b4912d3adb7
Date: 2014-11-07 23:12 +0100
http://bitbucket.org/pypy/pypy/changeset/6b4912d3adb7/

Log:	Merge float-opt by Toni Mattis: turn float divisions by a constant
	power of two into float multiplication

diff --git a/rpython/jit/metainterp/optimizeopt/rewrite.py b/rpython/jit/metainterp/optimizeopt/rewrite.py
--- a/rpython/jit/metainterp/optimizeopt/rewrite.py
+++ b/rpython/jit/metainterp/optimizeopt/rewrite.py
@@ -1,7 +1,7 @@
 from rpython.jit.codewriter.effectinfo import EffectInfo
 from rpython.jit.metainterp import compile
 from rpython.jit.metainterp.history import (Const, ConstInt, BoxInt, BoxFloat,
-    BoxPtr, make_hashable_int)
+    BoxPtr, make_hashable_int, ConstFloat)
 from rpython.jit.metainterp.optimize import InvalidLoop
 from rpython.jit.metainterp.optimizeopt.intutils import IntBound
 from rpython.jit.metainterp.optimizeopt.optimizer import (Optimization, REMOVED,
@@ -10,7 +10,7 @@
 from rpython.jit.metainterp.resoperation import (opboolinvers, opboolreflex, rop,
     ResOperation)
 from rpython.rlib.rarithmetic import highest_bit
-
+import math
 
 class OptRewrite(Optimization):
     """Rewrite operations into equivalent, cheaper operations.
@@ -231,6 +231,25 @@
         self.emit_operation(op)
         self.pure(rop.FLOAT_MUL, [arg2, arg1], op.result)
 
+    def optimize_FLOAT_TRUEDIV(self, op):
+        arg1 = op.getarg(0)
+        arg2 = op.getarg(1)
+        v2 = self.getvalue(arg2)
+
+        # replace "x / const" by "x * (1/const)" if possible
+        if v2.is_constant():
+            divisor = v2.box.getfloat()
+            fraction = math.frexp(divisor)[0]
+            # This optimization is valid for powers of two
+            # but not for zeroes, some denormals and NaN:
+            if fraction == 0.5 or fraction == -0.5:
+                reciprocal = 1.0 / divisor
+                rfraction = math.frexp(reciprocal)[0]
+                if rfraction == 0.5 or rfraction == -0.5:
+                    op = op.copy_and_change(rop.FLOAT_MUL,
+                                            args=[arg1, ConstFloat(reciprocal)])
+        self.emit_operation(op)
+
     def optimize_FLOAT_NEG(self, op):
         v1 = op.getarg(0)
         self.emit_operation(op)
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py
@@ -2364,6 +2364,28 @@
         """
         self.optimize_loop(ops, expected)
 
+    def test_float_division_by_multiplication(self):
+        ops = """
+        [f0]
+        f1 = float_truediv(f0, 2.0)
+        f2 = float_truediv(f1, 3.0)
+        f3 = float_truediv(f2, -0.25)
+        f4 = float_truediv(f3, 0.0)
+        f5 = escape(f4)
+        jump(f5)
+        """
+
+        expected = """
+        [f0]
+        f1 = float_mul(f0, 0.5)
+        f2 = float_truediv(f1, 3.0)
+        f3 = float_mul(f2, -4.0)
+        f4 = float_truediv(f3, 0.0)
+        f5 = escape(f4)
+        jump(f5)
+        """
+        self.optimize_loop(ops, expected)
+
     # ----------
 
     def _verify_fail_args(self, boxes, oparse, text):

From noreply at buildbot.pypy.org  Fri Nov  7 23:58:28 2014
From: noreply at buildbot.pypy.org (amauryfa)
Date: Fri,  7 Nov 2014 23:58:28 +0100 (CET)
Subject: [pypy-commit] pypy more-rposix: Lots of translation fixes for
	Windows
Message-ID: <20141107225828.DC59D1D28DC@cobra.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc 
Branch: more-rposix
Changeset: r74395:ab68fdb236c0
Date: 2014-11-07 23:57 +0100
http://bitbucket.org/pypy/pypy/changeset/ab68fdb236c0/

Log:	Lots of translation fixes for Windows

diff --git a/pypy/module/posix/interp_posix.py b/pypy/module/posix/interp_posix.py
--- a/pypy/module/posix/interp_posix.py
+++ b/pypy/module/posix/interp_posix.py
@@ -431,11 +431,11 @@
     try:
         if space.isinstance_w(w_path, space.w_unicode):
             path = FileEncoder(space, w_path)
-            fullpath = rposix._getfullpathname(path)
+            fullpath = rposix.getfullpathname(path)
             w_fullpath = space.wrap(fullpath)
         else:
             path = space.str0_w(w_path)
-            fullpath = rposix._getfullpathname(path)
+            fullpath = rposix.getfullpathname(path)
             w_fullpath = space.wrap(fullpath)
     except OSError, e:
         raise wrap_oserror2(space, e, w_path)
diff --git a/rpython/flowspace/objspace.py b/rpython/flowspace/objspace.py
--- a/rpython/flowspace/objspace.py
+++ b/rpython/flowspace/objspace.py
@@ -18,7 +18,7 @@
     if func.func_code.co_cellvars:
         raise ValueError(
 """RPython functions cannot create closures
-Possible casues:
+Possible causes:
     Function is inner function
     Function uses generator expressions
     Lambda expressions
diff --git a/rpython/rlib/rposix.py b/rpython/rlib/rposix.py
--- a/rpython/rlib/rposix.py
+++ b/rpython/rlib/rposix.py
@@ -3,6 +3,7 @@
 import errno
 from rpython.rtyper.lltypesystem.rffi import CConstant, CExternVariable, INT
 from rpython.rtyper.lltypesystem import lltype, ll2ctypes, rffi
+from rpython.rtyper.module.support import StringTraits, UnicodeTraits
 from rpython.rtyper.tool import rffi_platform
 from rpython.translator.tool.cbuild import ExternalCompilationInfo
 from rpython.rlib.rarithmetic import intmask, widen
@@ -234,6 +235,8 @@
 # Returns True when the unicode function should be called:
 # - on Windows
 # - if the path is Unicode.
+unicode_traits = UnicodeTraits()
+string_traits = StringTraits()
 if _WIN32:
     @specialize.argtype(0)
     def _prefer_unicode(path):
@@ -246,11 +249,10 @@
 
     @specialize.argtype(0)
     def _preferred_traits(path):
-        from rpython.rtyper.module.support import StringTraits, UnicodeTraits
         if _prefer_unicode(path):
-            return UnicodeTraits()
+            return unicode_traits
         else:
-            return StringTraits()
+            return string_traits
 else:
     @specialize.argtype(0)
     def _prefer_unicode(path):
@@ -258,8 +260,7 @@
 
     @specialize.argtype(0)
     def _preferred_traits(path):
-        from rpython.rtyper.module.support import StringTraits
-        return StringTraits()
+        return string_traits
     
 @specialize.argtype(0)
 def stat(path):
@@ -311,12 +312,6 @@
 def symlink(src, dest):
     os.symlink(_as_bytes(src), _as_bytes(dest))
 
-if os.name == 'nt':
-    import nt
-    @specialize.argtype(0)
-    def _getfullpathname(path):
-        return nt._getfullpathname(_as_bytes(path))
-
 @specialize.argtype(0, 1)
 def putenv(name, value):
     os.environ[_as_bytes(name)] = _as_bytes(value)
@@ -464,14 +459,16 @@
 @specialize.argtype(0)
 def getfullpathname(path):
     length = rwin32.MAX_PATH + 1
-    traits = _get_preferred_traits(path)
-    with traits.scoped_alloc_buffer(count) as buf:
+    traits = _preferred_traits(path)
+    from rpython.rlib.rwin32file import make_win32_traits
+    win32traits = make_win32_traits(traits)
+    with traits.scoped_alloc_buffer(length) as buf:
         res = win32traits.GetFullPathName(
-            path, rffi.cast(rwin32.DWORD, length),
+            traits.as_str0(path), rffi.cast(rwin32.DWORD, length),
             buf.raw, lltype.nullptr(win32traits.LPSTRP.TO))
         if res == 0:
             raise rwin32.lastWindowsError("_getfullpathname failed")
-        return buf.str()
+        return buf.str(intmask(res))
 
 c_getcwd = external(UNDERSCORE_ON_WIN32 + 'getcwd',
                     [rffi.CCHARP, rffi.SIZE_T], rffi.CCHARP)
@@ -506,7 +503,7 @@
         res = c_wgetcwd(buf, bufsize)
         if res:
             break   # ok
-        error = rposix.get_errno()
+        error = get_errno()
         lltype.free(buf, flavor='raw')
         if error != errno.ERANGE:
             raise OSError(error, "getcwd failed")
@@ -612,7 +609,7 @@
 c_spawnv = external('spawnv',
                     [rffi.INT, rffi.CCHARP, rffi.CCHARPP], rffi.INT)
 c_spawnve = external('spawnve',
-                    [rffi.INT, rffi.CCHARP, rffi.CCHARPP, rffi.CCHARP],
+                    [rffi.INT, rffi.CCHARP, rffi.CCHARPP, rffi.CCHARPP],
                      rffi.INT)
 
 @replace_os_function('execv')
@@ -829,7 +826,7 @@
                 lltype.free(l_utimbuf, flavor='raw')
         handle_posix_error('utime', error)
     else:  # _WIN32 case
-        from rpython.rlib.rwin32file import make_win32_traits
+        from rpython.rlib.rwin32file import make_win32_traits, time_t_to_FILE_TIME
         traits = _preferred_traits(path)
         win32traits = make_win32_traits(traits)
         path = traits.as_str0(path)
@@ -844,7 +841,7 @@
         atime = lltype.malloc(rwin32.FILETIME, flavor='raw')
         mtime = lltype.malloc(rwin32.FILETIME, flavor='raw')
         try:
-            if tp is None:
+            if times is None:
                 now = lltype.malloc(rwin32.SYSTEMTIME, flavor='raw')
                 try:
                     GetSystemTime(now)
@@ -854,7 +851,7 @@
                 finally:
                     lltype.free(now, flavor='raw')
             else:
-                actime, modtime = tp
+                actime, modtime = times
                 time_t_to_FILE_TIME(actime, atime)
                 time_t_to_FILE_TIME(modtime, mtime)
             if not SetFileTime(hFile, ctime, atime, mtime):
@@ -1125,22 +1122,23 @@
 def chroot(path):
     handle_posix_error('chroot', c_chroot(_as_bytes0(path)))
 
-CHARARRAY1 = lltype.FixedSizeArray(lltype.Char, 1)
-class CConfig:
-    _compilation_info_ = ExternalCompilationInfo(
-        includes = ['sys/utsname.h']
-    )
-    UTSNAME = rffi_platform.Struct('struct utsname', [
-        ('sysname',  CHARARRAY1),
-        ('nodename', CHARARRAY1),
-        ('release',  CHARARRAY1),
-        ('version',  CHARARRAY1),
-        ('machine',  CHARARRAY1)])
-config = rffi_platform.configure(CConfig)
-UTSNAMEP = lltype.Ptr(config['UTSNAME'])
+if not _WIN32:
+    CHARARRAY1 = lltype.FixedSizeArray(lltype.Char, 1)
+    class CConfig:
+        _compilation_info_ = ExternalCompilationInfo(
+            includes = ['sys/utsname.h']
+        )
+        UTSNAME = rffi_platform.Struct('struct utsname', [
+            ('sysname',  CHARARRAY1),
+            ('nodename', CHARARRAY1),
+            ('release',  CHARARRAY1),
+            ('version',  CHARARRAY1),
+            ('machine',  CHARARRAY1)])
+    config = rffi_platform.configure(CConfig)
+    UTSNAMEP = lltype.Ptr(config['UTSNAME'])
 
-c_uname = external('uname', [UTSNAMEP], rffi.INT,
-                    compilation_info=CConfig._compilation_info_)
+    c_uname = external('uname', [UTSNAMEP], rffi.INT,
+                        compilation_info=CConfig._compilation_info_)
 
 @replace_os_function('uname')
 def uname():
diff --git a/rpython/rlib/rwin32file.py b/rpython/rlib/rwin32file.py
--- a/rpython/rlib/rwin32file.py
+++ b/rpython/rlib/rwin32file.py
@@ -5,8 +5,9 @@
 from rpython.rtyper.lltypesystem import lltype, rffi
 from rpython.translator.tool.cbuild import ExternalCompilationInfo
 from rpython.rtyper.tool import rffi_platform as platform
+from rpython.rlib.objectmodel import specialize
 
-
+ at specialize.memo()
 def make_win32_traits(traits):
     from rpython.rlib import rwin32
 
@@ -190,3 +191,20 @@
             rwin32.BOOL)
 
     return Win32Traits
+
+def make_longlong(high, low):
+    return (rffi.r_longlong(high) << 32) + rffi.r_longlong(low)
+
+# Seconds between 1.1.1601 and 1.1.1970
+secs_between_epochs = rffi.r_longlong(11644473600)
+
+def FILE_TIME_to_time_t_float(filetime):
+    ft = make_longlong(filetime.c_dwHighDateTime, filetime.c_dwLowDateTime)
+    # FILETIME is in units of 100 nsec
+    return float(ft) * (1.0 / 10000000.0) - secs_between_epochs
+
+def time_t_to_FILE_TIME(time, filetime):
+    ft = rffi.r_longlong((time + secs_between_epochs) * 10000000)
+    filetime.c_dwHighDateTime = rffi.r_uint(ft >> 32)
+    filetime.c_dwLowDateTime = rffi.r_uint(ft)    # masking off high bits
+
diff --git a/rpython/rtyper/module/ll_os.py b/rpython/rtyper/module/ll_os.py
--- a/rpython/rtyper/module/ll_os.py
+++ b/rpython/rtyper/module/ll_os.py
@@ -20,7 +20,7 @@
 from rpython.rtyper.lltypesystem import rffi
 from rpython.rtyper.lltypesystem import lltype
 from rpython.rtyper.tool import rffi_platform as platform
-from rpython.rlib import rposix
+from rpython.rlib import rposix, rwin32
 from rpython.translator.tool.cbuild import ExternalCompilationInfo
 from rpython.rlib.objectmodel import specialize
 from rpython.translator import cdir
diff --git a/rpython/rtyper/module/support.py b/rpython/rtyper/module/support.py
--- a/rpython/rtyper/module/support.py
+++ b/rpython/rtyper/module/support.py
@@ -3,6 +3,7 @@
 from rpython.annotator import model as annmodel
 from rpython.rtyper.lltypesystem import lltype, rffi
 from rpython.rlib.objectmodel import specialize
+from rpython.rlib import rstring
 
 _WIN32 = sys.platform.startswith('win')
 UNDERSCORE_ON_WIN32 = '_' if _WIN32 else ''
@@ -64,19 +65,24 @@
     def ll_os_name(name):
         return 'll_os.ll_os_' + name
 
-    @classmethod
-    @specialize.argtype(1)
-    def as_str(cls, path):
+    @staticmethod
+    @specialize.argtype(0)
+    def as_str(path):
         assert path is not None
-        if isinstance(path, unicode):
+        if isinstance(path, str):
             return path
+        elif isinstance(path, unicode):
+            # This never happens in PyPy's Python interpreter!
+            # Only in raw RPython code that uses unicode strings.
+            # We implement python2 behavior: silently convert to ascii.
+            return path.encode('ascii')
         else:
-            return path.as_unicode()
-    
-    @classmethod
-    @specialize.argtype(1)
-    def as_str0(cls, path):
-        res = cls.as_str(path)
+            return path.as_bytes()    
+
+    @staticmethod
+    @specialize.argtype(0)
+    def as_str0(path):
+        res = StringTraits.as_str(path)
         rstring.check_str0(res)
         return res
 
@@ -97,25 +103,22 @@
         return UNDERSCORE_ON_WIN32 + 'w' + name
 
     @staticmethod
+    @specialize.argtype(0)
     def ll_os_name(name):
         return 'll_os.ll_os_w' + name
 
-    @classmethod
-    def as_str(cls, path):
+    @staticmethod
+    @specialize.argtype(0)
+    def as_str(path):
         assert path is not None
-        if isinstance(path, str):
+        if isinstance(path, unicode):
             return path
-        elif isinstance(path, unicode):
-            # This never happens in PyPy's Python interpreter!
-            # Only in raw RPython code that uses unicode strings.
-            # We implement python2 behavior: silently convert to ascii.
-            return path.encode('ascii')
         else:
-            return path.as_bytes()
+            return path.as_unicode()
     
-    @classmethod
-    def as_str0(cls, path):
-        res = cls.as_str(path)
+    @staticmethod
+    def as_str0(path):
+        res = UnicodeTraits.as_str(path)
         rstring.check_str0(res)
         return res
 

From noreply at buildbot.pypy.org  Sat Nov  8 08:06:19 2014
From: noreply at buildbot.pypy.org (fijal)
Date: Sat,  8 Nov 2014 08:06:19 +0100 (CET)
Subject: [pypy-commit] pypy ufuncapi: fix syntax and imports
Message-ID: <20141108070619.41B1F1C07F7@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: ufuncapi
Changeset: r74396:42b977217d4f
Date: 2014-11-08 09:06 +0200
http://bitbucket.org/pypy/pypy/changeset/42b977217d4f/

Log:	fix syntax and imports

diff --git a/pypy/module/micronumpy/nditer.py b/pypy/module/micronumpy/nditer.py
--- a/pypy/module/micronumpy/nditer.py
+++ b/pypy/module/micronumpy/nditer.py
@@ -2,7 +2,7 @@
 from pypy.interpreter.typedef import TypeDef, GetSetProperty
 from pypy.interpreter.gateway import interp2app, unwrap_spec, WrappedDefault
 from pypy.interpreter.error import OperationError, oefmt
-from pypy.module.micronumpy import ufuncs, support, concrete
+from pypy.module.micronumpy import support, concrete
 from pypy.module.micronumpy.base import W_NDimArray, convert_to_array
 from pypy.module.micronumpy.descriptor import decode_w_dtype
 from pypy.module.micronumpy.iterators import ArrayIter, SliceIter, OpFlag
@@ -288,8 +288,10 @@
 
 class W_NDIter(W_Root):
     _immutable_fields_ = ['ndim', ]
-    def __init__(self, space, w_seq, w_flags, w_op_flags, w_op_dtypes, w_casting,
-                 w_op_axes, w_itershape, w_buffersize, order):
+    def __init__(self, space, w_seq, w_flags, w_op_flags, w_op_dtypes,
+                 w_casting, w_op_axes, w_itershape, w_buffersize, order):
+        from pypy.module.micronumpy.ufuncs import find_binop_result_dtype
+        
         self.order = order
         self.external_loop = False
         self.buffered = False
@@ -355,7 +357,7 @@
                         continue
                     if self.op_flags[i].rw == 'w':
                         continue
-                    out_dtype = ufuncs.find_binop_result_dtype(
+                    out_dtype = find_binop_result_dtype(
                         space, self.seq[i].get_dtype(), out_dtype)
             for i in outargs:
                 if self.seq[i] is None:
diff --git a/pypy/module/micronumpy/ufuncs.py b/pypy/module/micronumpy/ufuncs.py
--- a/pypy/module/micronumpy/ufuncs.py
+++ b/pypy/module/micronumpy/ufuncs.py
@@ -706,7 +706,7 @@
         if self.stack_inputs:
             inargs = inargs + outargs
             it = W_NDIter(space, space.newlist(inargs + outargs),
-                            flags=flags, op_flags=op_flags,
+                            flags=flags, op_flags=op_flags)
             for args in it:
                 space.call_args(func, Arguments.frompacked(space, args))
             if len(outargs) > 1:

From noreply at buildbot.pypy.org  Sat Nov  8 11:36:52 2014
From: noreply at buildbot.pypy.org (hey_lu)
Date: Sat,  8 Nov 2014 11:36:52 +0100 (CET)
Subject: [pypy-commit] pypy improve-ptr-conv-error: Created new branch
	improve-ptr-conv-error
Message-ID: <20141108103652.83F151C0548@cobra.cs.uni-duesseldorf.de>

Author: Lucas Stadler 
Branch: improve-ptr-conv-error
Changeset: r74397:756f8bd7af4e
Date: 2014-11-07 17:39 +0000
http://bitbucket.org/pypy/pypy/changeset/756f8bd7af4e/

Log:	Created new branch improve-ptr-conv-error


From noreply at buildbot.pypy.org  Sat Nov  8 11:36:53 2014
From: noreply at buildbot.pypy.org (Lucas Stadler)
Date: Sat,  8 Nov 2014 11:36:53 +0100 (CET)
Subject: [pypy-commit] pypy improve-ptr-conv-error: improve ptr2ptr
	conversion error message
Message-ID: <20141108103653.D17CA1C0548@cobra.cs.uni-duesseldorf.de>

Author: Lucas Stadler 
Branch: improve-ptr-conv-error
Changeset: r74398:2b4ef02eb621
Date: 2014-11-07 18:42 +0100
http://bitbucket.org/pypy/pypy/changeset/2b4ef02eb621/

Log:	improve ptr2ptr conversion error message

diff --git a/rpython/rtyper/rptr.py b/rpython/rtyper/rptr.py
--- a/rpython/rtyper/rptr.py
+++ b/rpython/rtyper/rptr.py
@@ -115,9 +115,9 @@
 
 class __extend__(pairtype(PtrRepr, PtrRepr)):
     def convert_from_to((r_ptr1, r_ptr2), v, llop):
-        assert r_ptr1.lowleveltype == r_ptr2.lowleveltype
-        return v
-
+        if r_ptr1.lowleveltype == r_ptr2.lowleveltype:
+            return v
+        return NotImplemented
 
 class __extend__(pairtype(PtrRepr, IntegerRepr)):
 

From noreply at buildbot.pypy.org  Sat Nov  8 11:36:54 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Sat,  8 Nov 2014 11:36:54 +0100 (CET)
Subject: [pypy-commit] pypy default: Merged in
 hey_lu/pypy/improve-ptr-conv-error (pull request #290)
Message-ID: <20141108103654.F00491C0548@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r74399:3742fe206a81
Date: 2014-11-08 11:36 +0100
http://bitbucket.org/pypy/pypy/changeset/3742fe206a81/

Log:	Merged in hey_lu/pypy/improve-ptr-conv-error (pull request #290)

	Improve ptr2ptr conversion error

diff --git a/rpython/rtyper/rptr.py b/rpython/rtyper/rptr.py
--- a/rpython/rtyper/rptr.py
+++ b/rpython/rtyper/rptr.py
@@ -115,9 +115,9 @@
 
 class __extend__(pairtype(PtrRepr, PtrRepr)):
     def convert_from_to((r_ptr1, r_ptr2), v, llop):
-        assert r_ptr1.lowleveltype == r_ptr2.lowleveltype
-        return v
-
+        if r_ptr1.lowleveltype == r_ptr2.lowleveltype:
+            return v
+        return NotImplemented
 
 class __extend__(pairtype(PtrRepr, IntegerRepr)):
 

From noreply at buildbot.pypy.org  Sat Nov  8 14:15:41 2014
From: noreply at buildbot.pypy.org (ltratt)
Date: Sat,  8 Nov 2014 14:15:41 +0100 (CET)
Subject: [pypy-commit] pypy default: It's been six years since these lines
 were last uncommented. They can go.
Message-ID: <20141108131541.53BA51D387B@cobra.cs.uni-duesseldorf.de>

Author: Laurence Tratt 
Branch: 
Changeset: r74400:96df37fb66d7
Date: 2014-11-08 10:55 +0000
http://bitbucket.org/pypy/pypy/changeset/96df37fb66d7/

Log:	It's been six years since these lines were last uncommented. They
	can go.

diff --git a/rpython/jit/metainterp/pyjitpl.py b/rpython/jit/metainterp/pyjitpl.py
--- a/rpython/jit/metainterp/pyjitpl.py
+++ b/rpython/jit/metainterp/pyjitpl.py
@@ -1333,8 +1333,6 @@
             while True:
                 pc = self.pc
                 op = ord(self.bytecode[pc])
-                #debug_print(self.jitcode.name, pc)
-                #print staticdata.opcode_names[op]
                 staticdata.opcode_implementations[op](self, pc)
         except ChangeFrame:
             pass

From noreply at buildbot.pypy.org  Sat Nov  8 15:48:58 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Sat,  8 Nov 2014 15:48:58 +0100 (CET)
Subject: [pypy-commit] stmgc hashtable: fix for test_minor_collect_bug1 and
 demo_hashtable1: see comments
Message-ID: <20141108144858.AF8521C0548@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: hashtable
Changeset: r1500:cf427187d1e0
Date: 2014-11-08 15:49 +0100
http://bitbucket.org/pypy/stmgc/changeset/cf427187d1e0/

Log:	fix for test_minor_collect_bug1 and demo_hashtable1: see comments

diff --git a/c7/stm/hashtable.c b/c7/stm/hashtable.c
--- a/c7/stm/hashtable.c
+++ b/c7/stm/hashtable.c
@@ -240,7 +240,7 @@
     if (rc > 6) {
         /* we can only enter here once!  If we allocate stuff, we may
            run the GC, and so 'hashtableobj' might move afterwards. */
-        if (_is_from_same_transaction(hashtableobj)) {
+        if (_is_in_nursery(hashtableobj)) {
             entry = (stm_hashtable_entry_t *)
                 stm_allocate(sizeof(stm_hashtable_entry_t));
             entry->userdata = stm_hashtable_entry_userdata;
@@ -248,6 +248,26 @@
             entry->object = NULL;
         }
         else {
+            /* for a non-nursery 'hashtableobj', we pretend that the
+               'entry' object we're about to return was already
+               existing all along, with NULL in all segments.  If the
+               caller of this function is going to modify the 'object'
+               field, it will call stm_write(entry) first, which will
+               correctly schedule 'entry' for write propagation.  We
+               do that even if 'hashtableobj' was created by the
+               running transaction: the new 'entry' object is created
+               as if it was older than the transaction.
+
+               Note the following difference: if 'hashtableobj' is
+               still in the nursery (case above), the 'entry' object
+               is also allocated from the nursery, and after a minor
+               collection it ages as an old-but-created-by-the-
+               current-transaction object.  We could try to emulate
+               this here, or to create young 'entry' objects, but
+               doing either of these would require careful
+               synchronization with other pieces of the code that may
+               change.
+            */
             acquire_privatization_lock();
             char *p = allocate_outside_nursery_large(
                           sizeof(stm_hashtable_entry_t));
@@ -344,34 +364,9 @@
     }
 }
 
-static void _hashtable_minor_trace(object_t **pobj)
-{
-    abort();
-    object_t *obj = *pobj;
-    if (!_is_in_nursery(obj))
-        return;
-
-    TRACE_FOR_MINOR_COLLECTION(pobj);
-
-    obj = *pobj;
-    char *real_obj = REAL_ADDRESS(STM_SEGMENT->segment_base, obj);
-    assert(((struct stm_hashtable_entry_s *)real_obj)->userdata ==
-           stm_hashtable_entry_userdata);
-
-    long j, num = STM_SEGMENT->segment_num;
-    for (j = 0; j <= NB_SEGMENTS; j++) {
-        if (j == num)
-            continue;
-        memcpy(REAL_ADDRESS(get_segment_base(j), obj), real_obj,
-               sizeof(struct stm_hashtable_entry_s));
-    }
-}
-
 void stm_hashtable_tracefn(stm_hashtable_t *hashtable, void trace(object_t **))
 {
-    if (trace == TRACE_FOR_MINOR_COLLECTION)
-        trace = &_hashtable_minor_trace;
-    else if (trace == TRACE_FOR_MAJOR_COLLECTION)
+    if (trace == TRACE_FOR_MAJOR_COLLECTION)
         _stm_compact_hashtable(hashtable);
 
     stm_hashtable_table_t *table;
diff --git a/c7/test/test_hashtable.py b/c7/test/test_hashtable.py
--- a/c7/test/test_hashtable.py
+++ b/c7/test/test_hashtable.py
@@ -187,6 +187,26 @@
         assert htget(h, 1) == lp1
         stm_major_collect()       # to get rid of the hashtable object
 
+    def test_minor_collect_bug1_different_thread(self):
+        self.start_transaction()
+        lp1 = stm_allocate(32)
+        self.push_root(lp1)
+        h = self.allocate_hashtable()
+        self.push_root(h)
+        stm_minor_collect()
+        h = self.pop_root()
+        lp1 = self.pop_root()
+        print 'h', h                       # 0xa040010
+        print 'lp1', lp1                   # 0xa040040
+        tl0 = self.tls[self.current_thread]
+        htset(h, 1, lp1, tl0)
+        self.commit_transaction()
+        #
+        self.switch(1)            # in a different thread
+        self.start_transaction()
+        assert htget(h, 1) == lp1
+        stm_major_collect()       # to get rid of the hashtable object
+
 
 class TestRandomHashtable(BaseTestHashtable):
 

From noreply at buildbot.pypy.org  Sat Nov  8 22:25:43 2014
From: noreply at buildbot.pypy.org (mattip)
Date: Sat,  8 Nov 2014 22:25:43 +0100 (CET)
Subject: [pypy-commit] pypy default: change order of includes for windows
Message-ID: <20141108212543.F31351D283E@cobra.cs.uni-duesseldorf.de>

Author: Matti Picus 
Branch: 
Changeset: r74401:94d9b6624f0e
Date: 2014-11-08 20:38 -0600
http://bitbucket.org/pypy/pypy/changeset/94d9b6624f0e/

Log:	change order of includes for windows

diff --git a/pypy/module/test_lib_pypy/ctypes_tests/_ctypes_test.c b/pypy/module/test_lib_pypy/ctypes_tests/_ctypes_test.c
--- a/pypy/module/test_lib_pypy/ctypes_tests/_ctypes_test.c
+++ b/pypy/module/test_lib_pypy/ctypes_tests/_ctypes_test.c
@@ -1,10 +1,10 @@
-#include "src/precommondefs.h"
-
 #if defined(_MSC_VER) || defined(__CYGWIN__)
 #include 
 #define MS_WIN32
 #endif
 
+#include "src/precommondefs.h"
+
 #define EXPORT(x)  RPY_EXPORTED x
 
 #include 

From noreply at buildbot.pypy.org  Sat Nov  8 22:25:45 2014
From: noreply at buildbot.pypy.org (mattip)
Date: Sat,  8 Nov 2014 22:25:45 +0100 (CET)
Subject: [pypy-commit] pypy default: decorate another windows-only function
Message-ID: <20141108212545.4BF151D283E@cobra.cs.uni-duesseldorf.de>

Author: Matti Picus 
Branch: 
Changeset: r74402:bd587922a586
Date: 2014-11-08 23:24 -0600
http://bitbucket.org/pypy/pypy/changeset/bd587922a586/

Log:	decorate another windows-only function

diff --git a/rpython/translator/c/src/libffi_msvc/ffi.h b/rpython/translator/c/src/libffi_msvc/ffi.h
--- a/rpython/translator/c/src/libffi_msvc/ffi.h
+++ b/rpython/translator/c/src/libffi_msvc/ffi.h
@@ -222,6 +222,7 @@
   void      *user_data;
 } ffi_closure;
 
+RPY_EXPORTED_FOR_TESTS
 ffi_status
 ffi_prep_closure (ffi_closure*,
 		  ffi_cif *,

From noreply at buildbot.pypy.org  Sat Nov  8 22:25:46 2014
From: noreply at buildbot.pypy.org (mattip)
Date: Sat,  8 Nov 2014 22:25:46 +0100 (CET)
Subject: [pypy-commit] pypy default: more closely follow the logic in
 cpython's pyport.h for windows
Message-ID: <20141108212546.6F29E1D283E@cobra.cs.uni-duesseldorf.de>

Author: Matti Picus 
Branch: 
Changeset: r74403:b4a1c8dce962
Date: 2014-11-08 23:25 -0600
http://bitbucket.org/pypy/pypy/changeset/b4a1c8dce962/

Log:	more closely follow the logic in cpython's pyport.h for windows

diff --git a/pypy/module/cpyext/include/modsupport.h b/pypy/module/cpyext/include/modsupport.h
--- a/pypy/module/cpyext/include/modsupport.h
+++ b/pypy/module/cpyext/include/modsupport.h
@@ -78,11 +78,20 @@
 /*
  * This is from pyport.h.  Perhaps it belongs elsewhere.
  */
+#ifdef _WIN32
+/* explicitly export since PyAPI_FUNC is usually dllimport */
+#ifdef __cplusplus
+#define PyMODINIT_FUNC extern "C" __declspec(dllexport) void
+#else
+#define PyMODINIT_FUNC __declspec(dllexport) void
+#endif
+#else
 #ifdef __cplusplus
 #define PyMODINIT_FUNC extern "C" PyAPI_FUNC(void)
 #else
 #define PyMODINIT_FUNC PyAPI_FUNC(void)
 #endif
+#endif /* WIN32 */
 
 PyAPI_DATA(char *) _Py_PackageContext;
 

From noreply at buildbot.pypy.org  Sat Nov  8 22:41:33 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Sat,  8 Nov 2014 22:41:33 +0100 (CET)
Subject: [pypy-commit] pypy default: fix on 32-bit
Message-ID: <20141108214133.1ED051C0548@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r74404:9fd586fe0fe5
Date: 2014-11-08 22:41 +0100
http://bitbucket.org/pypy/pypy/changeset/9fd586fe0fe5/

Log:	fix on 32-bit

diff --git a/rpython/jit/metainterp/optimizeopt/rewrite.py b/rpython/jit/metainterp/optimizeopt/rewrite.py
--- a/rpython/jit/metainterp/optimizeopt/rewrite.py
+++ b/rpython/jit/metainterp/optimizeopt/rewrite.py
@@ -1,4 +1,5 @@
 from rpython.jit.codewriter.effectinfo import EffectInfo
+from rpython.jit.codewriter import longlong
 from rpython.jit.metainterp import compile
 from rpython.jit.metainterp.history import (Const, ConstInt, BoxInt, BoxFloat,
     BoxPtr, make_hashable_int, ConstFloat)
@@ -246,8 +247,8 @@
                 reciprocal = 1.0 / divisor
                 rfraction = math.frexp(reciprocal)[0]
                 if rfraction == 0.5 or rfraction == -0.5:
-                    op = op.copy_and_change(rop.FLOAT_MUL,
-                                            args=[arg1, ConstFloat(reciprocal)])
+                    c = ConstFloat(longlong.getfloatstorage(reciprocal))
+                    op = op.copy_and_change(rop.FLOAT_MUL, args=[arg1, c])
         self.emit_operation(op)
 
     def optimize_FLOAT_NEG(self, op):

From noreply at buildbot.pypy.org  Sat Nov  8 23:44:52 2014
From: noreply at buildbot.pypy.org (amauryfa)
Date: Sat,  8 Nov 2014 23:44:52 +0100 (CET)
Subject: [pypy-commit] pypy more-rposix: os.pipe(), os.chown()
Message-ID: <20141108224452.95FF51D283E@cobra.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc 
Branch: more-rposix
Changeset: r74405:70d839558bf3
Date: 2014-11-08 18:23 +0100
http://bitbucket.org/pypy/pypy/changeset/70d839558bf3/

Log:	os.pipe(), os.chown()

diff --git a/rpython/rlib/rposix.py b/rpython/rlib/rposix.py
--- a/rpython/rlib/rposix.py
+++ b/rpython/rlib/rposix.py
@@ -736,6 +736,46 @@
     finally:
         lltype.free(status_p, flavor='raw')
 
+if _WIN32:
+    CreatePipe = external('CreatePipe', [rwin32.LPHANDLE,
+                                         rwin32.LPHANDLE,
+                                         rffi.VOIDP,
+                                         rwin32.DWORD],
+                          rwin32.BOOL)
+    c_open_osfhandle = external('_open_osfhandle', [rffi.INTPTR_T,
+                                                    rffi.INT],
+                                rffi.INT)
+else:
+    INT_ARRAY_P = rffi.CArrayPtr(rffi.INT)
+    c_pipe = external('pipe', [INT_ARRAY_P], rffi.INT)
+
+ at replace_os_function('pipe')
+def pipe():
+    if _WIN32:
+        pread  = lltype.malloc(rwin32.LPHANDLE.TO, 1, flavor='raw')
+        pwrite = lltype.malloc(rwin32.LPHANDLE.TO, 1, flavor='raw')
+        try:
+            if not CreatePipe(
+                    pread, pwrite, lltype.nullptr(rffi.VOIDP.TO), 0):
+                raise WindowsError(rwin32.GetLastError(), "CreatePipe failed")
+            hread = rffi.cast(rffi.INTPTR_T, pread[0])
+            hwrite = rffi.cast(rffi.INTPTR_T, pwrite[0])
+        finally:
+            lltype.free(pwrite, flavor='raw')
+            lltype.free(pread, flavor='raw')
+        fdread = _open_osfhandle(hread, 0)
+        fdwrite = _open_osfhandle(hwrite, 1)
+        return (fdread, fdwrite)
+    else:
+        filedes = lltype.malloc(INT_ARRAY_P.TO, 2, flavor='raw')
+        try:
+            handle_posix_error('pipe', c_pipe(filedes))
+            return (widen(filedes[0]), widen(filedes[1]))
+        finally:
+            lltype.free(filedes, flavor='raw')
+
+#___________________________________________________________________
+
 c_getlogin = external('getlogin', [], rffi.CCHARP, releasegil=False)
 c_getloadavg = external('getloadavg', 
                         [rffi.CArrayPtr(lltype.Float), rffi.INT], rffi.INT)
@@ -760,6 +800,24 @@
 
 #___________________________________________________________________
 
+c_chown = external('chown', [rffi.CCHARP, rffi.INT, rffi.INT], rffi.INT)
+c_lchown = external('lchown', [rffi.CCHARP, rffi.INT, rffi.INT], rffi.INT)
+c_fchown = external('fchown', [rffi.INT, rffi.INT, rffi.INT], rffi.INT)
+
+ at replace_os_function('chown')
+def chown(path, uid, gid):
+    handle_posix_error('chown', c_chown(path, uid, gid))
+
+ at replace_os_function('lchown')
+def lchown(path, uid, gid):
+    handle_posix_error('lchown', c_lchown(path, uid, gid))
+
+ at replace_os_function('fchown')
+def fchown(fd, uid, gid):
+    handle_posix_error('fchown', c_fchown(fd, uid, gid))
+
+#___________________________________________________________________
+
 UTIMBUFP = lltype.Ptr(UTIMBUF)
 c_utime = external('utime', [rffi.CCHARP, UTIMBUFP], rffi.INT)
 if HAVE_UTIMES:
diff --git a/rpython/rtyper/module/ll_os.py b/rpython/rtyper/module/ll_os.py
--- a/rpython/rtyper/module/ll_os.py
+++ b/rpython/rtyper/module/ll_os.py
@@ -173,97 +173,6 @@
                 separate_module_sources = ["\n".join(defs)]
             ))
 
-    @registering(os.pipe)
-    def register_os_pipe(self):
-        # we need a different approach on Windows and on Posix
-        if sys.platform.startswith('win'):
-            from rpython.rlib import rwin32
-            CreatePipe = self.llexternal('CreatePipe', [rwin32.LPHANDLE,
-                                                        rwin32.LPHANDLE,
-                                                        rffi.VOIDP,
-                                                        rwin32.DWORD],
-                                         rwin32.BOOL)
-            _open_osfhandle = self.llexternal('_open_osfhandle', [rffi.INTPTR_T,
-                                                                  rffi.INT],
-                                              rffi.INT)
-            null = lltype.nullptr(rffi.VOIDP.TO)
-
-            def os_pipe_llimpl():
-                pread  = lltype.malloc(rwin32.LPHANDLE.TO, 1, flavor='raw')
-                pwrite = lltype.malloc(rwin32.LPHANDLE.TO, 1, flavor='raw')
-                ok = CreatePipe(pread, pwrite, null, 0)
-                if ok:
-                    error = 0
-                else:
-                    error = rwin32.GetLastError()
-                hread = rffi.cast(rffi.INTPTR_T, pread[0])
-                hwrite = rffi.cast(rffi.INTPTR_T, pwrite[0])
-                lltype.free(pwrite, flavor='raw')
-                lltype.free(pread, flavor='raw')
-                if error:
-                    raise WindowsError(error, "os_pipe failed")
-                fdread = _open_osfhandle(hread, 0)
-                fdwrite = _open_osfhandle(hwrite, 1)
-                return (fdread, fdwrite)
-
-        else:
-            INT_ARRAY_P = rffi.CArrayPtr(rffi.INT)
-            os_pipe = self.llexternal('pipe', [INT_ARRAY_P], rffi.INT)
-
-            def os_pipe_llimpl():
-                filedes = lltype.malloc(INT_ARRAY_P.TO, 2, flavor='raw')
-                error = rffi.cast(lltype.Signed, os_pipe(filedes))
-                read_fd = filedes[0]
-                write_fd = filedes[1]
-                lltype.free(filedes, flavor='raw')
-                if error != 0:
-                    raise OSError(rposix.get_errno(), "os_pipe failed")
-                return (rffi.cast(lltype.Signed, read_fd),
-                        rffi.cast(lltype.Signed, write_fd))
-
-        return extdef([], (int, int),
-                      "ll_os.ll_os_pipe",
-                      llimpl=os_pipe_llimpl)
-
-    @registering_if(os, 'chown')
-    def register_os_chown(self):
-        os_chown = self.llexternal('chown', [rffi.CCHARP, rffi.INT, rffi.INT],
-                                   rffi.INT)
-
-        def os_chown_llimpl(path, uid, gid):
-            res = os_chown(path, uid, gid)
-            if res == -1:
-                raise OSError(rposix.get_errno(), "os_chown failed")
-
-        return extdef([str0, int, int], None, "ll_os.ll_os_chown",
-                      llimpl=os_chown_llimpl)
-
-    @registering_if(os, 'lchown')
-    def register_os_lchown(self):
-        os_lchown = self.llexternal('lchown',[rffi.CCHARP, rffi.INT, rffi.INT],
-                                    rffi.INT)
-
-        def os_lchown_llimpl(path, uid, gid):
-            res = os_lchown(path, uid, gid)
-            if res == -1:
-                raise OSError(rposix.get_errno(), "os_lchown failed")
-
-        return extdef([str0, int, int], None, "ll_os.ll_os_lchown",
-                      llimpl=os_lchown_llimpl)
-
-    @registering_if(os, 'fchown')
-    def register_os_fchown(self):
-        os_fchown = self.llexternal('fchown',[rffi.INT, rffi.INT, rffi.INT],
-                                    rffi.INT)
-
-        def os_fchown_llimpl(fd, uid, gid):
-            res = os_fchown(fd, uid, gid)
-            if res == -1:
-                raise OSError(rposix.get_errno(), "os_fchown failed")
-
-        return extdef([int, int, int], None, "ll_os.ll_os_fchown",
-                      llimpl=os_fchown_llimpl)
-
     @registering_if(os, 'readlink')
     def register_os_readlink(self):
         os_readlink = self.llexternal('readlink',

From noreply at buildbot.pypy.org  Sat Nov  8 23:44:53 2014
From: noreply at buildbot.pypy.org (amauryfa)
Date: Sat,  8 Nov 2014 23:44:53 +0100 (CET)
Subject: [pypy-commit] pypy more-rposix: Move many functions at the same
	time...
Message-ID: <20141108224453.B76201D283E@cobra.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc 
Branch: more-rposix
Changeset: r74406:a3131c39081f
Date: 2014-11-08 22:16 +0100
http://bitbucket.org/pypy/pypy/changeset/a3131c39081f/

Log:	Move many functions at the same time...

diff --git a/rpython/rlib/rposix.py b/rpython/rlib/rposix.py
--- a/rpython/rlib/rposix.py
+++ b/rpython/rlib/rposix.py
@@ -270,44 +270,10 @@
 def lstat(path):
     return os.lstat(_as_bytes(path))
 
-
 @specialize.argtype(0)
 def statvfs(path):
     return os.statvfs(_as_bytes(path))
 
-
- at specialize.argtype(0)
-def unlink(path):
-    return os.unlink(_as_bytes(path))
-
- at specialize.argtype(0, 1)
-def rename(path1, path2):
-    return os.rename(_as_bytes(path1), _as_bytes(path2))
-
- at specialize.argtype(0)
-def chmod(path, mode):
-    return os.chmod(_as_bytes(path), mode)
-
- at specialize.argtype(0)
-def chdir(path):
-    return os.chdir(_as_bytes(path))
-
- at specialize.argtype(0)
-def mkdir(path, mode=0777):
-    return os.mkdir(_as_bytes(path), mode)
-
- at specialize.argtype(0)
-def rmdir(path):
-    return os.rmdir(_as_bytes(path))
-
- at specialize.argtype(0)
-def mkfifo(path, mode):
-    os.mkfifo(_as_bytes(path), mode)
-
- at specialize.argtype(0)
-def mknod(path, mode, device):
-    os.mknod(_as_bytes(path), mode, device)
-
 @specialize.argtype(0, 1)
 def symlink(src, dest):
     os.symlink(_as_bytes(src), _as_bytes(dest))
@@ -431,12 +397,63 @@
 
 #___________________________________________________________________
 
+c_chdir = external('chdir', [rffi.CCHARP], rffi.INT)
 c_fchdir = external('fchdir', [rffi.INT], rffi.INT)
 c_access = external(UNDERSCORE_ON_WIN32 + 'access',
                     [rffi.CCHARP, rffi.INT], rffi.INT)
 c_waccess = external(UNDERSCORE_ON_WIN32 + 'waccess',
                      [rffi.CWCHARP, rffi.INT], rffi.INT)
 
+ at replace_os_function('chdir')
+ at specialize.argtype(0)
+def chdir(path):
+    if not _WIN32:
+        handle_posix_error('chdir', c_chdir(_as_bytes0(path)))
+    else:
+        traits = _preferred_traits(path)
+        from rpython.rlib.rwin32file import make_win32_traits
+        win32traits = make_win32_traits(traits)
+        path = traits._as_str0(path)
+
+        # This is a reimplementation of the C library's chdir
+        # function, but one that produces Win32 errors instead of DOS
+        # error codes.
+        # chdir is essentially a wrapper around SetCurrentDirectory;
+        # however, it also needs to set "magic" environment variables
+        # indicating the per-drive current directory, which are of the
+        # form =:
+        if not win32traits.SetCurrentDirectory(path):
+            raise rwin32.lastWindowsError()
+        MAX_PATH = rwin32.MAX_PATH
+        assert MAX_PATH > 0
+
+        with traits.scoped_alloc_buffer(MAX_PATH) as path:
+            res = win32traits.GetCurrentDirectory(MAX_PATH + 1, path.raw)
+            if not res:
+                raise rwin32.lastWindowsError()
+            res = rffi.cast(lltype.Signed, res)
+            assert res > 0
+            if res <= MAX_PATH + 1:
+                new_path = path.str(res)
+            else:
+                with traits.scoped_alloc_buffer(res) as path:
+                    res = win32traits.GetCurrentDirectory(res, path.raw)
+                    if not res:
+                        raise rwin32.lastWindowsError()
+                    res = rffi.cast(lltype.Signed, res)
+                    assert res > 0
+                    new_path = path.str(res)
+        if traits.str is unicode:
+            if new_path[0] == u'\\' or new_path[0] == u'/':  # UNC path
+                return
+            magic_envvar = u'=' + new_path[0] + u':'
+        else:
+            if new_path[0] == '\\' or new_path[0] == '/':  # UNC path
+                return
+            magic_envvar = '=' + new_path[0] + ':'
+        if not win32traits.SetEnvironmentVariable(magic_envvar, new_path):
+            raise rwin32.lastWindowsError()
+
 @replace_os_function('fchdir')
 def fchdir(fd):
     validate_fd(fd)
@@ -800,6 +817,152 @@
 
 #___________________________________________________________________
 
+c_readlink = external('readlink',
+                      [rffi.CCHARP, rffi.CCHARP, rffi.SIZE_T], rffi.SSIZE_T)
+
+ at replace_os_function('readlink')
+def readlink(path):
+    path = _as_bytes0(path)
+    bufsize = 1023
+    while True:
+        buf = lltype.malloc(rffi.CCHARP.TO, bufsize, flavor='raw')
+        res = widen(c_readlink(path, buf, bufsize))
+        if res < 0:
+            lltype.free(buf, flavor='raw')
+            error = get_errno()    # failed
+            raise OSError(error, "readlink failed")
+        elif res < bufsize:
+            break                       # ok
+        else:
+            # buf too small, try again with a larger buffer
+            lltype.free(buf, flavor='raw')
+            bufsize *= 4
+    # convert the result to a string
+    result = rffi.charp2strn(buf, res)
+    lltype.free(buf, flavor='raw')
+    return result
+
+c_isatty = external(UNDERSCORE_ON_WIN32 + 'isatty', [rffi.INT], rffi.INT)
+
+ at replace_os_function('isatty')
+def isatty(fd):
+    if not is_valid_fd(fd):
+        return False
+    return c_isatty(fd) != 0
+    
+c_strerror = external('strerror', [rffi.INT], rffi.CCHARP,
+                      releasegil=False)
+
+ at replace_os_function('strerror')
+def strerror(errnum):
+    res = c_strerror(errnum)
+    if not res:
+        raise ValueError("os_strerror failed")
+    return rffi.charp2str(res)
+
+c_system = external('system', [rffi.CCHARP], rffi.INT)
+
+ at replace_os_function('system')
+def system(command):
+    return widen(c_system(command))
+
+c_unlink = external('unlink', [rffi.CCHARP], rffi.INT)
+c_mkdir = external('mkdir', [rffi.CCHARP, rffi.MODE_T], rffi.INT)
+c_rmdir = external(UNDERSCORE_ON_WIN32 + 'rmdir', [rffi.CCHARP], rffi.INT)
+c_wrmdir = external(UNDERSCORE_ON_WIN32 + 'wrmdir', [rffi.CWCHARP], rffi.INT)
+
+ at replace_os_function('unlink')
+ at specialize.argtype(0)
+def unlink(path):
+    if not _WIN32:
+        handle_posix_error('unlink', c_unlink(_as_bytes0(path)))
+    else:
+        traits = _preferred_traits(path)
+        win32traits = make_win32_traits(traits)
+        if not win32traits.DeleteFile(traits.as_str0(path)):
+            raise rwin32.lastWindowsError()
+
+ at replace_os_function('mkdir')
+ at specialize.argtype(0)
+def mkdir(path, mode=0o777):
+    if not _WIN32:
+        handle_posix_error('mkdir', c_mkdir(_as_bytes0(path), mode))
+    else:
+        traits = _preferred_traits(path)
+        win32traits = make_win32_traits(traits)
+        if not win32traits.CreateDirectory(traits.as_str0(path), None):
+            raise rwin32.lastWindowsError()
+
+ at replace_os_function('rmdir')
+ at specialize.argtype(0)
+def rmdir(path):
+    if _prefer_unicode(path):
+        handle_posix_error('wrmdir', c_wrmdir(_as_unicode0(path)))
+    else:
+        handle_posix_error('rmdir', c_rmdir(_as_bytes0(path)))
+
+c_chmod = external('chmod', [rffi.CCHARP, rffi.MODE_T], rffi.INT)
+c_fchmod = external('fchmod', [rffi.INT, rffi.MODE_T], rffi.INT)
+c_rename = external('rename', [rffi.CCHARP, rffi.CCHARP], rffi.INT)
+
+ at replace_os_function('chmod')
+ at specialize.argtype(0)
+def chmod(path, mode):
+    if not _WIN32:
+        handle_posix_error('chmod', c_chmod(_as_bytes0(path), mode))
+    else:
+        traits = _preferred_traits(path)
+        win32traits = make_win32_traits(traits)
+        path = traits.as_str0(path)
+        attr = win32traits.GetFileAttributes(path)
+        if attr == win32traits.INVALID_FILE_ATTRIBUTES:
+            raise rwin32.lastWindowsError()
+        if mode & 0200: # _S_IWRITE
+            attr &= ~win32traits.FILE_ATTRIBUTE_READONLY
+        else:
+            attr |= win32traits.FILE_ATTRIBUTE_READONLY
+        if not win32traits.SetFileAttributes(path, attr):
+            raise rwin32.lastWindowsError()
+
+ at replace_os_function('fchmod')
+def fchmod(fd, mode):
+    handle_posix_error('fchmod', c_fchmod(fd, mode))
+
+ at replace_os_function('rename')
+ at specialize.argtype(0, 1)
+def rename(path1, path2):
+    if not _WIN32:
+        handle_posix_error('rename',
+                           c_rename(_as_bytes0(path1), _as_bytes0(path2)))
+    else:
+        traits = _preferred_traits(path)
+        win32traits = make_win32_traits(traits)
+        path1 = traits.as_str0(path1)
+        path2 = traits.as_str0(path2)
+        if not win32traits.MoveFile(path1, path2):
+            raise rwin32.lastWindowsError()
+
+c_mkfifo = external('mkfifo', [rffi.CCHARP, rffi.MODE_T], rffi.INT)
+c_mknod = external('mknod', [rffi.CCHARP, rffi.MODE_T, rffi.INT], rffi.INT,
+                   macro=True)
+#                                           # xxx: actually ^^^ dev_t
+
+ at replace_os_function('mkfifo')
+ at specialize.argtype(0)
+def mkfifo(path, mode):
+    handle_posix_error('mkfifo', c_mkfifo(_as_bytes0(path), mode))
+
+ at replace_os_function('mknod')
+ at specialize.argtype(0)
+def mknod(path, mode, dev):
+    handle_posix_error('mknod', c_mknod(_as_bytes0(path), mode, dev))
+
+c_umask = external(UNDERSCORE_ON_WIN32 + 'umask', [rffi.MODE_T], rffi.MODE_T)
+
+ at replace_os_function('umask')
+def umask(newmask):
+    return widen(c_umask(newmask))
+
 c_chown = external('chown', [rffi.CCHARP, rffi.INT, rffi.INT], rffi.INT)
 c_lchown = external('lchown', [rffi.CCHARP, rffi.INT, rffi.INT], rffi.INT)
 c_fchown = external('fchown', [rffi.INT, rffi.INT, rffi.INT], rffi.INT)
diff --git a/rpython/rtyper/module/ll_os.py b/rpython/rtyper/module/ll_os.py
--- a/rpython/rtyper/module/ll_os.py
+++ b/rpython/rtyper/module/ll_os.py
@@ -173,246 +173,6 @@
                 separate_module_sources = ["\n".join(defs)]
             ))
 
-    @registering_if(os, 'readlink')
-    def register_os_readlink(self):
-        os_readlink = self.llexternal('readlink',
-                                   [rffi.CCHARP, rffi.CCHARP, rffi.SIZE_T],
-                                   rffi.INT)
-        # XXX SSIZE_T in POSIX.1-2001
-
-        def os_readlink_llimpl(path):
-            bufsize = 1023
-            while True:
-                l_path = rffi.str2charp(path)
-                buf = lltype.malloc(rffi.CCHARP.TO, bufsize,
-                                    flavor='raw')
-                res = rffi.cast(lltype.Signed, os_readlink(l_path, buf, bufsize))
-                lltype.free(l_path, flavor='raw')
-                if res < 0:
-                    error = rposix.get_errno()    # failed
-                    lltype.free(buf, flavor='raw')
-                    raise OSError(error, "readlink failed")
-                elif res < bufsize:
-                    break                       # ok
-                else:
-                    # buf too small, try again with a larger buffer
-                    lltype.free(buf, flavor='raw')
-                    bufsize *= 4
-            # convert the result to a string
-            result = rffi.charp2strn(buf, res)
-            lltype.free(buf, flavor='raw')
-            return result
-
-        return extdef([str0], str0,
-                      "ll_os.ll_os_readlink",
-                      llimpl=os_readlink_llimpl)
-
-    @registering(os.isatty)
-    def register_os_isatty(self):
-        os_isatty = self.llexternal(UNDERSCORE_ON_WIN32 + 'isatty',
-                                    [rffi.INT], rffi.INT)
-
-        def isatty_llimpl(fd):
-            if not rposix.is_valid_fd(fd):
-                return False
-            res = rffi.cast(lltype.Signed, os_isatty(rffi.cast(rffi.INT, fd)))
-            return res != 0
-
-        return extdef([int], bool, llimpl=isatty_llimpl,
-                      export_name="ll_os.ll_os_isatty")
-
-    @registering(os.strerror)
-    def register_os_strerror(self):
-        os_strerror = self.llexternal('strerror', [rffi.INT], rffi.CCHARP, releasegil=False)
-
-        def strerror_llimpl(errnum):
-            res = os_strerror(rffi.cast(rffi.INT, errnum))
-            if not res:
-                raise ValueError("os_strerror failed")
-            return rffi.charp2str(res)
-
-        return extdef([int], str, llimpl=strerror_llimpl,
-                      export_name="ll_os.ll_os_strerror")
-
-    @registering(os.system)
-    def register_os_system(self):
-        os_system = self.llexternal('system', [rffi.CCHARP], rffi.INT)
-
-        def system_llimpl(command):
-            res = os_system(command)
-            return rffi.cast(lltype.Signed, res)
-
-        return extdef([str0], int, llimpl=system_llimpl,
-                      export_name="ll_os.ll_os_system")
-
-    @registering_str_unicode(os.unlink)
-    def register_os_unlink(self, traits):
-        os_unlink = self.llexternal(traits.posix_function_name('unlink'),
-                                    [traits.CCHARP], rffi.INT)
-
-        def unlink_llimpl(pathname):
-            res = rffi.cast(lltype.Signed, os_unlink(pathname))
-            if res < 0:
-                raise OSError(rposix.get_errno(), "os_unlink failed")
-
-        if sys.platform == 'win32':
-            from rpython.rlib.rwin32file import make_win32_traits
-            win32traits = make_win32_traits(traits)
-
-            @func_renamer('unlink_llimpl_%s' % traits.str.__name__)
-            def unlink_llimpl(path):
-                if not win32traits.DeleteFile(path):
-                    raise rwin32.lastWindowsError()
-
-        return extdef([traits.str0], s_None, llimpl=unlink_llimpl,
-                      export_name=traits.ll_os_name('unlink'))
-
-    @registering_str_unicode(os.chdir)
-    def register_os_chdir(self, traits):
-        os_chdir = self.llexternal(traits.posix_function_name('chdir'),
-                                   [traits.CCHARP], rffi.INT)
-
-        def os_chdir_llimpl(path):
-            res = rffi.cast(lltype.Signed, os_chdir(path))
-            if res < 0:
-                raise OSError(rposix.get_errno(), "os_chdir failed")
-
-        # On Windows, use an implementation that will produce Win32 errors
-        if sys.platform == 'win32':
-            from rpython.rtyper.module.ll_win32file import make_chdir_impl
-            os_chdir_llimpl = make_chdir_impl(traits)
-
-        return extdef([traits.str0], s_None, llimpl=os_chdir_llimpl,
-                      export_name=traits.ll_os_name('chdir'))
-
-    @registering_str_unicode(os.mkdir)
-    def register_os_mkdir(self, traits):
-        os_mkdir = self.llexternal(traits.posix_function_name('mkdir'),
-                                   [traits.CCHARP, rffi.MODE_T], rffi.INT)
-
-        if sys.platform == 'win32':
-            from rpython.rlib.rwin32file import make_win32_traits
-            win32traits = make_win32_traits(traits)
-
-            @func_renamer('mkdir_llimpl_%s' % traits.str.__name__)
-            def os_mkdir_llimpl(path, mode):
-                if not win32traits.CreateDirectory(path, None):
-                    raise rwin32.lastWindowsError()
-        else:
-            def os_mkdir_llimpl(pathname, mode):
-                res = os_mkdir(pathname, mode)
-                res = rffi.cast(lltype.Signed, res)
-                if res < 0:
-                    raise OSError(rposix.get_errno(), "os_mkdir failed")
-
-        return extdef([traits.str0, int], s_None, llimpl=os_mkdir_llimpl,
-                      export_name=traits.ll_os_name('mkdir'))
-
-    @registering_str_unicode(os.rmdir)
-    def register_os_rmdir(self, traits):
-        os_rmdir = self.llexternal(traits.posix_function_name('rmdir'),
-                                   [traits.CCHARP], rffi.INT)
-
-        def rmdir_llimpl(pathname):
-            res = rffi.cast(lltype.Signed, os_rmdir(pathname))
-            if res < 0:
-                raise OSError(rposix.get_errno(), "os_rmdir failed")
-
-        return extdef([traits.str0], s_None, llimpl=rmdir_llimpl,
-                      export_name=traits.ll_os_name('rmdir'))
-
-    @registering_str_unicode(os.chmod)
-    def register_os_chmod(self, traits):
-        os_chmod = self.llexternal(traits.posix_function_name('chmod'),
-                                   [traits.CCHARP, rffi.MODE_T], rffi.INT)
-
-        def chmod_llimpl(path, mode):
-            res = rffi.cast(lltype.Signed, os_chmod(path, rffi.cast(rffi.MODE_T, mode)))
-            if res < 0:
-                raise OSError(rposix.get_errno(), "os_chmod failed")
-
-        if sys.platform == 'win32':
-            from rpython.rtyper.module.ll_win32file import make_chmod_impl
-            chmod_llimpl = make_chmod_impl(traits)
-
-        return extdef([traits.str0, int], s_None, llimpl=chmod_llimpl,
-                      export_name=traits.ll_os_name('chmod'))
-
-    @registering_if(os, 'fchmod')
-    def register_os_fchmod(self):
-        os_fchmod = self.llexternal('fchmod', [rffi.INT, rffi.MODE_T],
-                                    rffi.INT)
-
-        def fchmod_llimpl(fd, mode):
-            mode = rffi.cast(rffi.MODE_T, mode)
-            res = rffi.cast(lltype.Signed, os_fchmod(fd, mode))
-            if res < 0:
-                raise OSError(rposix.get_errno(), "os_fchmod failed")
-
-        return extdef([int, int], s_None, "ll_os.ll_os_fchmod",
-                      llimpl=fchmod_llimpl)
-
-    @registering_str_unicode(os.rename)
-    def register_os_rename(self, traits):
-        os_rename = self.llexternal(traits.posix_function_name('rename'),
-                                    [traits.CCHARP, traits.CCHARP], rffi.INT)
-
-        def rename_llimpl(oldpath, newpath):
-            res = rffi.cast(lltype.Signed, os_rename(oldpath, newpath))
-            if res < 0:
-                raise OSError(rposix.get_errno(), "os_rename failed")
-
-        if sys.platform == 'win32':
-            from rpython.rlib.rwin32file import make_win32_traits
-            win32traits = make_win32_traits(traits)
-
-            @func_renamer('rename_llimpl_%s' % traits.str.__name__)
-            def rename_llimpl(oldpath, newpath):
-                if not win32traits.MoveFile(oldpath, newpath):
-                    raise rwin32.lastWindowsError()
-
-        return extdef([traits.str0, traits.str0], s_None, llimpl=rename_llimpl,
-                      export_name=traits.ll_os_name('rename'))
-
-    @registering_str_unicode(getattr(os, 'mkfifo', None))
-    def register_os_mkfifo(self, traits):
-        os_mkfifo = self.llexternal(traits.posix_function_name('mkfifo'),
-                                    [traits.CCHARP, rffi.MODE_T], rffi.INT)
-
-        def mkfifo_llimpl(path, mode):
-            res = rffi.cast(lltype.Signed, os_mkfifo(path, mode))
-            if res < 0:
-                raise OSError(rposix.get_errno(), "os_mkfifo failed")
-
-        return extdef([traits.str0, int], s_None, llimpl=mkfifo_llimpl,
-                      export_name=traits.ll_os_name('mkfifo'))
-
-    @registering_str_unicode(getattr(os, 'mknod', None))
-    def register_os_mknod(self, traits):
-        os_mknod = self.llexternal(traits.posix_function_name('mknod'),
-                                   [traits.CCHARP, rffi.MODE_T, rffi.INT],
-                                   rffi.INT)      # xxx: actually ^^^ dev_t
-
-        def mknod_llimpl(path, mode, dev):
-            res = rffi.cast(lltype.Signed, os_mknod(path, mode, dev))
-            if res < 0:
-                raise OSError(rposix.get_errno(), "os_mknod failed")
-
-        return extdef([traits.str0, int, int], s_None, llimpl=mknod_llimpl,
-                      export_name=traits.ll_os_name('mknod'))
-
-    @registering(os.umask)
-    def register_os_umask(self):
-        os_umask = self.llexternal(UNDERSCORE_ON_WIN32 + 'umask',
-                                   [rffi.MODE_T], rffi.MODE_T)
-
-        def umask_llimpl(newmask):
-            res = os_umask(rffi.cast(rffi.MODE_T, newmask))
-            return rffi.cast(lltype.Signed, res)
-
-        return extdef([int], int, llimpl=umask_llimpl,
-                      export_name="ll_os.ll_os_umask")
-
     @registering_if(os, 'kill', sys.platform != 'win32')
     def register_os_kill(self):
         os_kill = self.llexternal('kill', [rffi.PID_T, rffi.INT],
diff --git a/rpython/rtyper/module/ll_win32file.py b/rpython/rtyper/module/ll_win32file.py
deleted file mode 100644
--- a/rpython/rtyper/module/ll_win32file.py
+++ /dev/null
@@ -1,115 +0,0 @@
-"""
-The Windows implementation of some posix modules,
-based on the Win32 API.
-"""
-from __future__ import with_statement
-
-from rpython.rtyper.lltypesystem import lltype, rffi
-from rpython.translator.tool.cbuild import ExternalCompilationInfo
-from rpython.rtyper.tool import rffi_platform as platform
-from rpython.tool.sourcetools import func_renamer
-from rpython.rlib.objectmodel import specialize
-
-
-#_______________________________________________________________
-# chdir
-
-def make_chdir_impl(traits):
-    from rpython.rlib import rwin32
-    from rpython.rlib.rwin32file import make_win32_traits
-
-    win32traits = make_win32_traits(traits)
-
-    if traits.str is unicode:
-        def isUNC(path):
-            return path[0] == u'\\' or path[0] == u'/'
-        def magic_envvar(path):
-            return u'=' + path[0] + u':'
-    else:
-        def isUNC(path):
-            return path[0] == '\\' or path[0] == '/'
-        def magic_envvar(path):
-            return '=' + path[0] + ':'
-
-    @func_renamer('chdir_llimpl_%s' % traits.str.__name__)
-    def chdir_llimpl(path):
-        """This is a reimplementation of the C library's chdir function,
-        but one that produces Win32 errors instead of DOS error codes.
-        chdir is essentially a wrapper around SetCurrentDirectory; however,
-        it also needs to set "magic" environment variables indicating
-        the per-drive current directory, which are of the form =:
-        """
-        if not win32traits.SetCurrentDirectory(path):
-            raise rwin32.lastWindowsError()
-        MAX_PATH = rwin32.MAX_PATH
-        assert MAX_PATH > 0
-
-        with traits.scoped_alloc_buffer(MAX_PATH) as path:
-            res = win32traits.GetCurrentDirectory(MAX_PATH + 1, path.raw)
-            if not res:
-                raise rwin32.lastWindowsError()
-            res = rffi.cast(lltype.Signed, res)
-            assert res > 0
-            if res <= MAX_PATH + 1:
-                new_path = path.str(res)
-            else:
-                with traits.scoped_alloc_buffer(res) as path:
-                    res = win32traits.GetCurrentDirectory(res, path.raw)
-                    if not res:
-                        raise rwin32.lastWindowsError()
-                    res = rffi.cast(lltype.Signed, res)
-                    assert res > 0
-                    new_path = path.str(res)
-        if isUNC(new_path):
-            return
-        if not win32traits.SetEnvironmentVariable(magic_envvar(new_path), new_path):
-            raise rwin32.lastWindowsError()
-
-    return chdir_llimpl
-
-#_______________________________________________________________
-# chmod
-
-def make_chmod_impl(traits):
-    from rpython.rlib import rwin32
-    from rpython.rlib.rwin32file import make_win32_traits
-
-    win32traits = make_win32_traits(traits)
-
-    @func_renamer('chmod_llimpl_%s' % traits.str.__name__)
-    def chmod_llimpl(path, mode):
-        attr = win32traits.GetFileAttributes(path)
-        if attr == win32traits.INVALID_FILE_ATTRIBUTES:
-            raise rwin32.lastWindowsError()
-        if mode & 0200: # _S_IWRITE
-            attr &= ~win32traits.FILE_ATTRIBUTE_READONLY
-        else:
-            attr |= win32traits.FILE_ATTRIBUTE_READONLY
-        if not win32traits.SetFileAttributes(path, attr):
-            raise rwin32.lastWindowsError()
-
-    return chmod_llimpl
-
-#_______________________________________________________________
-# getfullpathname
-
-def make_getfullpathname_impl(traits):
-    from rpython.rlib import rwin32
-    win32traits = make_win32_traits(traits)
-
-    @func_renamer('getfullpathname_llimpl_%s' % traits.str.__name__)
-    def getfullpathname_llimpl(path):
-        nBufferLength = rwin32.MAX_PATH + 1
-        lpBuffer = lltype.malloc(traits.CCHARP.TO, nBufferLength, flavor='raw')
-        try:
-            res = win32traits.GetFullPathName(
-                path, rffi.cast(rwin32.DWORD, nBufferLength),
-                lpBuffer, lltype.nullptr(win32traits.LPSTRP.TO))
-            if res == 0:
-                raise rwin32.lastWindowsError("_getfullpathname failed")
-            result = traits.charp2str(lpBuffer)
-            return result
-        finally:
-            lltype.free(lpBuffer, flavor='raw')
-
-    return getfullpathname_llimpl

From noreply at buildbot.pypy.org  Sat Nov  8 23:44:54 2014
From: noreply at buildbot.pypy.org (amauryfa)
Date: Sat,  8 Nov 2014 23:44:54 +0100 (CET)
Subject: [pypy-commit] pypy more-rposix: Port most remaining functions of
	ll_os.py
Message-ID: <20141108224454.E25E61D283E@cobra.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc 
Branch: more-rposix
Changeset: r74407:c7d5e4e3a801
Date: 2014-11-08 23:25 +0100
http://bitbucket.org/pypy/pypy/changeset/c7d5e4e3a801/

Log:	Port most remaining functions of ll_os.py

	To do: os.stat...

diff --git a/pypy/module/posix/interp_posix.py b/pypy/module/posix/interp_posix.py
--- a/pypy/module/posix/interp_posix.py
+++ b/pypy/module/posix/interp_posix.py
@@ -665,7 +665,7 @@
 def kill(space, pid, sig):
     "Kill a process with a signal."
     try:
-        rposix.os_kill(pid, sig)
+        rposix.kill(pid, sig)
     except OSError, e:
         raise wrap_oserror(space, e)
 
@@ -681,7 +681,7 @@
     """Abort the interpreter immediately.  This 'dumps core' or otherwise fails
 in the hardest way possible on the hosting operating system."""
     import signal
-    rposix.os_kill(os.getpid(), signal.SIGABRT)
+    rposix.kill(os.getpid(), signal.SIGABRT)
 
 @unwrap_spec(src='str0', dst='str0')
 def link(space, src, dst):
diff --git a/rpython/rlib/rposix.py b/rpython/rlib/rposix.py
--- a/rpython/rlib/rposix.py
+++ b/rpython/rlib/rposix.py
@@ -275,10 +275,6 @@
     return os.statvfs(_as_bytes(path))
 
 @specialize.argtype(0, 1)
-def symlink(src, dest):
-    os.symlink(_as_bytes(src), _as_bytes(dest))
-
- at specialize.argtype(0, 1)
 def putenv(name, value):
     os.environ[_as_bytes(name)] = _as_bytes(value)
 
@@ -286,12 +282,6 @@
 def unsetenv(name):
     del os.environ[_as_bytes(name)]
 
-if os.name == 'nt':
-    from rpython.rlib import rwin32
-    os_kill = rwin32.os_kill
-else:
-    os_kill = os.kill
-
 #___________________________________________________________________
 # Implementation of many posix functions.
 # They usually check the return value and raise an (RPython) OSError
@@ -957,6 +947,23 @@
 def mknod(path, mode, dev):
     handle_posix_error('mknod', c_mknod(_as_bytes0(path), mode, dev))
 
+c_link = external('link', [rffi.CCHARP, rffi.CCHARP], rffi.INT)
+c_symlink = external('symlink', [rffi.CCHARP, rffi.CCHARP], rffi.INT)
+
+ at replace_os_function('link')
+ at specialize.argtype(0, 1)
+def link(oldpath, newpath):
+    oldpath = _as_bytes0(oldpath)
+    newpath = _as_bytes0(newpath)
+    handle_posix_error('link', c_link(oldpath, newpath))
+
+ at replace_os_function('symlink')
+ at specialize.argtype(0, 1)
+def symlink(oldpath, newpath):
+    oldpath = _as_bytes0(oldpath)
+    newpath = _as_bytes0(newpath)
+    handle_posix_error('symlink', c_symlink(oldpath, newpath))
+
 c_umask = external(UNDERSCORE_ON_WIN32 + 'umask', [rffi.MODE_T], rffi.MODE_T)
 
 @replace_os_function('umask')
@@ -1145,6 +1152,64 @@
             lltype.free(pexit,   flavor='raw')
             lltype.free(pcreate, flavor='raw')
 
+c_kill = external('kill', [rffi.PID_T, rffi.INT], rffi.INT)
+c_killpg = external('killpg', [rffi.INT, rffi.INT], rffi.INT)
+c_exit = external('_exit', [rffi.INT], lltype.Void)
+c_nice = external('nice', [rffi.INT], rffi.INT)
+
+ at replace_os_function('kill')
+def kill(pid, sig):
+    if not _WIN32:
+        return handle_posix_error('kill', c_kill(pid, sig))
+    else:
+        if sig == rwin32.CTRL_C_EVENT or sig == rwin32.CTRL_BREAK_EVENT:
+            if rwin32.GenerateConsoleCtrlEvent(sig, pid) == 0:
+                raise rwin32.lastWindowsError('kill() failed generating event')
+            return
+        handle = rwin32.OpenProcess(rwin32.PROCESS_ALL_ACCESS, False, pid)
+        if not handle:
+            raise rwin32.lastWindowsError('kill() failed opening process')
+        try:
+            if rwin32.TerminateProcess(handle, sig) == 0:
+                raise rwin32.lastWindowsError(
+                    'kill() failed to terminate process')
+        finally:
+            rwin32.CloseHandle(handle)
+
+ at replace_os_function('killpg')
+def killpg(pgrp, sig):
+    return handle_posix_error('killpg', c_killpg(pgrp, sig))
+
+ at replace_os_function('_exit')
+def exit(status):
+    debug.debug_flush()
+    c_exit(status)
+
+ at replace_os_function('nice')
+def nice(inc):
+    # Assume that the system provides a standard-compliant version
+    # of nice() that returns the new priority.  Nowadays, FreeBSD
+    # might be the last major non-compliant system (xxx check me).
+    set_errno(0)
+    res = widen(c_nice(inc))
+    if res == -1:
+        err = get_errno()
+        if err != 0:
+            raise OSError(err, "os_nice failed")
+    return res
+
+c_ctermid = external('ctermid', [rffi.CCHARP], rffi.CCHARP)
+
+ at replace_os_function('ctermid')
+def ctermid():
+    return rffi.charp2str(c_ctermid(lltype.nullptr(rffi.CCHARP.TO)))
+
+c_tmpnam = external('tmpnam', [rffi.CCHARP], rffi.CCHARP)
+
+ at replace_os_function('tmpnam')
+def tmpnam():
+    return rffi.charp2str(c_tmpnam(lltype.nullptr(rffi.CCHARP.TO)))
+
 #___________________________________________________________________
 
 c_getpid = external('getpid', [], rffi.PID_T, releasegil=False)
diff --git a/rpython/rlib/rwin32.py b/rpython/rlib/rwin32.py
--- a/rpython/rlib/rwin32.py
+++ b/rpython/rlib/rwin32.py
@@ -389,18 +389,4 @@
     def GetConsoleOutputCP():
         return rffi.cast(lltype.Signed, _GetConsoleOutputCP())
 
-    def os_kill(pid, sig):
-        if sig == CTRL_C_EVENT or sig == CTRL_BREAK_EVENT:
-            if GenerateConsoleCtrlEvent(sig, pid) == 0:
-                raise lastWindowsError('os_kill failed generating event')
-            return
-        handle = OpenProcess(PROCESS_ALL_ACCESS, False, pid)
-        if handle == NULL_HANDLE:
-            raise lastWindowsError('os_kill failed opening process')
-        try:
-            if TerminateProcess(handle, sig) == 0:
-                raise lastWindowsError('os_kill failed to terminate process')
-        finally:
-            CloseHandle(handle)
-
     _wenviron_items, _wgetenv, _wputenv = make_env_impls(win32=True)
diff --git a/rpython/rtyper/module/ll_os.py b/rpython/rtyper/module/ll_os.py
--- a/rpython/rtyper/module/ll_os.py
+++ b/rpython/rtyper/module/ll_os.py
@@ -173,109 +173,6 @@
                 separate_module_sources = ["\n".join(defs)]
             ))
 
-    @registering_if(os, 'kill', sys.platform != 'win32')
-    def register_os_kill(self):
-        os_kill = self.llexternal('kill', [rffi.PID_T, rffi.INT],
-                                  rffi.INT)
-        def kill_llimpl(pid, sig):
-            res = rffi.cast(lltype.Signed, os_kill(rffi.cast(rffi.PID_T, pid),
-                                                   rffi.cast(rffi.INT, sig)))
-            if res < 0:
-                raise OSError(rposix.get_errno(), "os_kill failed")
-        return extdef([int, int], s_None, llimpl=kill_llimpl,
-                      export_name="ll_os.ll_os_kill")
-
-    @registering_if(os, 'killpg')
-    def register_os_killpg(self):
-        os_killpg = self.llexternal('killpg', [rffi.INT, rffi.INT],
-                                    rffi.INT)
-
-        def killpg_llimpl(pid, sig):
-            res = rffi.cast(lltype.Signed, os_killpg(rffi.cast(rffi.INT, pid),
-                                                     rffi.cast(rffi.INT, sig)))
-            if res < 0:
-                raise OSError(rposix.get_errno(), "os_killpg failed")
-
-        return extdef([int, int], s_None, llimpl=killpg_llimpl,
-                      export_name="ll_os.ll_os_killpg")
-
-    @registering_if(os, 'link')
-    def register_os_link(self):
-        os_link = self.llexternal('link', [rffi.CCHARP, rffi.CCHARP],
-                                  rffi.INT)
-
-        def link_llimpl(oldpath, newpath):
-            res = rffi.cast(lltype.Signed, os_link(oldpath, newpath))
-            if res < 0:
-                raise OSError(rposix.get_errno(), "os_link failed")
-
-        return extdef([str0, str0], s_None, llimpl=link_llimpl,
-                      export_name="ll_os.ll_os_link")
-
-    @registering_if(os, 'symlink')
-    def register_os_symlink(self):
-        os_symlink = self.llexternal('symlink', [rffi.CCHARP, rffi.CCHARP],
-                                     rffi.INT)
-
-        def symlink_llimpl(oldpath, newpath):
-            res = rffi.cast(lltype.Signed, os_symlink(oldpath, newpath))
-            if res < 0:
-                raise OSError(rposix.get_errno(), "os_symlink failed")
-
-        return extdef([str0, str0], s_None, llimpl=symlink_llimpl,
-                      export_name="ll_os.ll_os_symlink")
-
-    @registering(os._exit)
-    def register_os__exit(self):
-        from rpython.rlib import debug
-        os__exit = self.llexternal('_exit', [rffi.INT], lltype.Void)
-
-        def _exit_llimpl(status):
-            debug.debug_flush()
-            os__exit(rffi.cast(rffi.INT, status))
-
-        return extdef([int], s_None, llimpl=_exit_llimpl,
-                      export_name="ll_os.ll_os__exit")
-
-    @registering_if(os, 'nice')
-    def register_os_nice(self):
-        os_nice = self.llexternal('nice', [rffi.INT], rffi.INT)
-
-        def nice_llimpl(inc):
-            # Assume that the system provides a standard-compliant version
-            # of nice() that returns the new priority.  Nowadays, FreeBSD
-            # might be the last major non-compliant system (xxx check me).
-            rposix.set_errno(0)
-            res = rffi.cast(lltype.Signed, os_nice(inc))
-            if res == -1:
-                err = rposix.get_errno()
-                if err != 0:
-                    raise OSError(err, "os_nice failed")
-            return res
-
-        return extdef([int], int, llimpl=nice_llimpl,
-                      export_name="ll_os.ll_os_nice")
-
-    @registering_if(os, 'ctermid')
-    def register_os_ctermid(self):
-        os_ctermid = self.llexternal('ctermid', [rffi.CCHARP], rffi.CCHARP)
-
-        def ctermid_llimpl():
-            return rffi.charp2str(os_ctermid(lltype.nullptr(rffi.CCHARP.TO)))
-
-        return extdef([], str, llimpl=ctermid_llimpl,
-                      export_name="ll_os.ll_os_ctermid")
-
-    @registering_if(os, 'tmpnam')
-    def register_os_tmpnam(self):
-        os_tmpnam = self.llexternal('tmpnam', [rffi.CCHARP], rffi.CCHARP)
-
-        def tmpnam_llimpl():
-            return rffi.charp2str(os_tmpnam(lltype.nullptr(rffi.CCHARP.TO)))
-
-        return extdef([], str, llimpl=tmpnam_llimpl,
-                      export_name="ll_os.ll_os_tmpnam")
-
 # --------------------------- os.stat & variants ---------------------------
 
     @registering(os.fstat)
diff --git a/rpython/rtyper/module/test/test_ll_os.py b/rpython/rtyper/module/test/test_ll_os.py
--- a/rpython/rtyper/module/test/test_ll_os.py
+++ b/rpython/rtyper/module/test/test_ll_os.py
@@ -89,7 +89,7 @@
     posix = __import__(os.name)
     sysdrv = os.getenv('SystemDrive', 'C:')
     stuff = sysdrv + 'stuff'
-    data = getllimpl(posix._getfullpathname)(stuff)
+    data = rposix.getfullpathname(stuff)
     assert data == posix._getfullpathname(stuff)
     # the most intriguing failure of ntpath.py should not repeat, here:
     assert not data.endswith(stuff)
@@ -116,7 +116,7 @@
     pwd = os.getcwd()
     try:
         check_special_envvar()
-        getllimpl(os.chdir)('..')
+        rposix.chdir('..')
         assert os.getcwd() == os.path.dirname(pwd)
         check_special_envvar()
     finally:
@@ -124,20 +124,19 @@
 
 def test_mkdir():
     filename = str(udir.join('test_mkdir.dir'))
-    getllimpl(os.mkdir)(filename, 0)
-    exc = py.test.raises(OSError, getllimpl(os.mkdir), filename, 0)
+    rposix.mkdir(filename, 0)
+    exc = py.test.raises(OSError, rposix.mkdir, filename, 0)
     assert exc.value.errno == errno.EEXIST
     if sys.platform == 'win32':
         assert exc.type is WindowsError
 
 def test_strerror():
-    data = getllimpl(os.strerror)(2)
-    assert data == os.strerror(2)
+    assert rposix.strerror(2) == os.strerror(2)
 
 def test_system():
     filename = str(udir.join('test_system.txt'))
     arg = '%s -c "print 1+1" > %s' % (sys.executable, filename)
-    data = getllimpl(os.system)(arg)
+    data = rposix.system(arg)
     assert data == 0
     assert file(filename).read().strip() == '2'
     os.unlink(filename)
@@ -267,9 +266,6 @@
 
 
 def test_os_kill():
-    if not hasattr(os,'kill') or sys.platform == 'win32':
-        py.test.skip('No kill in os')
-    f = getllimpl(os.kill)
     import subprocess
     import signal
     proc = subprocess.Popen([sys.executable, "-c",
@@ -277,16 +273,12 @@
                          "time.sleep(10)",
                          ],
                         )
-    f(proc.pid, signal.SIGTERM)
+    rposix.kill(proc.pid, signal.SIGTERM)
     expected = -signal.SIGTERM
     assert proc.wait() == expected
 
 def test_isatty():
-    try:
-        f = getllimpl(os.isatty)
-    except:
-        py.test.skip('No isatty in os')
-    assert f(-1)  == False
+    assert rposix.isatty(-1) is False
 
 
 class TestOsExpect(ExpectTest):

From noreply at buildbot.pypy.org  Sat Nov  8 23:44:56 2014
From: noreply at buildbot.pypy.org (amauryfa)
Date: Sat,  8 Nov 2014 23:44:56 +0100 (CET)
Subject: [pypy-commit] pypy more-rposix: Move os.WIFEXITED and other macros
Message-ID: <20141108224456.1C0D11D283E@cobra.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc 
Branch: more-rposix
Changeset: r74408:9eb5ecbc8570
Date: 2014-11-08 23:44 +0100
http://bitbucket.org/pypy/pypy/changeset/9eb5ecbc8570/

Log:	Move os.WIFEXITED and other macros

diff --git a/rpython/rlib/rposix.py b/rpython/rlib/rposix.py
--- a/rpython/rlib/rposix.py
+++ b/rpython/rlib/rposix.py
@@ -5,6 +5,7 @@
 from rpython.rtyper.lltypesystem import lltype, ll2ctypes, rffi
 from rpython.rtyper.module.support import StringTraits, UnicodeTraits
 from rpython.rtyper.tool import rffi_platform
+from rpython.tool.sourcetools import func_renamer
 from rpython.translator.tool.cbuild import ExternalCompilationInfo
 from rpython.rlib.rarithmetic import intmask, widen
 from rpython.rlib.objectmodel import (
@@ -143,7 +144,7 @@
     includes = ['io.h', 'sys/utime.h', 'sys/types.h']
     libraries = []
 else:
-    includes = ['unistd.h',  'sys/types.h',
+    includes = ['unistd.h',  'sys/types.h', 'sys/wait.h',
                 'utime.h', 'sys/time.h', 'sys/times.h',
                 'grp.h', 'dirent.h']
     libraries = ['util']
@@ -157,6 +158,7 @@
     SEEK_SET = rffi_platform.DefinedConstantInteger('SEEK_SET')
     SEEK_CUR = rffi_platform.DefinedConstantInteger('SEEK_CUR')
     SEEK_END = rffi_platform.DefinedConstantInteger('SEEK_END')
+    OFF_T_SIZE = rffi_platform.SizeOf('off_t')
 
     HAVE_UTIMES = rffi_platform.Has('utimes')
     UTIMBUF = rffi_platform.Struct('struct %sutimbuf' % UNDERSCORE_ON_WIN32,
@@ -181,6 +183,11 @@
     return rffi.llexternal(name, args, result,
                            compilation_info=compilation_info, **kwds)
 
+# For now we require off_t to be the same size as LONGLONG, which is the
+# interface required by callers of functions that thake an argument of type
+# off_t.
+assert OFF_T_SIZE == rffi.sizeof(rffi.LONGLONG)
+
 c_dup = external(UNDERSCORE_ON_WIN32 + 'dup', [rffi.INT], rffi.INT)
 c_dup2 = external(UNDERSCORE_ON_WIN32 + 'dup2', [rffi.INT, rffi.INT], rffi.INT)
 c_open = external(UNDERSCORE_ON_WIN32 + 'open',
@@ -743,43 +750,23 @@
     finally:
         lltype.free(status_p, flavor='raw')
 
-if _WIN32:
-    CreatePipe = external('CreatePipe', [rwin32.LPHANDLE,
-                                         rwin32.LPHANDLE,
-                                         rffi.VOIDP,
-                                         rwin32.DWORD],
-                          rwin32.BOOL)
-    c_open_osfhandle = external('_open_osfhandle', [rffi.INTPTR_T,
-                                                    rffi.INT],
-                                rffi.INT)
-else:
-    INT_ARRAY_P = rffi.CArrayPtr(rffi.INT)
-    c_pipe = external('pipe', [INT_ARRAY_P], rffi.INT)
+def _make_waitmacro(name):
+    c_func = external(name, [lltype.Signed], lltype.Signed,
+                      macro=True)
+    returning_int = name in ('WEXITSTATUS', 'WSTOPSIG', 'WTERMSIG')
 
- at replace_os_function('pipe')
-def pipe():
-    if _WIN32:
-        pread  = lltype.malloc(rwin32.LPHANDLE.TO, 1, flavor='raw')
-        pwrite = lltype.malloc(rwin32.LPHANDLE.TO, 1, flavor='raw')
-        try:
-            if not CreatePipe(
-                    pread, pwrite, lltype.nullptr(rffi.VOIDP.TO), 0):
-                raise WindowsError(rwin32.GetLastError(), "CreatePipe failed")
-            hread = rffi.cast(rffi.INTPTR_T, pread[0])
-            hwrite = rffi.cast(rffi.INTPTR_T, pwrite[0])
-        finally:
-            lltype.free(pwrite, flavor='raw')
-            lltype.free(pread, flavor='raw')
-        fdread = _open_osfhandle(hread, 0)
-        fdwrite = _open_osfhandle(hwrite, 1)
-        return (fdread, fdwrite)
-    else:
-        filedes = lltype.malloc(INT_ARRAY_P.TO, 2, flavor='raw')
-        try:
-            handle_posix_error('pipe', c_pipe(filedes))
-            return (widen(filedes[0]), widen(filedes[1]))
-        finally:
-            lltype.free(filedes, flavor='raw')
+    @replace_os_function(name)
+    @func_renamer(name)
+    def _waitmacro(status):
+        if returning_int:
+            return c_func(status)
+        else:
+            return bool(c_func(status))
+
+for name in ['WCOREDUMP', 'WIFCONTINUED', 'WIFSTOPPED',
+             'WIFSIGNALED', 'WIFEXITED',
+             'WEXITSTATUS', 'WSTOPSIG', 'WTERMSIG']:
+    _make_waitmacro(name)
 
 #___________________________________________________________________
 
@@ -932,6 +919,8 @@
         if not win32traits.MoveFile(path1, path2):
             raise rwin32.lastWindowsError()
 
+#___________________________________________________________________
+
 c_mkfifo = external('mkfifo', [rffi.CCHARP, rffi.MODE_T], rffi.INT)
 c_mknod = external('mknod', [rffi.CCHARP, rffi.MODE_T, rffi.INT], rffi.INT,
                    macro=True)
@@ -947,9 +936,49 @@
 def mknod(path, mode, dev):
     handle_posix_error('mknod', c_mknod(_as_bytes0(path), mode, dev))
 
+if _WIN32:
+    CreatePipe = external('CreatePipe', [rwin32.LPHANDLE,
+                                         rwin32.LPHANDLE,
+                                         rffi.VOIDP,
+                                         rwin32.DWORD],
+                          rwin32.BOOL)
+    c_open_osfhandle = external('_open_osfhandle', [rffi.INTPTR_T,
+                                                    rffi.INT],
+                                rffi.INT)
+else:
+    INT_ARRAY_P = rffi.CArrayPtr(rffi.INT)
+    c_pipe = external('pipe', [INT_ARRAY_P], rffi.INT)
+
+ at replace_os_function('pipe')
+def pipe():
+    if _WIN32:
+        pread  = lltype.malloc(rwin32.LPHANDLE.TO, 1, flavor='raw')
+        pwrite = lltype.malloc(rwin32.LPHANDLE.TO, 1, flavor='raw')
+        try:
+            if not CreatePipe(
+                    pread, pwrite, lltype.nullptr(rffi.VOIDP.TO), 0):
+                raise WindowsError(rwin32.GetLastError(), "CreatePipe failed")
+            hread = rffi.cast(rffi.INTPTR_T, pread[0])
+            hwrite = rffi.cast(rffi.INTPTR_T, pwrite[0])
+        finally:
+            lltype.free(pwrite, flavor='raw')
+            lltype.free(pread, flavor='raw')
+        fdread = _open_osfhandle(hread, 0)
+        fdwrite = _open_osfhandle(hwrite, 1)
+        return (fdread, fdwrite)
+    else:
+        filedes = lltype.malloc(INT_ARRAY_P.TO, 2, flavor='raw')
+        try:
+            handle_posix_error('pipe', c_pipe(filedes))
+            return (widen(filedes[0]), widen(filedes[1]))
+        finally:
+            lltype.free(filedes, flavor='raw')
+
 c_link = external('link', [rffi.CCHARP, rffi.CCHARP], rffi.INT)
 c_symlink = external('symlink', [rffi.CCHARP, rffi.CCHARP], rffi.INT)
 
+#___________________________________________________________________
+
 @replace_os_function('link')
 @specialize.argtype(0, 1)
 def link(oldpath, newpath):
diff --git a/rpython/rtyper/module/ll_os.py b/rpython/rtyper/module/ll_os.py
--- a/rpython/rtyper/module/ll_os.py
+++ b/rpython/rtyper/module/ll_os.py
@@ -128,50 +128,16 @@
                            ('tms_cutime', rffi.INT),
                            ('tms_cstime', rffi.INT)])
 
-    # For now we require off_t to be the same size as LONGLONG, which is the
-    # interface required by callers of functions that thake an argument of type
-    # off_t
-    OFF_T_SIZE = platform.SizeOf('off_t')
 
     SEEK_SET = platform.DefinedConstantInteger('SEEK_SET')
     SEEK_CUR = platform.DefinedConstantInteger('SEEK_CUR')
     SEEK_END = platform.DefinedConstantInteger('SEEK_END')
 
-    UTIMBUF = platform.Struct('struct %sutimbuf' % UNDERSCORE_ON_WIN32,
-                              [('actime', rffi.INT),
-                               ('modtime', rffi.INT)])
-
 
 class RegisterOs(BaseLazyRegistering):
 
     def __init__(self):
         self.configure(CConfig)
-        if not _WIN32:
-            assert self.OFF_T_SIZE == rffi.sizeof(rffi.LONGLONG)
-
-        # we need an indirection via c functions to get macro calls working on llvm XXX still?
-        if hasattr(os, 'WCOREDUMP'):
-            decl_snippet = """
-            %(ret_type)s pypy_macro_wrapper_%(name)s (int status);
-            """
-            def_snippet = """
-            %(ret_type)s pypy_macro_wrapper_%(name)s (int status) {
-            return %(name)s(status);
-            }
-            """
-            decls = []
-            defs = []
-            for name in self.w_star:
-                if hasattr(os, name):
-                    data = {'ret_type': 'int', 'name': name}
-                    decls.append((decl_snippet % data).strip())
-                    defs.append((def_snippet % data).strip())
-
-            self.compilation_info = self.compilation_info.merge(
-                ExternalCompilationInfo(
-                post_include_bits = decls,
-                separate_module_sources = ["\n".join(defs)]
-            ))
 
 # --------------------------- os.stat & variants ---------------------------
 
@@ -204,43 +170,6 @@
 
     # ------------------------------- os.W* ---------------------------------
 
-    w_star = ['WCOREDUMP', 'WIFCONTINUED', 'WIFSTOPPED',
-              'WIFSIGNALED', 'WIFEXITED', 'WEXITSTATUS',
-              'WSTOPSIG', 'WTERMSIG']
-    # last 3 are returning int
-    w_star_returning_int = dict.fromkeys(w_star[-3:])
-
-
-
-    def declare_new_w_star(self, name):
-        """ stupid workaround for the python late-binding
-        'feature'
-        """
-
-        def fake(status):
-            return int(getattr(os, name)(status))
-        fake.func_name = 'fake_' + name
-
-        os_c_func = self.llexternal("pypy_macro_wrapper_" + name,
-                                    [lltype.Signed], lltype.Signed,
-                                    _callable=fake)
-
-        if name in self.w_star_returning_int:
-            def llimpl(status):
-                return os_c_func(status)
-            resulttype = int
-        else:
-            def llimpl(status):
-                return bool(os_c_func(status))
-            resulttype = bool
-        llimpl.func_name = name + '_llimpl'
-        return extdef([int], resulttype, "ll_os." + name,
-                      llimpl=llimpl)
-
-    for name in w_star:
-        locals()['register_w_' + name] = registering_if(os, name)(
-            lambda self, xname=name : self.declare_new_w_star(xname))
-
     @registering_if(os, 'ttyname')
     def register_os_ttyname(self):
         os_ttyname = self.llexternal('ttyname', [lltype.Signed], rffi.CCHARP, releasegil=False)

From noreply at buildbot.pypy.org  Sun Nov  9 01:16:19 2014
From: noreply at buildbot.pypy.org (rlamy)
Date: Sun,  9 Nov 2014 01:16:19 +0100 (CET)
Subject: [pypy-commit] pypy default: Use sets in
	transform_dead_op_vars_in_blocks(); cleanup
Message-ID: <20141109001619.CAF9A1C0548@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy 
Branch: 
Changeset: r74409:bece2fadfa70
Date: 2014-11-08 23:56 +0000
http://bitbucket.org/pypy/pypy/changeset/bece2fadfa70/

Log:	Use sets in transform_dead_op_vars_in_blocks(); cleanup

diff --git a/rpython/translator/simplify.py b/rpython/translator/simplify.py
--- a/rpython/translator/simplify.py
+++ b/rpython/translator/simplify.py
@@ -5,6 +5,7 @@
 simplify_graph() applies all simplifications defined in this file.
 """
 import py
+from collections import defaultdict
 
 from rpython.flowspace.model import (Variable, Constant,
                                      c_last_exception, checkgraph, mkentrymap)
@@ -401,8 +402,8 @@
 def transform_dead_op_vars_in_blocks(blocks, graphs, translator=None):
     """Remove dead operations and variables that are passed over a link
     but not used in the target block. Input is a set of blocks"""
-    read_vars = {}  # set of variables really used
-    variable_flow = {}  # map {Var: list-of-Vars-it-depends-on}
+    read_vars = set()  # set of variables really used
+    dependencies = defaultdict(set) # map {Var: list-of-Vars-it-depends-on}
     set_of_blocks = set(blocks)
     start_blocks = find_start_blocks(graphs)
 
@@ -414,53 +415,48 @@
         # cannot remove the exc-raising operation
         return op is not block.operations[-1]
 
-    # compute variable_flow and an initial read_vars
+    # compute dependencies and an initial read_vars
     for block in blocks:
         # figure out which variables are ever read
         for op in block.operations:
-            if not canremove(op, block):   # mark the inputs as really needed
-                for arg in op.args:
-                    read_vars[arg] = True
+            if not canremove(op, block):   # the inputs are always needed
+                read_vars.update(op.args)
             else:
-                # if CanRemove, only mark dependencies of the result
-                # on the input variables
-                deps = variable_flow.setdefault(op.result, [])
-                deps.extend(op.args)
+                dependencies[op.result].update(op.args)
 
         if isinstance(block.exitswitch, Variable):
-            read_vars[block.exitswitch] = True
+            read_vars.add(block.exitswitch)
 
         if block.exits:
             for link in block.exits:
                 if link.target not in set_of_blocks:
                     for arg, targetarg in zip(link.args, link.target.inputargs):
-                        read_vars[arg] = True
-                        read_vars[targetarg] = True
+                        read_vars.add(arg)
+                        read_vars.add(targetarg)
                 else:
                     for arg, targetarg in zip(link.args, link.target.inputargs):
-                        deps = variable_flow.setdefault(targetarg, [])
-                        deps.append(arg)
+                        dependencies[targetarg].add(arg)
         else:
             # return and except blocks implicitely use their input variable(s)
             for arg in block.inputargs:
-                read_vars[arg] = True
-        # an input block's inputargs should not be modified, even if some
+                read_vars.add(arg)
+        # a start block's inputargs should not be modified, even if some
         # of the function's input arguments are not actually used
         if block in start_blocks:
             for arg in block.inputargs:
-                read_vars[arg] = True
+                read_vars.add(arg)
 
     # flow read_vars backwards so that any variable on which a read_vars
     # depends is also included in read_vars
     def flow_read_var_backward(pending):
-        pending = list(pending)
-        for var in pending:
-            for prevvar in variable_flow.get(var, []):
+        while pending:
+            var = pending.pop()
+            for prevvar in dependencies[var]:
                 if prevvar not in read_vars:
-                    read_vars[prevvar] = True
-                    pending.append(prevvar)
+                    read_vars.add(prevvar)
+                    pending.add(prevvar)
 
-    flow_read_var_backward(read_vars)
+    flow_read_var_backward(set(read_vars))
 
     for block in blocks:
 

From noreply at buildbot.pypy.org  Sun Nov  9 01:27:05 2014
From: noreply at buildbot.pypy.org (rlamy)
Date: Sun,  9 Nov 2014 01:27:05 +0100 (CET)
Subject: [pypy-commit] pypy online-transforms: store assignment value on the
 result of op.assign
Message-ID: <20141109002705.2868E1D283E@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy 
Branch: online-transforms
Changeset: r74410:1e1f2ee75e44
Date: 2014-11-07 18:51 +0000
http://bitbucket.org/pypy/pypy/changeset/1e1f2ee75e44/

Log:	store assignment value on the result of op.assign

diff --git a/rpython/annotator/unaryop.py b/rpython/annotator/unaryop.py
--- a/rpython/annotator/unaryop.py
+++ b/rpython/annotator/unaryop.py
@@ -9,7 +9,7 @@
 from rpython.annotator.model import (SomeObject, SomeInteger, SomeBool,
     SomeString, SomeChar, SomeList, SomeDict, SomeTuple, SomeImpossibleValue,
     SomeUnicodeCodePoint, SomeInstance, SomeBuiltin, SomeUnboundMethod, SomeBuiltinMethod,
-    SomeFloat, SomeIterator, SomePBC, SomeNone, SomeType, s_ImpossibleValue,
+    SomeFloat, SomeIterator, SomePBC, SomeNone, s_ImpossibleValue,
     s_Bool, s_None, unionof, add_knowntypedata,
     HarmlesslyBlocked, SomeWeakRef, SomeUnicodeString, SomeByteArray)
 from rpython.annotator.bookkeeper import getbookkeeper, immutablevalue
@@ -17,7 +17,6 @@
 from rpython.annotator.binaryop import _clone ## XXX where to put this?
 from rpython.annotator.model import AnnotatorError
 from rpython.annotator.argument import simple_args, complex_args
-from rpython.annotator.expression import V_Type
 
 UNARY_OPERATIONS = set([oper.opname for oper in op.__dict__.values()
                         if oper.dispatch == 1])
@@ -27,10 +26,6 @@
 def assign(annotator, v_obj):
     return annotator.annotation(v_obj)
 
- at op.type.register_transform(SomeObject)
-def type_SomeObject(annotator, v_arg):
-    return [op.assign(V_Type(v_arg))]
-
 @op.bool.register(SomeObject)
 def bool_SomeObject(annotator, obj):
     r = SomeBool()
diff --git a/rpython/flowspace/model.py b/rpython/flowspace/model.py
--- a/rpython/flowspace/model.py
+++ b/rpython/flowspace/model.py
@@ -250,7 +250,7 @@
 
 
 class Variable(object):
-    __slots__ = ["_name", "_nr", "annotation", "concretetype"]
+    __slots__ = ["_name", "_nr", "annotation", "concretetype", "equals"]
 
     dummyname = 'v'
     namesdict = {dummyname: (dummyname, 0)}
diff --git a/rpython/flowspace/operation.py b/rpython/flowspace/operation.py
--- a/rpython/flowspace/operation.py
+++ b/rpython/flowspace/operation.py
@@ -498,6 +498,18 @@
     def get_specialization(cls, s_seq, s_elem):
         return cls._dispatch(type(s_seq))
 
+class Type(SingleDispatchMixin, PureOperation):
+    opname = 'type'
+    arity = 1
+    canraise = []
+    pyfunc = staticmethod(new_style_type)
+
+    def transform(self, annotator):
+        from rpython.annotator.expression import V_Type
+        value = V_Type(self.args[0])
+        self.result.equals = value
+        return [op.assign(value)]
+
 
 class NewDict(HLOperation):
     opname = 'newdict'

From noreply at buildbot.pypy.org  Sun Nov  9 01:27:06 2014
From: noreply at buildbot.pypy.org (rlamy)
Date: Sun,  9 Nov 2014 01:27:06 +0100 (CET)
Subject: [pypy-commit] pypy online-transforms: rename all the things in
 annotator.follow_link()
Message-ID: <20141109002706.758471D283E@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy 
Branch: online-transforms
Changeset: r74411:388640976ffd
Date: 2014-11-07 19:31 +0000
http://bitbucket.org/pypy/pypy/changeset/388640976ffd/

Log:	rename all the things in annotator.follow_link()

diff --git a/rpython/annotator/annrpython.py b/rpython/annotator/annrpython.py
--- a/rpython/annotator/annrpython.py
+++ b/rpython/annotator/annrpython.py
@@ -494,87 +494,84 @@
 
     def follow_link(self, graph, link, knowntypedata):
         in_except_block = False
-        last_exception_var = link.last_exception  # may be None for non-exception link
-        last_exc_value_var = link.last_exc_value  # may be None for non-exception link
+        v_last_exc_type = link.last_exception  # may be None for non-exception link
+        v_last_exc_value = link.last_exc_value  # may be None for non-exception link
 
-        if isinstance(link.exitcase, (types.ClassType, type)) \
-                and issubclass(link.exitcase, py.builtin.BaseException):
-            assert last_exception_var and last_exc_value_var
-            last_exc_value_object = self.bookkeeper.valueoftype(link.exitcase)
-            last_exception_object = annmodel.SomeType()
-            if isinstance(last_exception_var, Constant):
-                last_exception_object.const = last_exception_var.value
-            last_exception_object.is_type_of = [last_exc_value_var]
+        if (isinstance(link.exitcase, (types.ClassType, type)) and
+                issubclass(link.exitcase, BaseException)):
+            assert v_last_exc_type and v_last_exc_value
+            s_last_exc_value = self.bookkeeper.valueoftype(link.exitcase)
+            s_last_exc_type = annmodel.SomeType()
+            if isinstance(v_last_exc_type, Constant):
+                s_last_exc_type.const = v_last_exc_type.value
+            s_last_exc_type.is_type_of = [v_last_exc_value]
 
-            if isinstance(last_exception_var, Variable):
-                self.setbinding(last_exception_var, last_exception_object)
-            if isinstance(last_exc_value_var, Variable):
-                self.setbinding(last_exc_value_var, last_exc_value_object)
+            if isinstance(v_last_exc_type, Variable):
+                self.setbinding(v_last_exc_type, s_last_exc_type)
+            if isinstance(v_last_exc_value, Variable):
+                self.setbinding(v_last_exc_value, s_last_exc_value)
 
-            last_exception_object = annmodel.SomeType()
-            if isinstance(last_exception_var, Constant):
-                last_exception_object.const = last_exception_var.value
-            #if link.exitcase is Exception:
-            #    last_exc_value_object = annmodel.SomeObject()
-            #else:
+            s_last_exc_type = annmodel.SomeType()
+            if isinstance(v_last_exc_type, Constant):
+                s_last_exc_type.const = v_last_exc_type.value
             last_exc_value_vars = []
             in_except_block = True
 
         ignore_link = False
-        cells = []
+        inputs_s = []
         renaming = {}
-        for a, v in zip(link.args, link.target.inputargs):
-            renaming.setdefault(a, []).append(v)
-        for a, v in zip(link.args, link.target.inputargs):
-            if a == last_exception_var:
+        for v_out, v_input in zip(link.args, link.target.inputargs):
+            renaming.setdefault(v_out, []).append(v_input)
+        for v_out, v_input in zip(link.args, link.target.inputargs):
+            if v_out == v_last_exc_type:
                 assert in_except_block
-                cells.append(last_exception_object)
-            elif a == last_exc_value_var:
+                inputs_s.append(s_last_exc_type)
+            elif v_out == v_last_exc_value:
                 assert in_except_block
-                cells.append(last_exc_value_object)
-                last_exc_value_vars.append(v)
+                inputs_s.append(s_last_exc_value)
+                last_exc_value_vars.append(v_input)
             else:
-                cell = self.binding(a)
-                if (link.exitcase, a) in knowntypedata:
-                    knownvarvalue = knowntypedata[(link.exitcase, a)]
-                    cell = pair(cell, knownvarvalue).improve()
+                s_out = self.annotation(v_out)
+                if (link.exitcase, v_out) in knowntypedata:
+                    knownvarvalue = knowntypedata[(link.exitcase, v_out)]
+                    s_out = pair(s_out, knownvarvalue).improve()
                     # ignore links that try to pass impossible values
-                    if cell == annmodel.s_ImpossibleValue:
+                    if s_out == annmodel.s_ImpossibleValue:
                         ignore_link = True
 
-                if hasattr(cell,'is_type_of'):
+                if hasattr(s_out,'is_type_of'):
                     renamed_is_type_of = []
-                    for v in cell.is_type_of:
-                        new_vs = renaming.get(v,[])
+                    for v in s_out.is_type_of:
+                        new_vs = renaming.get(v, [])
                         renamed_is_type_of += new_vs
-                    assert cell.knowntype is type
+                    assert s_out.knowntype is type
                     newcell = annmodel.SomeType()
-                    if cell.is_constant():
-                        newcell.const = cell.const
-                    cell = newcell
-                    cell.is_type_of = renamed_is_type_of
+                    if s_out.is_constant():
+                        newcell.const = s_out.const
+                    s_out = newcell
+                    s_out.is_type_of = renamed_is_type_of
 
-                if hasattr(cell, 'knowntypedata'):
+                if hasattr(s_out, 'knowntypedata'):
                     renamed_knowntypedata = {}
-                    for (value, v), s in cell.knowntypedata.items():
+                    for (value, v), s in s_out.knowntypedata.items():
                         new_vs = renaming.get(v, [])
                         for new_v in new_vs:
                             renamed_knowntypedata[value, new_v] = s
-                    assert isinstance(cell, annmodel.SomeBool)
+                    assert isinstance(s_out, annmodel.SomeBool)
                     newcell = annmodel.SomeBool()
-                    if cell.is_constant():
-                        newcell.const = cell.const
-                    cell = newcell
-                    cell.set_knowntypedata(renamed_knowntypedata)
+                    if s_out.is_constant():
+                        newcell.const = s_out.const
+                    s_out = newcell
+                    s_out.set_knowntypedata(renamed_knowntypedata)
 
-                cells.append(cell)
+                inputs_s.append(s_out)
         if ignore_link:
             return
 
         if in_except_block:
-            last_exception_object.is_type_of = last_exc_value_vars
+            s_last_exc_type.is_type_of = last_exc_value_vars
         self.links_followed[link] = True
-        self.addpendingblock(graph, link.target, cells)
+        self.addpendingblock(graph, link.target, inputs_s)
 
     #___ creating the annotations based on operations ______
 

From noreply at buildbot.pypy.org  Sun Nov  9 01:27:07 2014
From: noreply at buildbot.pypy.org (rlamy)
Date: Sun,  9 Nov 2014 01:27:07 +0100 (CET)
Subject: [pypy-commit] pypy online-transforms: generate V_Type objects
 directly during flowing
Message-ID: <20141109002707.B66401D283E@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy 
Branch: online-transforms
Changeset: r74412:c6b3999a0d78
Date: 2014-11-08 01:57 +0000
http://bitbucket.org/pypy/pypy/changeset/c6b3999a0d78/

Log:	generate V_Type objects directly during flowing

diff --git a/rpython/annotator/expression.py b/rpython/annotator/expression.py
--- a/rpython/annotator/expression.py
+++ b/rpython/annotator/expression.py
@@ -12,3 +12,6 @@
 
     def as_operation(self):
         return op.type(self.arg)
+
+    def __eq__(self, other):
+        return isinstance(other, V_Type) and other.arg == self.arg
diff --git a/rpython/flowspace/model.py b/rpython/flowspace/model.py
--- a/rpython/flowspace/model.py
+++ b/rpython/flowspace/model.py
@@ -649,14 +649,10 @@
                     assert link.last_exc_value is None
                 for v in link.args:
                     assert isinstance(v, (Constant, Variable))
-                    if isinstance(v, Variable):
+                    if type(v) is Variable:
                         usevar(v, in_link=link)
                         if exc_link:
                             assert v != block.operations[-1].result
-                    #else:
-                    #    if not exc_link:
-                    #        assert v.value is not last_exception
-                    #        #assert v.value != last_exc_value
                 allexitcases[link.exitcase] = True
             assert len(allexitcases) == len(block.exits)
             vars_previous_blocks.update(vars)
diff --git a/rpython/flowspace/operation.py b/rpython/flowspace/operation.py
--- a/rpython/flowspace/operation.py
+++ b/rpython/flowspace/operation.py
@@ -411,7 +411,7 @@
 
 add_operator('is_', 2, dispatch=2, pure=True)
 add_operator('id', 1, dispatch=1, pyfunc=id)
-add_operator('type', 1, dispatch=1, pyfunc=new_style_type, pure=True)
+#add_operator('type', 1, dispatch=1, pyfunc=new_style_type, pure=True)
 add_operator('issubtype', 2, dispatch=1, pyfunc=issubclass, pure=True)  # not for old-style classes
 add_operator('repr', 1, dispatch=1, pyfunc=repr, pure=True)
 add_operator('str', 1, dispatch=1, pyfunc=str, pure=True)
@@ -504,11 +504,14 @@
     canraise = []
     pyfunc = staticmethod(new_style_type)
 
-    def transform(self, annotator):
+    def eval(self, ctx):
+        result = self.constfold()
+        if result is not None:
+            return result
         from rpython.annotator.expression import V_Type
-        value = V_Type(self.args[0])
-        self.result.equals = value
-        return [op.assign(value)]
+        v_instance, = self.args
+        result = V_Type(v_instance)
+        return ctx.do_op(op.assign(result))
 
 
 class NewDict(HLOperation):

From noreply at buildbot.pypy.org  Sun Nov  9 01:27:08 2014
From: noreply at buildbot.pypy.org (rlamy)
Date: Sun,  9 Nov 2014 01:27:08 +0100 (CET)
Subject: [pypy-commit] pypy online-transforms: apply variable renaming to
	V_Type
Message-ID: <20141109002708.EA48F1D283E@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy 
Branch: online-transforms
Changeset: r74413:40193595f3d6
Date: 2014-11-08 18:31 +0000
http://bitbucket.org/pypy/pypy/changeset/40193595f3d6/

Log:	apply variable renaming to V_Type

diff --git a/rpython/annotator/expression.py b/rpython/annotator/expression.py
--- a/rpython/annotator/expression.py
+++ b/rpython/annotator/expression.py
@@ -15,3 +15,9 @@
 
     def __eq__(self, other):
         return isinstance(other, V_Type) and other.arg == self.arg
+
+    def replace(self, mapping):
+        if self.arg in mapping:
+            return V_Type(mapping[self.arg])
+        else:
+            return self
diff --git a/rpython/flowspace/flowcontext.py b/rpython/flowspace/flowcontext.py
--- a/rpython/flowspace/flowcontext.py
+++ b/rpython/flowspace/flowcontext.py
@@ -465,7 +465,7 @@
             msg = "implicit %s shouldn't occur" % exc_cls.__name__
             w_type = Constant(AssertionError)
             w_value = Constant(AssertionError(msg))
-            link = Link([w_type, w_value], self.graph.exceptblock)
+            link = self.graph.new_raise_link(w_value, w_type)
             self.recorder.crnt_block.closeblock(link)
 
         except Raise as e:
@@ -473,7 +473,7 @@
             if w_exc.w_type == const(ImportError):
                 msg = 'import statement always raises %s' % e
                 raise ImportError(msg)
-            link = Link([w_exc.w_type, w_exc.w_value], self.graph.exceptblock)
+            link = self.graph.new_raise_link(w_exc.w_value, w_exc.w_type)
             self.recorder.crnt_block.closeblock(link)
 
         except StopFlowing:
diff --git a/rpython/flowspace/model.py b/rpython/flowspace/model.py
--- a/rpython/flowspace/model.py
+++ b/rpython/flowspace/model.py
@@ -31,6 +31,9 @@
     def getreturnvar(self):
         return self.returnblock.inputargs[0]
 
+    def new_raise_link(self, v_exception, v_exc_type):
+        return Link([v_exc_type, v_exception], self.exceptblock)
+
     @property
     def source(self):
         if hasattr(self, "_source"):
@@ -208,13 +211,12 @@
     def renamevariables(self, mapping):
         for a in mapping:
             assert isinstance(a, Variable), a
-        self.inputargs = [mapping.get(a, a) for a in self.inputargs]
-        for op in self.operations:
-            op.args = [mapping.get(a, a) for a in op.args]
-            op.result = mapping.get(op.result, op.result)
-        self.exitswitch = mapping.get(self.exitswitch, self.exitswitch)
+        self.inputargs = [a.replace(mapping) for a in self.inputargs]
+        self.operations = [op.replace(mapping) for op in self.operations]
+        if self.exitswitch is not None:
+            self.exitswitch = self.exitswitch.replace(mapping)
         for link in self.exits:
-            link.args = [mapping.get(a, a) for a in link.args]
+            link.args = [a.replace(mapping) for a in link.args]
 
     def closeblock(self, *exits):
         assert self.exits == [], "block already closed"
@@ -320,6 +322,8 @@
             newvar.concretetype = self.concretetype
         return newvar
 
+    def replace(self, mapping):
+        return mapping.get(self, self)
 
 
 class Constant(Hashable):
@@ -349,6 +353,9 @@
             # cannot count on it not mutating at runtime!
             return False
 
+    def replace(self, mapping):
+        return self
+
 
 class FSException(object):
     def __init__(self, w_type, w_value):
@@ -424,8 +431,8 @@
                                 ", ".join(map(repr, self.args)))
 
     def replace(self, mapping):
-        newargs = [mapping.get(arg, arg) for arg in self.args]
-        newresult = mapping.get(self.result, self.result)
+        newargs = [arg.replace(mapping) for arg in self.args]
+        newresult = self.result.replace(mapping)
         return type(self)(self.opname, newargs, newresult, self.offset)
 
 class Atom(object):
@@ -580,6 +587,8 @@
                     if isinstance(v, Variable):
                         if type(v) is Variable:
                             usevar(v)
+                        else:
+                            usevar(v.arg)
                     else:
                         assert v.value is not last_exception
                         #assert v.value != last_exc_value
diff --git a/rpython/flowspace/operation.py b/rpython/flowspace/operation.py
--- a/rpython/flowspace/operation.py
+++ b/rpython/flowspace/operation.py
@@ -76,8 +76,8 @@
         self.offset = -1
 
     def replace(self, mapping):
-        newargs = [mapping.get(arg, arg) for arg in self.args]
-        newresult = mapping.get(self.result, self.result)
+        newargs = [arg.replace(mapping) for arg in self.args]
+        newresult = self.result.replace(mapping)
         newop = type(self)(*newargs)
         newop.result = newresult
         newop.offset = self.offset

From noreply at buildbot.pypy.org  Sun Nov  9 01:27:10 2014
From: noreply at buildbot.pypy.org (rlamy)
Date: Sun,  9 Nov 2014 01:27:10 +0100 (CET)
Subject: [pypy-commit] pypy online-transforms: hg merge default
Message-ID: <20141109002710.EDF2E1D283E@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy 
Branch: online-transforms
Changeset: r74414:0ba2b288f039
Date: 2014-11-09 00:22 +0000
http://bitbucket.org/pypy/pypy/changeset/0ba2b288f039/

Log:	hg merge default

diff --git a/pypy/module/cpyext/include/modsupport.h b/pypy/module/cpyext/include/modsupport.h
--- a/pypy/module/cpyext/include/modsupport.h
+++ b/pypy/module/cpyext/include/modsupport.h
@@ -78,11 +78,20 @@
 /*
  * This is from pyport.h.  Perhaps it belongs elsewhere.
  */
+#ifdef _WIN32
+/* explicitly export since PyAPI_FUNC is usually dllimport */
+#ifdef __cplusplus
+#define PyMODINIT_FUNC extern "C" __declspec(dllexport) void
+#else
+#define PyMODINIT_FUNC __declspec(dllexport) void
+#endif
+#else
 #ifdef __cplusplus
 #define PyMODINIT_FUNC extern "C" PyAPI_FUNC(void)
 #else
 #define PyMODINIT_FUNC PyAPI_FUNC(void)
 #endif
+#endif /* WIN32 */
 
 PyAPI_DATA(char *) _Py_PackageContext;
 
diff --git a/pypy/module/rctime/interp_time.py b/pypy/module/rctime/interp_time.py
--- a/pypy/module/rctime/interp_time.py
+++ b/pypy/module/rctime/interp_time.py
@@ -178,19 +178,19 @@
 if _WIN:
     win_eci = ExternalCompilationInfo(
         includes = ["time.h"],
-        post_include_bits = ["RPY_EXPORTED_FOR_TESTS\n"
+        post_include_bits = ["RPY_EXPORTED_FOR_TESTS "
                              "long pypy_get_timezone();\n"
-                             "RPY_EXPORTED_FOR_TESTS\n"
+                             "RPY_EXPORTED_FOR_TESTS "
                              "int pypy_get_daylight();\n"
-                             "RPY_EXPORTED_FOR_TESTS\n"
+                             "RPY_EXPORTED_FOR_TESTS "
                              "char** pypy_get_tzname();\n"
-                             "RPY_EXPORTED_FOR_TESTS\n"
-                             "void* pypy__tzset();"],
+                             "RPY_EXPORTED_FOR_TESTS "
+                             "void pypy__tzset();"],
         separate_module_sources = ["""
         long pypy_get_timezone() { return timezone; }
         int pypy_get_daylight() { return daylight; }
         char** pypy_get_tzname() { return tzname; }
-        void pypy__tzset() { return _tzset(); }
+        void pypy__tzset() { _tzset(); }
         """])
     # Ensure sure that we use _tzset() and timezone from the same C Runtime.
     c_tzset = external('pypy__tzset', [], lltype.Void, win_eci)
diff --git a/pypy/module/test_lib_pypy/ctypes_tests/_ctypes_test.c b/pypy/module/test_lib_pypy/ctypes_tests/_ctypes_test.c
--- a/pypy/module/test_lib_pypy/ctypes_tests/_ctypes_test.c
+++ b/pypy/module/test_lib_pypy/ctypes_tests/_ctypes_test.c
@@ -1,10 +1,10 @@
-#include "src/precommondefs.h"
-
 #if defined(_MSC_VER) || defined(__CYGWIN__)
 #include 
 #define MS_WIN32
 #endif
 
+#include "src/precommondefs.h"
+
 #define EXPORT(x)  RPY_EXPORTED x
 
 #include 
diff --git a/rpython/jit/metainterp/optimizeopt/rewrite.py b/rpython/jit/metainterp/optimizeopt/rewrite.py
--- a/rpython/jit/metainterp/optimizeopt/rewrite.py
+++ b/rpython/jit/metainterp/optimizeopt/rewrite.py
@@ -1,7 +1,8 @@
 from rpython.jit.codewriter.effectinfo import EffectInfo
+from rpython.jit.codewriter import longlong
 from rpython.jit.metainterp import compile
 from rpython.jit.metainterp.history import (Const, ConstInt, BoxInt, BoxFloat,
-    BoxPtr, make_hashable_int)
+    BoxPtr, make_hashable_int, ConstFloat)
 from rpython.jit.metainterp.optimize import InvalidLoop
 from rpython.jit.metainterp.optimizeopt.intutils import IntBound
 from rpython.jit.metainterp.optimizeopt.optimizer import (Optimization, REMOVED,
@@ -10,7 +11,7 @@
 from rpython.jit.metainterp.resoperation import (opboolinvers, opboolreflex, rop,
     ResOperation)
 from rpython.rlib.rarithmetic import highest_bit
-
+import math
 
 class OptRewrite(Optimization):
     """Rewrite operations into equivalent, cheaper operations.
@@ -231,6 +232,25 @@
         self.emit_operation(op)
         self.pure(rop.FLOAT_MUL, [arg2, arg1], op.result)
 
+    def optimize_FLOAT_TRUEDIV(self, op):
+        arg1 = op.getarg(0)
+        arg2 = op.getarg(1)
+        v2 = self.getvalue(arg2)
+
+        # replace "x / const" by "x * (1/const)" if possible
+        if v2.is_constant():
+            divisor = v2.box.getfloat()
+            fraction = math.frexp(divisor)[0]
+            # This optimization is valid for powers of two
+            # but not for zeroes, some denormals and NaN:
+            if fraction == 0.5 or fraction == -0.5:
+                reciprocal = 1.0 / divisor
+                rfraction = math.frexp(reciprocal)[0]
+                if rfraction == 0.5 or rfraction == -0.5:
+                    c = ConstFloat(longlong.getfloatstorage(reciprocal))
+                    op = op.copy_and_change(rop.FLOAT_MUL, args=[arg1, c])
+        self.emit_operation(op)
+
     def optimize_FLOAT_NEG(self, op):
         v1 = op.getarg(0)
         self.emit_operation(op)
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py
@@ -2364,6 +2364,28 @@
         """
         self.optimize_loop(ops, expected)
 
+    def test_float_division_by_multiplication(self):
+        ops = """
+        [f0]
+        f1 = float_truediv(f0, 2.0)
+        f2 = float_truediv(f1, 3.0)
+        f3 = float_truediv(f2, -0.25)
+        f4 = float_truediv(f3, 0.0)
+        f5 = escape(f4)
+        jump(f5)
+        """
+
+        expected = """
+        [f0]
+        f1 = float_mul(f0, 0.5)
+        f2 = float_truediv(f1, 3.0)
+        f3 = float_mul(f2, -4.0)
+        f4 = float_truediv(f3, 0.0)
+        f5 = escape(f4)
+        jump(f5)
+        """
+        self.optimize_loop(ops, expected)
+
     # ----------
 
     def _verify_fail_args(self, boxes, oparse, text):
diff --git a/rpython/jit/metainterp/pyjitpl.py b/rpython/jit/metainterp/pyjitpl.py
--- a/rpython/jit/metainterp/pyjitpl.py
+++ b/rpython/jit/metainterp/pyjitpl.py
@@ -1333,8 +1333,6 @@
             while True:
                 pc = self.pc
                 op = ord(self.bytecode[pc])
-                #debug_print(self.jitcode.name, pc)
-                #print staticdata.opcode_names[op]
                 staticdata.opcode_implementations[op](self, pc)
         except ChangeFrame:
             pass
diff --git a/rpython/jit/metainterp/test/test_rawmem.py b/rpython/jit/metainterp/test/test_rawmem.py
--- a/rpython/jit/metainterp/test/test_rawmem.py
+++ b/rpython/jit/metainterp/test/test_rawmem.py
@@ -89,6 +89,16 @@
                                        'finish': 1})
         self.metainterp.staticdata.stats.check_resops({'finish': 1}, omit_finish=False)
 
+    def test_scoped_alloc_buffer(self):
+        def f():
+            with rffi.scoped_alloc_buffer(42) as p:
+                p.raw[0] = 'X'
+                s = p.str(1)
+            return ord(s[0])
+
+        res = self.interp_operations(f, [])
+        assert res == ord('X')
+
 
 class TestRawMem(RawMemTests, LLJitMixin):
 
diff --git a/rpython/rlib/_rsocket_rffi.py b/rpython/rlib/_rsocket_rffi.py
--- a/rpython/rlib/_rsocket_rffi.py
+++ b/rpython/rlib/_rsocket_rffi.py
@@ -499,7 +499,7 @@
 getnameinfo = external('getnameinfo', [sockaddr_ptr, socklen_t, CCHARP,
                        size_t, CCHARP, size_t, rffi.INT], rffi.INT)
 
-if sys.platform.startswith("openbsd"):
+if sys.platform.startswith("openbsd") or sys.platform.startswith("darwin"):
     htonl = external('htonl', [rffi.UINT], rffi.UINT, releasegil=False, macro=True)
     htons = external('htons', [rffi.USHORT], rffi.USHORT, releasegil=False, macro=True)
     ntohl = external('ntohl', [rffi.UINT], rffi.UINT, releasegil=False, macro=True)
diff --git a/rpython/rlib/rfile.py b/rpython/rlib/rfile.py
--- a/rpython/rlib/rfile.py
+++ b/rpython/rlib/rfile.py
@@ -96,9 +96,12 @@
 c_ferror = llexternal('ferror', [FILEP], rffi.INT)
 c_clearerr = llexternal('clearerr', [FILEP], lltype.Void)
 
-c_stdin = rffi.CExternVariable(FILEP, 'stdin', eci, c_type='FILE*')[0]
-c_stdout = rffi.CExternVariable(FILEP, 'stdout', eci, c_type='FILE*')[0]
-c_stderr = rffi.CExternVariable(FILEP, 'stderr', eci, c_type='FILE*')[0]
+c_stdin = rffi.CExternVariable(FILEP, 'stdin', eci, c_type='FILE*',
+                               getter_only=True)
+c_stdout = rffi.CExternVariable(FILEP, 'stdout', eci, c_type='FILE*',
+                                getter_only=True)
+c_stderr = rffi.CExternVariable(FILEP, 'stderr', eci, c_type='FILE*',
+                                getter_only=True)
 
 
 def _error(ll_file):
diff --git a/rpython/rtyper/lltypesystem/rffi.py b/rpython/rtyper/lltypesystem/rffi.py
--- a/rpython/rtyper/lltypesystem/rffi.py
+++ b/rpython/rtyper/lltypesystem/rffi.py
@@ -605,7 +605,7 @@
 
 def CExternVariable(TYPE, name, eci, _CConstantClass=CConstant,
                     sandboxsafe=False, _nowrapper=False,
-                    c_type=None):
+                    c_type=None, getter_only=False):
     """Return a pair of functions - a getter and a setter - to access
     the given global C variable.
     """
@@ -638,19 +638,26 @@
     if sys.platform != 'win32':
         lines.append('extern %s %s;' % (c_type, name))
     lines.append(c_getter)
-    lines.append(c_setter)
+    if not getter_only:
+        lines.append(c_setter)
+    prototypes = [getter_prototype]
+    if not getter_only:
+        prototypes.append(setter_prototype)
     sources = ('\n'.join(lines),)
     new_eci = eci.merge(ExternalCompilationInfo(
         separate_module_sources = sources,
-        post_include_bits = [getter_prototype, setter_prototype],
+        post_include_bits = prototypes,
     ))
 
     getter = llexternal(getter_name, [], TYPE, compilation_info=new_eci,
                         sandboxsafe=sandboxsafe, _nowrapper=_nowrapper)
-    setter = llexternal(setter_name, [TYPE], lltype.Void,
-                        compilation_info=new_eci, sandboxsafe=sandboxsafe,
-                        _nowrapper=_nowrapper)
-    return getter, setter
+    if getter_only:
+        return getter
+    else:
+        setter = llexternal(setter_name, [TYPE], lltype.Void,
+                            compilation_info=new_eci, sandboxsafe=sandboxsafe,
+                            _nowrapper=_nowrapper)
+        return getter, setter
 
 # char, represented as a Python character
 # (use SIGNEDCHAR or UCHAR for the small integer types)
@@ -815,6 +822,8 @@
     free_nonmovingbuffer._annenforceargs_ = [strtype, None, bool, bool]
 
     # int -> (char*, str, int)
+    # Can't inline this because of the raw address manipulation.
+    @jit.dont_look_inside
     def alloc_buffer(count):
         """
         Returns a (raw_buffer, gc_buffer, case_num) triple,
diff --git a/rpython/rtyper/rptr.py b/rpython/rtyper/rptr.py
--- a/rpython/rtyper/rptr.py
+++ b/rpython/rtyper/rptr.py
@@ -115,9 +115,9 @@
 
 class __extend__(pairtype(PtrRepr, PtrRepr)):
     def convert_from_to((r_ptr1, r_ptr2), v, llop):
-        assert r_ptr1.lowleveltype == r_ptr2.lowleveltype
-        return v
-
+        if r_ptr1.lowleveltype == r_ptr2.lowleveltype:
+            return v
+        return NotImplemented
 
 class __extend__(pairtype(PtrRepr, IntegerRepr)):
 
diff --git a/rpython/translator/c/src/libffi_msvc/ffi.h b/rpython/translator/c/src/libffi_msvc/ffi.h
--- a/rpython/translator/c/src/libffi_msvc/ffi.h
+++ b/rpython/translator/c/src/libffi_msvc/ffi.h
@@ -60,6 +60,7 @@
 
 /* ---- System configuration information --------------------------------- */
 
+#include  /* for RPY_EXPORTED_FOR_TESTS */
 #include 
 
 #ifndef LIBFFI_ASM
@@ -221,6 +222,7 @@
   void      *user_data;
 } ffi_closure;
 
+RPY_EXPORTED_FOR_TESTS
 ffi_status
 ffi_prep_closure (ffi_closure*,
 		  ffi_cif *,
@@ -264,12 +266,14 @@
 
 /* ---- Public interface definition -------------------------------------- */
 
+RPY_EXPORTED_FOR_TESTS
 ffi_status ffi_prep_cif(/*@out@*/ /*@partial@*/ ffi_cif *cif, 
 			ffi_abi abi,
 			unsigned int nargs, 
 			/*@dependent@*/ /*@out@*/ /*@partial@*/ ffi_type *rtype, 
 			/*@dependent@*/ ffi_type **atypes);
 
+RPY_EXPORTED_FOR_TESTS
 int
 ffi_call(/*@dependent@*/ ffi_cif *cif, 
 	 void (*fn)(), 
diff --git a/rpython/translator/c/src/profiling.c b/rpython/translator/c/src/profiling.c
--- a/rpython/translator/c/src/profiling.c
+++ b/rpython/translator/c/src/profiling.c
@@ -10,7 +10,7 @@
 static int profiling_setup = 0;
 
 RPY_EXPORTED_FOR_TESTS
-void pypy_setup_profiling()
+void pypy_setup_profiling(void)
 {
   if (!profiling_setup) {
     cpu_set_t set;
@@ -23,7 +23,7 @@
 }
 
 RPY_EXPORTED_FOR_TESTS
-void pypy_teardown_profiling()
+void pypy_teardown_profiling(void)
 {
   if (profiling_setup) {
     sched_setaffinity(0, sizeof(cpu_set_t), &base_cpu_set);
@@ -40,7 +40,8 @@
 static DWORD_PTR base_affinity_mask;
 static int profiling_setup = 0;
 
-void pypy_setup_profiling() { 
+RPY_EXPORTED_FOR_TESTS
+void pypy_setup_profiling(void) {
     if (!profiling_setup) {
         DWORD_PTR affinity_mask, system_affinity_mask;
         GetProcessAffinityMask(GetCurrentProcess(),
@@ -55,7 +56,8 @@
     }
 }
 
-void pypy_teardown_profiling() {
+RPY_EXPORTED_FOR_TESTS
+void pypy_teardown_profiling(void) {
     if (profiling_setup) {
         SetProcessAffinityMask(GetCurrentProcess(), base_affinity_mask);
         profiling_setup = 0;
@@ -65,7 +67,7 @@
 #else
 
 /* Empty implementations for other platforms */
-void pypy_setup_profiling() { }
-void pypy_teardown_profiling() { }
+RPY_EXPORTED_FOR_TESTS void pypy_setup_profiling(void) { }
+RPY_EXPORTED_FOR_TESTS void pypy_teardown_profiling(void) { }
 
 #endif
diff --git a/rpython/translator/c/src/profiling.h b/rpython/translator/c/src/profiling.h
--- a/rpython/translator/c/src/profiling.h
+++ b/rpython/translator/c/src/profiling.h
@@ -1,7 +1,7 @@
 #ifndef _PYPY_PROFILING_H
 #define _PYPY_PROFILING_H
 
-void pypy_setup_profiling();
-void pypy_teardown_profiling();
+RPY_EXPORTED_FOR_TESTS void pypy_setup_profiling(void);
+RPY_EXPORTED_FOR_TESTS void pypy_teardown_profiling(void);
 
 #endif
diff --git a/rpython/translator/simplify.py b/rpython/translator/simplify.py
--- a/rpython/translator/simplify.py
+++ b/rpython/translator/simplify.py
@@ -5,6 +5,7 @@
 simplify_graph() applies all simplifications defined in this file.
 """
 import py
+from collections import defaultdict
 
 from rpython.flowspace.model import (Variable, Constant,
                                      c_last_exception, checkgraph, mkentrymap)
@@ -401,8 +402,8 @@
 def transform_dead_op_vars_in_blocks(blocks, graphs, translator=None):
     """Remove dead operations and variables that are passed over a link
     but not used in the target block. Input is a set of blocks"""
-    read_vars = {}  # set of variables really used
-    variable_flow = {}  # map {Var: list-of-Vars-it-depends-on}
+    read_vars = set()  # set of variables really used
+    dependencies = defaultdict(set) # map {Var: list-of-Vars-it-depends-on}
     set_of_blocks = set(blocks)
     start_blocks = find_start_blocks(graphs)
 
@@ -414,53 +415,48 @@
         # cannot remove the exc-raising operation
         return op is not block.operations[-1]
 
-    # compute variable_flow and an initial read_vars
+    # compute dependencies and an initial read_vars
     for block in blocks:
         # figure out which variables are ever read
         for op in block.operations:
-            if not canremove(op, block):   # mark the inputs as really needed
-                for arg in op.args:
-                    read_vars[arg] = True
+            if not canremove(op, block):   # the inputs are always needed
+                read_vars.update(op.args)
             else:
-                # if CanRemove, only mark dependencies of the result
-                # on the input variables
-                deps = variable_flow.setdefault(op.result, [])
-                deps.extend(op.args)
+                dependencies[op.result].update(op.args)
 
         if isinstance(block.exitswitch, Variable):
-            read_vars[block.exitswitch] = True
+            read_vars.add(block.exitswitch)
 
         if block.exits:
             for link in block.exits:
                 if link.target not in set_of_blocks:
                     for arg, targetarg in zip(link.args, link.target.inputargs):
-                        read_vars[arg] = True
-                        read_vars[targetarg] = True
+                        read_vars.add(arg)
+                        read_vars.add(targetarg)
                 else:
                     for arg, targetarg in zip(link.args, link.target.inputargs):
-                        deps = variable_flow.setdefault(targetarg, [])
-                        deps.append(arg)
+                        dependencies[targetarg].add(arg)
         else:
             # return and except blocks implicitely use their input variable(s)
             for arg in block.inputargs:
-                read_vars[arg] = True
-        # an input block's inputargs should not be modified, even if some
+                read_vars.add(arg)
+        # a start block's inputargs should not be modified, even if some
         # of the function's input arguments are not actually used
         if block in start_blocks:
             for arg in block.inputargs:
-                read_vars[arg] = True
+                read_vars.add(arg)
 
     # flow read_vars backwards so that any variable on which a read_vars
     # depends is also included in read_vars
     def flow_read_var_backward(pending):
-        pending = list(pending)
-        for var in pending:
-            for prevvar in variable_flow.get(var, []):
+        while pending:
+            var = pending.pop()
+            for prevvar in dependencies[var]:
                 if prevvar not in read_vars:
-                    read_vars[prevvar] = True
-                    pending.append(prevvar)
+                    read_vars.add(prevvar)
+                    pending.add(prevvar)
 
-    flow_read_var_backward(read_vars)
+    flow_read_var_backward(set(read_vars))
 
     for block in blocks:
 

From noreply at buildbot.pypy.org  Sun Nov  9 14:51:04 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Sun,  9 Nov 2014 14:51:04 +0100 (CET)
Subject: [pypy-commit] stmgc hashtable: Remove a dead file
Message-ID: <20141109135104.962761C33C0@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: hashtable
Changeset: r1501:7ff130375645
Date: 2014-11-09 14:48 +0100
http://bitbucket.org/pypy/stmgc/changeset/7ff130375645/

Log:	Remove a dead file

diff --git a/c7/stm/hashtable.h b/c7/stm/hashtable.h
deleted file mode 100644
--- a/c7/stm/hashtable.h
+++ /dev/null
@@ -1,4 +0,0 @@
-
-#if 0
-static void stm_compact_hashtable(stm_hashtable_t *hashtable);
-#endif
diff --git a/c7/stmgc.c b/c7/stmgc.c
--- a/c7/stmgc.c
+++ b/c7/stmgc.c
@@ -16,7 +16,6 @@
 #include "stm/weakref.h"
 #include "stm/marker.h"
 #include "stm/finalizer.h"
-#include "stm/hashtable.h"
 
 #include "stm/misc.c"
 #include "stm/list.c"

From noreply at buildbot.pypy.org  Sun Nov  9 17:47:21 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Sun,  9 Nov 2014 17:47:21 +0100 (CET)
Subject: [pypy-commit] pypy default: We were missing the
 __attribute__((visibility("hidden"))) on all
Message-ID: <20141109164721.411EC1C3526@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r74415:56ba97730a23
Date: 2014-11-09 11:27 +0100
http://bitbucket.org/pypy/pypy/changeset/56ba97730a23/

Log:	We were missing the __attribute__((visibility("hidden"))) on all
	function and global variables declared in translator/c/src/. This
	should in theory close the performance gap between shared and non-
	shared pypy.

	See comments in translator/c/src/precommondefs.h.

diff --git a/rpython/rlib/clibffi.py b/rpython/rlib/clibffi.py
--- a/rpython/rlib/clibffi.py
+++ b/rpython/rlib/clibffi.py
@@ -49,7 +49,7 @@
     #include 
 
     /* Get the module where the "fopen" function resides in */
-    RPY_EXPORTED_FOR_TESTS
+    RPY_EXTERN
     HANDLE pypy_get_libc_handle() {
         MEMORY_BASIC_INFORMATION  mi;
         char buf[1000];
diff --git a/rpython/rlib/rposix.py b/rpython/rlib/rposix.py
--- a/rpython/rlib/rposix.py
+++ b/rpython/rlib/rposix.py
@@ -44,7 +44,7 @@
 
         /* This function emulates what the windows CRT
             does to validate file handles */
-        RPY_EXPORTED_FOR_TESTS int
+        RPY_EXTERN int
         _PyVerify_fd(int fd)
         {
             const int i1 = fd >> IOINFO_L2E;
diff --git a/rpython/rtyper/lltypesystem/rffi.py b/rpython/rtyper/lltypesystem/rffi.py
--- a/rpython/rtyper/lltypesystem/rffi.py
+++ b/rpython/rtyper/lltypesystem/rffi.py
@@ -383,7 +383,7 @@
     else:
         pattern = '%s%s { return %s(%s); }'
     source = pattern % (
-        'RPY_EXPORTED_FOR_TESTS ',
+        'RPY_EXTERN ',
         cdecl(implementationtypename, wrapper_name),
         macro, ', '.join(argnames))
 
@@ -628,9 +628,9 @@
     getter_name = 'get_' + name
     setter_name = 'set_' + name
     getter_prototype = (
-       "RPY_EXPORTED_FOR_TESTS %(c_type)s %(getter_name)s ();" % locals())
+       "RPY_EXTERN %(c_type)s %(getter_name)s ();" % locals())
     setter_prototype = (
-       "RPY_EXPORTED_FOR_TESTS void %(setter_name)s (%(c_type)s v);" % locals())
+       "RPY_EXTERN void %(setter_name)s (%(c_type)s v);" % locals())
     c_getter = "%(c_type)s %(getter_name)s () { return %(name)s; }" % locals()
     c_setter = "void %(setter_name)s (%(c_type)s v) { %(name)s = v; }" % locals()
 
diff --git a/rpython/rtyper/lltypesystem/test/test_rffi.py b/rpython/rtyper/lltypesystem/test/test_rffi.py
--- a/rpython/rtyper/lltypesystem/test/test_rffi.py
+++ b/rpython/rtyper/lltypesystem/test/test_rffi.py
@@ -391,7 +391,7 @@
         h_source = py.code.Source("""
         #ifndef _CALLBACK_H
         #define _CALLBACK_H
-        extern Signed eating_callback(Signed arg, Signed(*call)(Signed));
+        RPY_EXTERN Signed eating_callback(Signed arg, Signed(*call)(Signed));
         #endif /* _CALLBACK_H */
         """)
 
@@ -401,7 +401,7 @@
         c_source = py.code.Source("""
         #include "src/precommondefs.h"
 
-        RPY_EXPORTED Signed eating_callback(Signed arg, Signed(*call)(Signed))
+        RPY_EXTERN Signed eating_callback(Signed arg, Signed(*call)(Signed))
         {
             Signed res = call(arg);
             if (res == -1)
diff --git a/rpython/translator/c/src/allocator.h b/rpython/translator/c/src/allocator.h
--- a/rpython/translator/c/src/allocator.h
+++ b/rpython/translator/c/src/allocator.h
@@ -1,4 +1,4 @@
 /* allocation functions prototypes */
-void *PyObject_Malloc(size_t n);
-void *PyObject_Realloc(void *p, size_t n);
-void PyObject_Free(void *p);
+RPY_EXTERN void *PyObject_Malloc(size_t n);
+RPY_EXTERN void *PyObject_Realloc(void *p, size_t n);
+RPY_EXTERN void PyObject_Free(void *p);
diff --git a/rpython/translator/c/src/asm_gcc_x86.h b/rpython/translator/c/src/asm_gcc_x86.h
--- a/rpython/translator/c/src/asm_gcc_x86.h
+++ b/rpython/translator/c/src/asm_gcc_x86.h
@@ -52,7 +52,7 @@
         : "0"(x), "g"(y)     /* inputs  */      \
         : "cc", "memory")    /* clobber */
 
-extern void op_int_overflowed(void)
+RPY_EXTERN void op_int_overflowed(void)
      asm ("_op_int_overflowed")
      __attribute__((used));
 
@@ -104,5 +104,5 @@
 
 #ifdef PYPY_X86_CHECK_SSE2
 #define PYPY_X86_CHECK_SSE2_DEFINED
-extern void pypy_x86_check_sse2(void);
+RPY_EXTERN void pypy_x86_check_sse2(void);
 #endif
diff --git a/rpython/translator/c/src/asm_msvc.h b/rpython/translator/c/src/asm_msvc.h
--- a/rpython/translator/c/src/asm_msvc.h
+++ b/rpython/translator/c/src/asm_msvc.h
@@ -1,4 +1,4 @@
 #ifdef PYPY_X86_CHECK_SSE2
 #define PYPY_X86_CHECK_SSE2_DEFINED
-extern void pypy_x86_check_sse2(void);
+RPY_EXTERN void pypy_x86_check_sse2(void);
 #endif
diff --git a/rpython/translator/c/src/debug_print.h b/rpython/translator/c/src/debug_print.h
--- a/rpython/translator/c/src/debug_print.h
+++ b/rpython/translator/c/src/debug_print.h
@@ -37,14 +37,14 @@
 /************************************************************/
 
 /* prototypes (internal use only) */
-void pypy_debug_ensure_opened(void);
-void pypy_debug_start(const char *category);
-void pypy_debug_stop(const char *category);
-long pypy_debug_offset(void);
-void pypy_debug_forked(long original_offset);
+RPY_EXTERN void pypy_debug_ensure_opened(void);
+RPY_EXTERN void pypy_debug_start(const char *category);
+RPY_EXTERN void pypy_debug_stop(const char *category);
+RPY_EXTERN long pypy_debug_offset(void);
+RPY_EXTERN void pypy_debug_forked(long original_offset);
 
-extern long pypy_have_debug_prints;
-extern RPY_EXPORTED FILE *pypy_debug_file;
+RPY_EXTERN long pypy_have_debug_prints;
+RPY_EXPORTED FILE *pypy_debug_file;
 
 #define OP_LL_READ_TIMESTAMP(val) READ_TIMESTAMP(val)
 
@@ -59,7 +59,7 @@
 #    define READ_TIMESTAMP(val) QueryPerformanceCounter((LARGE_INTEGER*)&(val))
 #  else
 
-long long pypy_read_timestamp();
+RPY_EXTERN long long pypy_read_timestamp(void);
 
 #    define READ_TIMESTAMP(val)  (val) = pypy_read_timestamp()
 
diff --git a/rpython/translator/c/src/debug_traceback.h b/rpython/translator/c/src/debug_traceback.h
--- a/rpython/translator/c/src/debug_traceback.h
+++ b/rpython/translator/c/src/debug_traceback.h
@@ -60,8 +60,8 @@
   void *exctype;
 };
 
-extern int pypydtcount;
-extern struct pypydtentry_s pypy_debug_tracebacks[PYPY_DEBUG_TRACEBACK_DEPTH];
+RPY_EXTERN int pypydtcount;
+RPY_EXTERN struct pypydtentry_s pypy_debug_tracebacks[PYPY_DEBUG_TRACEBACK_DEPTH];
 
-void pypy_debug_traceback_print(void);
-void pypy_debug_catch_fatal_exception(void);
+RPY_EXTERN void pypy_debug_traceback_print(void);
+RPY_EXTERN void pypy_debug_catch_fatal_exception(void);
diff --git a/rpython/translator/c/src/dtoa.h b/rpython/translator/c/src/dtoa.h
--- a/rpython/translator/c/src/dtoa.h
+++ b/rpython/translator/c/src/dtoa.h
@@ -1,11 +1,11 @@
 /* Exported functions from dtoa.c */
 
-RPY_EXPORTED_FOR_TESTS
+RPY_EXTERN
 double _PyPy_dg_strtod(const char *str, char **ptr);
 
-RPY_EXPORTED_FOR_TESTS
+RPY_EXTERN
 char * _PyPy_dg_dtoa(double d, int mode, int ndigits,
 		     int *decpt, int *sign, char **rve);
 
-RPY_EXPORTED_FOR_TESTS
+RPY_EXTERN
 void _PyPy_dg_freedtoa(char *s);
diff --git a/rpython/translator/c/src/entrypoint.h b/rpython/translator/c/src/entrypoint.h
--- a/rpython/translator/c/src/entrypoint.h
+++ b/rpython/translator/c/src/entrypoint.h
@@ -8,6 +8,6 @@
 #define PYPY_MAIN_FUNCTION main
 #endif
 
-char *RPython_StartupCode(void);
+RPY_EXTERN char *RPython_StartupCode(void);
 RPY_EXPORTED int PYPY_MAIN_FUNCTION(int argc, char *argv[]);
 #endif  /* PYPY_STANDALONE */
diff --git a/rpython/translator/c/src/exception.h b/rpython/translator/c/src/exception.h
--- a/rpython/translator/c/src/exception.h
+++ b/rpython/translator/c/src/exception.h
@@ -22,6 +22,7 @@
 #endif
 /* !DO_LOG_EXC: define the function anyway, so that we can shut
    off the prints of a debug_exc by remaking only testing_1.o */
+RPY_EXTERN
 void RPyDebugReturnShowException(const char *msg, const char *filename,
                                  long lineno, const char *functionname);
 
@@ -36,6 +37,7 @@
 
 /* prototypes */
 
+RPY_EXTERN
 void _RPyRaiseSimpleException(RPYTHON_EXCEPTION rexc);
 
 #endif
diff --git a/rpython/translator/c/src/instrument.c b/rpython/translator/c/src/instrument.c
--- a/rpython/translator/c/src/instrument.c
+++ b/rpython/translator/c/src/instrument.c
@@ -1,5 +1,5 @@
+#include "common_header.h"
 #include 
-#include "common_header.h"
 
 #ifdef  PYPY_INSTRUMENT
 
diff --git a/rpython/translator/c/src/instrument.h b/rpython/translator/c/src/instrument.h
--- a/rpython/translator/c/src/instrument.h
+++ b/rpython/translator/c/src/instrument.h
@@ -1,10 +1,10 @@
 #ifndef _PYPY_INSTRUMENT_H
 #define _PYPY_INSTRUMENT_H
 
-void instrument_setup();
+RPY_EXTERN void instrument_setup();
 
 #ifdef PYPY_INSTRUMENT
-void instrument_count(long);
+RPY_EXTERN void instrument_count(long);
 #define PYPY_INSTRUMENT_COUNT(label) instrument_count(label)
 #else
 #define PYPY_INSTRUMENT_COUNT
diff --git a/rpython/translator/c/src/int.h b/rpython/translator/c/src/int.h
--- a/rpython/translator/c/src/int.h
+++ b/rpython/translator/c/src/int.h
@@ -238,7 +238,7 @@
 
 #define OP_BOOL_NOT(x, r) r = !(x)
 
-long long op_llong_mul_ovf(long long a, long long b);
+RPY_EXTERN long long op_llong_mul_ovf(long long a, long long b);
 
 /* The definitions above can be used with various types */ 
 
diff --git a/rpython/translator/c/src/libffi_msvc/ffi.h b/rpython/translator/c/src/libffi_msvc/ffi.h
--- a/rpython/translator/c/src/libffi_msvc/ffi.h
+++ b/rpython/translator/c/src/libffi_msvc/ffi.h
@@ -60,7 +60,7 @@
 
 /* ---- System configuration information --------------------------------- */
 
-#include  /* for RPY_EXPORTED_FOR_TESTS */
+#include  /* for RPY_EXTERN */
 #include 
 
 #ifndef LIBFFI_ASM
@@ -222,7 +222,7 @@
   void      *user_data;
 } ffi_closure;
 
-RPY_EXPORTED_FOR_TESTS
+RPY_EXTERN
 ffi_status
 ffi_prep_closure (ffi_closure*,
 		  ffi_cif *,
@@ -266,14 +266,14 @@
 
 /* ---- Public interface definition -------------------------------------- */
 
-RPY_EXPORTED_FOR_TESTS
+RPY_EXTERN
 ffi_status ffi_prep_cif(/*@out@*/ /*@partial@*/ ffi_cif *cif, 
 			ffi_abi abi,
 			unsigned int nargs, 
 			/*@dependent@*/ /*@out@*/ /*@partial@*/ ffi_type *rtype, 
 			/*@dependent@*/ ffi_type **atypes);
 
-RPY_EXPORTED_FOR_TESTS
+RPY_EXTERN
 int
 ffi_call(/*@dependent@*/ ffi_cif *cif, 
 	 void (*fn)(), 
diff --git a/rpython/translator/c/src/ll_math.c b/rpython/translator/c/src/ll_math.c
--- a/rpython/translator/c/src/ll_math.c
+++ b/rpython/translator/c/src/ll_math.c
@@ -36,7 +36,7 @@
  * ====================================================
  */
 
-double _pypy_math_log1p(double x);
+RPY_EXTERN double _pypy_math_log1p(double x);
 
 static const double ln2 = 6.93147180559945286227E-01;
 static const double two_pow_m28 = 3.7252902984619141E-09; /* 2**-28 */
diff --git a/rpython/translator/c/src/ll_math.h b/rpython/translator/c/src/ll_math.h
--- a/rpython/translator/c/src/ll_math.h
+++ b/rpython/translator/c/src/ll_math.h
@@ -3,9 +3,9 @@
 
 #include "src/precommondefs.h"
 
-RPY_EXPORTED_FOR_TESTS double _pypy_math_acosh(double x);
-RPY_EXPORTED_FOR_TESTS double _pypy_math_asinh(double x);
-RPY_EXPORTED_FOR_TESTS double _pypy_math_atanh(double x);
+RPY_EXTERN double _pypy_math_acosh(double x);
+RPY_EXTERN double _pypy_math_asinh(double x);
+RPY_EXTERN double _pypy_math_atanh(double x);
 
-RPY_EXPORTED_FOR_TESTS double _pypy_math_expm1(double x);
-RPY_EXPORTED_FOR_TESTS double _pypy_math_log1p(double x);
+RPY_EXTERN double _pypy_math_expm1(double x);
+RPY_EXTERN double _pypy_math_log1p(double x);
diff --git a/rpython/translator/c/src/ll_strtod.h b/rpython/translator/c/src/ll_strtod.h
--- a/rpython/translator/c/src/ll_strtod.h
+++ b/rpython/translator/c/src/ll_strtod.h
@@ -6,8 +6,10 @@
 #ifndef _PYPY_LL_STRTOD_H
 #define _PYPY_LL_STRTOD_H
 
+RPY_EXTERN
 double LL_strtod_parts_to_float(char *sign, char *beforept,
 				char *afterpt, char *exponent);
+RPY_EXTERN
 char *LL_strtod_formatd(double x, char code, int precision);
 
 #endif
diff --git a/rpython/translator/c/src/mem.h b/rpython/translator/c/src/mem.h
--- a/rpython/translator/c/src/mem.h
+++ b/rpython/translator/c/src/mem.h
@@ -70,9 +70,9 @@
                                                                 __FUNCTION__)
 #  define OP_TRACK_ALLOC_STOP(addr, r)   pypy_debug_alloc_stop(addr)
 
-void pypy_debug_alloc_start(void*, const char*);
-void pypy_debug_alloc_stop(void*);
-void pypy_debug_alloc_results(void);
+RPY_EXTERN void pypy_debug_alloc_start(void*, const char*);
+RPY_EXTERN void pypy_debug_alloc_stop(void*);
+RPY_EXTERN void pypy_debug_alloc_results(void);
 
 #endif /* RPY_ASSERT */
 
@@ -101,9 +101,9 @@
     else								\
 	GC_GENERAL_REGISTER_DISAPPEARING_LINK(link, obj)
 
-extern int boehm_gc_finalizer_lock;
-void boehm_gc_startup_code(void);
-void boehm_gc_finalizer_notifier(void);
+RPY_EXTERN int boehm_gc_finalizer_lock;
+RPY_EXTERN void boehm_gc_startup_code(void);
+RPY_EXTERN void boehm_gc_finalizer_notifier(void);
 
 #define OP_GC__DISABLE_FINALIZERS(r)  boehm_gc_finalizer_lock++
 #define OP_GC__ENABLE_FINALIZERS(r)  (boehm_gc_finalizer_lock--,	\
@@ -146,10 +146,10 @@
 
 #ifndef _MSC_VER
 /* Implementation for Linux */
-extern char __gcmapstart;
-extern char __gcmapend;
-extern char __gccallshapes;
-extern long pypy_asm_stackwalk(void*, void*);
+RPY_EXTERN char __gcmapstart;
+RPY_EXTERN char __gcmapend;
+RPY_EXTERN char __gccallshapes;
+RPY_EXTERN long pypy_asm_stackwalk(void*, void*);
 #define __gcnoreorderhack __gcmapend
 
 /* The following pseudo-instruction is used by --gcrootfinder=asmgcc
@@ -186,7 +186,7 @@
 #define pypy_asm_stack_bottom() { asm volatile ("/* GC_STACK_BOTTOM */" : : : \
                                   "memory"); pypy_check_stack_count(); }
 #ifdef RPY_ASSERT
-void pypy_check_stack_count(void);
+RPY_EXTERN void pypy_check_stack_count(void);
 #else
 static void pypy_check_stack_count(void) { }
 #endif
@@ -200,10 +200,10 @@
 
 #else
 /* implementation of asmgcroot for Windows */
-extern void* __gcmapstart;
-extern void* __gcmapend;
-extern char* __gccallshapes;
-extern Signed pypy_asm_stackwalk(void*, void*);
+RPY_EXTERN void* __gcmapstart;
+RPY_EXTERN void* __gcmapend;
+RPY_EXTERN char* __gccallshapes;
+RPY_EXTERN Signed pypy_asm_stackwalk(void*, void*);
 
 /* With the msvc Microsoft Compiler, the optimizer seems free to move
    any code (even asm) that involves local memory (registers and stack).
diff --git a/rpython/translator/c/src/precommondefs.h b/rpython/translator/c/src/precommondefs.h
--- a/rpython/translator/c/src/precommondefs.h
+++ b/rpython/translator/c/src/precommondefs.h
@@ -42,15 +42,33 @@
 #endif
 
 
+/* All functions and global variables declared anywhere should use
+   one of the following attributes:
+
+   RPY_EXPORTED:  the symbol is exported out of libpypy-c.so.
+
+   RPY_EXTERN:    the symbol is not exported out of libpypy-c.so, but
+                  otherwise works like 'extern' by being available to
+                  other C sources.
+
+   static:        as usual, this means the symbol is local to this C file.
+
+   Don't use _RPY_HIDDEN directly.  For tests involving building a custom
+   .so, translator/tool/cbuild.py overrides RPY_EXTERN so that it becomes
+   equal to RPY_EXPORTED.
+
+   Any function or global variable declared with no attribute at all is
+   a bug; please report or fix it.
+*/
 #ifdef __GNUC__
-#  define RPY_EXPORTED __attribute__((visibility("default")))
-#  define RPY_HIDDEN   __attribute__((visibility("hidden")))
+#  define RPY_EXPORTED extern __attribute__((visibility("default")))
+#  define _RPY_HIDDEN  __attribute__((visibility("hidden")))
 #else
-#  define RPY_EXPORTED __declspec(dllexport)
-#  define RPY_HIDDEN   /* nothing */
+#  define RPY_EXPORTED extern __declspec(dllexport)
+#  define _RPY_HIDDEN  /* nothing */
 #endif
-#ifndef RPY_EXPORTED_FOR_TESTS
-#  define RPY_EXPORTED_FOR_TESTS  /* nothing */
+#ifndef RPY_EXTERN
+#  define RPY_EXTERN   extern _RPY_HIDDEN
 #endif
 
 
diff --git a/rpython/translator/c/src/profiling.c b/rpython/translator/c/src/profiling.c
--- a/rpython/translator/c/src/profiling.c
+++ b/rpython/translator/c/src/profiling.c
@@ -9,7 +9,7 @@
 static cpu_set_t base_cpu_set;
 static int profiling_setup = 0;
 
-RPY_EXPORTED_FOR_TESTS
+RPY_EXTERN
 void pypy_setup_profiling(void)
 {
   if (!profiling_setup) {
@@ -22,7 +22,7 @@
   }
 }
 
-RPY_EXPORTED_FOR_TESTS
+RPY_EXTERN
 void pypy_teardown_profiling(void)
 {
   if (profiling_setup) {
@@ -40,7 +40,7 @@
 static DWORD_PTR base_affinity_mask;
 static int profiling_setup = 0;
 
-RPY_EXPORTED_FOR_TESTS
+RPY_EXTERN
 void pypy_setup_profiling(void) {
     if (!profiling_setup) {
         DWORD_PTR affinity_mask, system_affinity_mask;
@@ -56,7 +56,7 @@
     }
 }
 
-RPY_EXPORTED_FOR_TESTS
+RPY_EXTERN
 void pypy_teardown_profiling(void) {
     if (profiling_setup) {
         SetProcessAffinityMask(GetCurrentProcess(), base_affinity_mask);
@@ -67,7 +67,7 @@
 #else
 
 /* Empty implementations for other platforms */
-RPY_EXPORTED_FOR_TESTS void pypy_setup_profiling(void) { }
-RPY_EXPORTED_FOR_TESTS void pypy_teardown_profiling(void) { }
+RPY_EXTERN void pypy_setup_profiling(void) { }
+RPY_EXTERN void pypy_teardown_profiling(void) { }
 
 #endif
diff --git a/rpython/translator/c/src/profiling.h b/rpython/translator/c/src/profiling.h
--- a/rpython/translator/c/src/profiling.h
+++ b/rpython/translator/c/src/profiling.h
@@ -1,7 +1,7 @@
 #ifndef _PYPY_PROFILING_H
 #define _PYPY_PROFILING_H
 
-RPY_EXPORTED_FOR_TESTS void pypy_setup_profiling(void);
-RPY_EXPORTED_FOR_TESTS void pypy_teardown_profiling(void);
+RPY_EXTERN void pypy_setup_profiling(void);
+RPY_EXTERN void pypy_teardown_profiling(void);
 
 #endif
diff --git a/rpython/translator/c/src/rtyper.h b/rpython/translator/c/src/rtyper.h
--- a/rpython/translator/c/src/rtyper.h
+++ b/rpython/translator/c/src/rtyper.h
@@ -9,6 +9,6 @@
 #define RPyUnicode_Size(rpu)		((rpu)->ru_chars.length)
 #define _RPyUnicode_AsUnicode(rpu)	((rpu)->ru_chars.items)
 
-char *RPyString_AsCharP(RPyString *rps);
-void RPyString_FreeCache(void);
-RPyString *RPyString_FromString(char *buf);
+RPY_EXTERN char *RPyString_AsCharP(RPyString *rps);
+RPY_EXTERN void RPyString_FreeCache(void);
+RPY_EXTERN RPyString *RPyString_FromString(char *buf);
diff --git a/rpython/translator/c/src/signals.h b/rpython/translator/c/src/signals.h
--- a/rpython/translator/c/src/signals.h
+++ b/rpython/translator/c/src/signals.h
@@ -5,22 +5,22 @@
 
 
 /* utilities to set a signal handler */
-RPY_EXPORTED_FOR_TESTS
+RPY_EXTERN
 void pypysig_ignore(int signum);  /* signal will be ignored (SIG_IGN) */
-RPY_EXPORTED_FOR_TESTS
+RPY_EXTERN
 void pypysig_default(int signum); /* signal will do default action (SIG_DFL) */
-RPY_EXPORTED_FOR_TESTS
+RPY_EXTERN
 void pypysig_setflag(int signum); /* signal will set a flag which can be
                                      queried with pypysig_poll() */
-RPY_EXPORTED_FOR_TESTS
+RPY_EXTERN
 void pypysig_reinstall(int signum);
-RPY_EXPORTED_FOR_TESTS
+RPY_EXTERN
 int pypysig_set_wakeup_fd(int fd);
 
 /* utility to poll for signals that arrived */
-RPY_EXPORTED_FOR_TESTS
+RPY_EXTERN
 int pypysig_poll(void);   /* => signum or -1 */
-RPY_EXPORTED_FOR_TESTS
+RPY_EXTERN
 void pypysig_pushback(int signum);
 
 /* When a signal is received, pypysig_counter is set to -1. */
@@ -28,12 +28,12 @@
 struct pypysig_long_struct {
     long value;
 };
-extern struct pypysig_long_struct pypysig_counter;
+RPY_EXTERN struct pypysig_long_struct pypysig_counter;
 
 /* some C tricks to get/set the variable as efficiently as possible:
    use macros when compiling as a stand-alone program, but still
    export a function with the correct name for testing */
-RPY_EXPORTED_FOR_TESTS
+RPY_EXTERN
 void *pypysig_getaddr_occurred(void);
 #define pypysig_getaddr_occurred()   ((void *)(&pypysig_counter))
 
diff --git a/rpython/translator/c/src/stack.h b/rpython/translator/c/src/stack.h
--- a/rpython/translator/c/src/stack.h
+++ b/rpython/translator/c/src/stack.h
@@ -11,11 +11,13 @@
  * It is needed to have RPyThreadStaticTLS, too. */
 #include "threadlocal.h"
 
-extern char *_LLstacktoobig_stack_end;
-extern long _LLstacktoobig_stack_length;
-extern char _LLstacktoobig_report_error;
+RPY_EXTERN char *_LLstacktoobig_stack_end;
+RPY_EXTERN long _LLstacktoobig_stack_length;
+RPY_EXTERN char _LLstacktoobig_report_error;
 
+RPY_EXTERN
 char LL_stack_too_big_slowpath(long);    /* returns 0 (ok) or 1 (too big) */
+RPY_EXTERN
 void LL_stack_set_length_fraction(double);
 
 /* some macros referenced from rpython.rlib.rstack */
diff --git a/rpython/translator/c/src/stacklet/stacklet.h b/rpython/translator/c/src/stacklet/stacklet.h
--- a/rpython/translator/c/src/stacklet/stacklet.h
+++ b/rpython/translator/c/src/stacklet/stacklet.h
@@ -22,8 +22,8 @@
  */
 typedef struct stacklet_thread_s *stacklet_thread_handle;
 
-RPY_EXPORTED_FOR_TESTS stacklet_thread_handle stacklet_newthread(void);
-void stacklet_deletethread(stacklet_thread_handle thrd);
+RPY_EXTERN stacklet_thread_handle stacklet_newthread(void);
+RPY_EXTERN void stacklet_deletethread(stacklet_thread_handle thrd);
 
 
 /* The "run" function of a stacklet.  The first argument is the handle
@@ -35,7 +35,7 @@
 /* Call 'run(source, run_arg)' in a new stack.  See stacklet_switch()
  * for the return value.
  */
-RPY_EXPORTED_FOR_TESTS stacklet_handle stacklet_new(stacklet_thread_handle thrd,
+RPY_EXTERN stacklet_handle stacklet_new(stacklet_thread_handle thrd,
                              stacklet_run_fn run, void *run_arg);
 
 /* Switch to the target handle, resuming its stack.  This returns:
@@ -45,18 +45,19 @@
  * Don't call this with an already-used target, with EMPTY_STACKLET_HANDLE,
  * or with a stack handle from another thread (in multithreaded apps).
  */
-RPY_EXPORTED_FOR_TESTS stacklet_handle stacklet_switch(stacklet_handle target);
+RPY_EXTERN stacklet_handle stacklet_switch(stacklet_handle target);
 
 /* Delete a stack handle without resuming it at all.
  * (This works even if the stack handle is of a different thread)
  */
-void stacklet_destroy(stacklet_handle target);
+RPY_EXTERN void stacklet_destroy(stacklet_handle target);
 
 /* stacklet_handle _stacklet_switch_to_copy(stacklet_handle) --- later */
 
 /* Hack: translate a pointer into the stack of a stacklet into a pointer
  * to where it is really stored so far.  Only to access word-sized data.
  */
+RPY_EXTERN
 char **_stacklet_translate_pointer(stacklet_handle context, char **ptr);
 
 #endif /* _STACKLET_H_ */
diff --git a/rpython/translator/c/src/support.h b/rpython/translator/c/src/support.h
--- a/rpython/translator/c/src/support.h
+++ b/rpython/translator/c/src/support.h
@@ -27,12 +27,14 @@
 #  define RPyAssert(x, msg)                                             \
      if (!(x)) RPyAssertFailed(__FILE__, __LINE__, __FUNCTION__, msg)
 
+RPY_EXTERN
 void RPyAssertFailed(const char* filename, long lineno,
                      const char* function, const char *msg);
 #else
 #  define RPyAssert(x, msg)   /* nothing */
 #endif
 
+RPY_EXTERN
 void RPyAbort(void);
 
 #if defined(RPY_LL_ASSERT) || defined(RPY_SANDBOXED)
diff --git a/rpython/translator/c/src/thread.h b/rpython/translator/c/src/thread.h
--- a/rpython/translator/c/src/thread.h
+++ b/rpython/translator/c/src/thread.h
@@ -26,9 +26,9 @@
 
 #endif /* !_WIN32 */
 
-RPY_EXPORTED_FOR_TESTS void RPyGilAllocate(void);
-RPY_EXPORTED_FOR_TESTS long RPyGilYieldThread(void);
-RPY_EXPORTED_FOR_TESTS void RPyGilAcquire(void);
+RPY_EXTERN void RPyGilAllocate(void);
+RPY_EXTERN long RPyGilYieldThread(void);
+RPY_EXTERN void RPyGilAcquire(void);
 #define RPyGilRelease _RPyGilRelease
 #define RPyFetchFastGil _RPyFetchFastGil
 
@@ -38,7 +38,7 @@
 # define RPY_FASTGIL_LOCKED(x)   (x != 0)
 #endif
 
-extern long rpy_fastgil;
+RPY_EXTERN long rpy_fastgil;
 
 static inline void _RPyGilRelease(void) {
     assert(RPY_FASTGIL_LOCKED(rpy_fastgil));
diff --git a/rpython/translator/c/src/thread_gil.c b/rpython/translator/c/src/thread_gil.c
--- a/rpython/translator/c/src/thread_gil.c
+++ b/rpython/translator/c/src/thread_gil.c
@@ -162,7 +162,7 @@
 */
 
 #undef RPyGilRelease
-RPY_EXPORTED_FOR_TESTS
+RPY_EXTERN
 void RPyGilRelease(void)
 {
     /* Releases the GIL in order to do an external function call.
@@ -173,7 +173,7 @@
 }
 
 #undef RPyFetchFastGil
-RPY_EXPORTED_FOR_TESTS
+RPY_EXTERN
 long *RPyFetchFastGil(void)
 {
     return _RPyFetchFastGil();
diff --git a/rpython/translator/c/src/thread_nt.h b/rpython/translator/c/src/thread_nt.h
--- a/rpython/translator/c/src/thread_nt.h
+++ b/rpython/translator/c/src/thread_nt.h
@@ -12,23 +12,23 @@
 } NRMUTEX, *PNRMUTEX;
 
 /* prototypes */
-RPY_EXPORTED_FOR_TESTS
+RPY_EXTERN
 long RPyThreadGetIdent(void);
-RPY_EXPORTED_FOR_TESTS
+RPY_EXTERN
 long RPyThreadStart(void (*func)(void));
-RPY_EXPORTED_FOR_TESTS
+RPY_EXTERN
 int RPyThreadLockInit(struct RPyOpaque_ThreadLock *lock);
-RPY_EXPORTED_FOR_TESTS
+RPY_EXTERN
 void RPyOpaqueDealloc_ThreadLock(struct RPyOpaque_ThreadLock *lock);
-RPY_EXPORTED_FOR_TESTS
+RPY_EXTERN
 int RPyThreadAcquireLock(struct RPyOpaque_ThreadLock *lock, int waitflag);
-RPY_EXPORTED_FOR_TESTS
+RPY_EXTERN
 RPyLockStatus RPyThreadAcquireLockTimed(struct RPyOpaque_ThreadLock *lock,
 					RPY_TIMEOUT_T timeout, int intr_flag);
-RPY_EXPORTED_FOR_TESTS
+RPY_EXTERN
 void RPyThreadReleaseLock(struct RPyOpaque_ThreadLock *lock);
-RPY_EXPORTED_FOR_TESTS
+RPY_EXTERN
 long RPyThreadGetStackSize(void);
-RPY_EXPORTED_FOR_TESTS
+RPY_EXTERN
 long RPyThreadSetStackSize(long);
 #endif
diff --git a/rpython/translator/c/src/thread_pthread.h b/rpython/translator/c/src/thread_pthread.h
--- a/rpython/translator/c/src/thread_pthread.h
+++ b/rpython/translator/c/src/thread_pthread.h
@@ -59,24 +59,24 @@
 
 /* prototypes */
 
-RPY_EXPORTED_FOR_TESTS
+RPY_EXTERN
 long RPyThreadGetIdent(void);
-RPY_EXPORTED_FOR_TESTS
+RPY_EXTERN
 long RPyThreadStart(void (*func)(void));
-RPY_EXPORTED_FOR_TESTS
+RPY_EXTERN
 int RPyThreadLockInit(struct RPyOpaque_ThreadLock *lock);
-RPY_EXPORTED_FOR_TESTS
+RPY_EXTERN
 void RPyOpaqueDealloc_ThreadLock(struct RPyOpaque_ThreadLock *lock);
-RPY_EXPORTED_FOR_TESTS
+RPY_EXTERN
 int RPyThreadAcquireLock(struct RPyOpaque_ThreadLock *lock, int waitflag);
-RPY_EXPORTED_FOR_TESTS
+RPY_EXTERN
 RPyLockStatus RPyThreadAcquireLockTimed(struct RPyOpaque_ThreadLock *lock,
 					RPY_TIMEOUT_T timeout, int intr_flag);
-RPY_EXPORTED_FOR_TESTS
+RPY_EXTERN
 void RPyThreadReleaseLock(struct RPyOpaque_ThreadLock *lock);
-RPY_EXPORTED_FOR_TESTS
+RPY_EXTERN
 long RPyThreadGetStackSize(void);
-RPY_EXPORTED_FOR_TESTS
+RPY_EXTERN
 long RPyThreadSetStackSize(long);
-RPY_EXPORTED_FOR_TESTS
+RPY_EXTERN
 void RPyThreadAfterFork(void);
diff --git a/rpython/translator/c/src/threadlocal.h b/rpython/translator/c/src/threadlocal.h
--- a/rpython/translator/c/src/threadlocal.h
+++ b/rpython/translator/c/src/threadlocal.h
@@ -2,6 +2,8 @@
 #ifndef _SRC_THREADLOCAL_H
 #define _SRC_THREADLOCAL_H
 
+#include 
+
 
 #ifdef _WIN32
 
@@ -38,7 +40,7 @@
 #define RPyThreadStaticTLS_Create(key) RPyThreadTLS_Create(key)
 #define RPyThreadStaticTLS_Get(key)    RPyThreadTLS_Get(key)
 #define RPyThreadStaticTLS_Set(key, value) RPyThreadTLS_Set(key, value)
-void RPyThreadTLS_Create(RPyThreadTLS *result);
+RPY_EXTERN void RPyThreadTLS_Create(RPyThreadTLS *result);
 
 #endif
 
diff --git a/rpython/translator/c/support.py b/rpython/translator/c/support.py
--- a/rpython/translator/c/support.py
+++ b/rpython/translator/c/support.py
@@ -32,21 +32,16 @@
 
 def forward_cdecl(ctype, cname, standalone, is_thread_local=False,
                   is_exported=False):
-    prefix = ""
-    if is_thread_local:
-        prefix = "__thread "
+    # 'standalone' ignored
     if is_exported:
         assert not is_thread_local
         prefix = "RPY_EXPORTED "
-    elif standalone:
-        prefix += "RPY_HIDDEN "
+    else:
+        prefix = "RPY_EXTERN "
+        if is_thread_local:
+            prefix += "__thread "
+    return prefix + cdecl(ctype, cname)
 
-    cdecl_str = prefix + cdecl(ctype, cname)
-    if standalone:
-        return 'extern ' + cdecl_str
-    else:
-        return cdecl_str
-    
 def somelettersfrom(s):
     upcase = [c for c in s if c.isupper()]
     if not upcase:
diff --git a/rpython/translator/tool/cbuild.py b/rpython/translator/tool/cbuild.py
--- a/rpython/translator/tool/cbuild.py
+++ b/rpython/translator/tool/cbuild.py
@@ -315,7 +315,7 @@
             d['link_files'] = [fn for fn in d['link_files']
                                   if not fn.endswith('.a')]
         d['compile_extra'] = d['compile_extra'] + (
-            '-DRPY_EXPORTED_FOR_TESTS=RPY_EXPORTED',)
+            '-DRPY_EXTERN=RPY_EXPORTED',)
         self = ExternalCompilationInfo(**d)
 
         lib = str(host.compile([], self, outputfilename=outputfilename,
diff --git a/rpython/translator/tool/test/test_cbuild.py b/rpython/translator/tool/test/test_cbuild.py
--- a/rpython/translator/tool/test/test_cbuild.py
+++ b/rpython/translator/tool/test/test_cbuild.py
@@ -74,7 +74,7 @@
     def test_make_shared_lib(self):
         eci = ExternalCompilationInfo(
             separate_module_sources = ['''
-            RPY_EXPORTED_FOR_TESTS int get()
+            RPY_EXTERN int get()
             {
                 return 42;
             }

From noreply at buildbot.pypy.org  Sun Nov  9 17:47:22 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Sun,  9 Nov 2014 17:47:22 +0100 (CET)
Subject: [pypy-commit] pypy default: more RPY_EXPORTED_FOR_TESTS =>
	RPY_EXTERN
Message-ID: <20141109164722.782A71C3526@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r74416:ac5c44882bb1
Date: 2014-11-09 13:44 +0100
http://bitbucket.org/pypy/pypy/changeset/ac5c44882bb1/

Log:	more RPY_EXPORTED_FOR_TESTS => RPY_EXTERN

diff --git a/pypy/module/_multibytecodec/src/cjkcodecs/multibytecodec.h b/pypy/module/_multibytecodec/src/cjkcodecs/multibytecodec.h
--- a/pypy/module/_multibytecodec/src/cjkcodecs/multibytecodec.h
+++ b/pypy/module/_multibytecodec/src/cjkcodecs/multibytecodec.h
@@ -97,24 +97,24 @@
   Py_UNICODE *outbuf_start, *outbuf, *outbuf_end;
 };
 
-RPY_EXPORTED_FOR_TESTS
+RPY_EXTERN
 struct pypy_cjk_dec_s *pypy_cjk_dec_new(const MultibyteCodec *codec);
-RPY_EXPORTED_FOR_TESTS
+RPY_EXTERN
 Py_ssize_t pypy_cjk_dec_init(struct pypy_cjk_dec_s *d,
                              char *inbuf, Py_ssize_t inlen);
-RPY_EXPORTED_FOR_TESTS
+RPY_EXTERN
 void pypy_cjk_dec_free(struct pypy_cjk_dec_s *);
-RPY_EXPORTED_FOR_TESTS
+RPY_EXTERN
 Py_ssize_t pypy_cjk_dec_chunk(struct pypy_cjk_dec_s *);
-RPY_EXPORTED_FOR_TESTS
+RPY_EXTERN
 Py_UNICODE *pypy_cjk_dec_outbuf(struct pypy_cjk_dec_s *);
-RPY_EXPORTED_FOR_TESTS
+RPY_EXTERN
 Py_ssize_t pypy_cjk_dec_outlen(struct pypy_cjk_dec_s *);
-RPY_EXPORTED_FOR_TESTS
+RPY_EXTERN
 Py_ssize_t pypy_cjk_dec_inbuf_remaining(struct pypy_cjk_dec_s *d);
-RPY_EXPORTED_FOR_TESTS
+RPY_EXTERN
 Py_ssize_t pypy_cjk_dec_inbuf_consumed(struct pypy_cjk_dec_s* d);
-RPY_EXPORTED_FOR_TESTS
+RPY_EXTERN
 Py_ssize_t pypy_cjk_dec_replace_on_error(struct pypy_cjk_dec_s* d,
                                          Py_UNICODE *, Py_ssize_t, Py_ssize_t);
 
@@ -125,35 +125,35 @@
   unsigned char *outbuf_start, *outbuf, *outbuf_end;
 };
 
-RPY_EXPORTED_FOR_TESTS
+RPY_EXTERN
 struct pypy_cjk_enc_s *pypy_cjk_enc_new(const MultibyteCodec *codec);
-RPY_EXPORTED_FOR_TESTS
+RPY_EXTERN
 Py_ssize_t pypy_cjk_enc_init(struct pypy_cjk_enc_s *d,
                              Py_UNICODE *inbuf, Py_ssize_t inlen);
-RPY_EXPORTED_FOR_TESTS
+RPY_EXTERN
 void pypy_cjk_enc_free(struct pypy_cjk_enc_s *);
-RPY_EXPORTED_FOR_TESTS
+RPY_EXTERN
 Py_ssize_t pypy_cjk_enc_chunk(struct pypy_cjk_enc_s *, Py_ssize_t);
-RPY_EXPORTED_FOR_TESTS
+RPY_EXTERN
 Py_ssize_t pypy_cjk_enc_reset(struct pypy_cjk_enc_s *);
-RPY_EXPORTED_FOR_TESTS
+RPY_EXTERN
 char *pypy_cjk_enc_outbuf(struct pypy_cjk_enc_s *);
-RPY_EXPORTED_FOR_TESTS
+RPY_EXTERN
 Py_ssize_t pypy_cjk_enc_outlen(struct pypy_cjk_enc_s *);
-RPY_EXPORTED_FOR_TESTS
+RPY_EXTERN
 Py_ssize_t pypy_cjk_enc_inbuf_remaining(struct pypy_cjk_enc_s *d);
-RPY_EXPORTED_FOR_TESTS
+RPY_EXTERN
 Py_ssize_t pypy_cjk_enc_inbuf_consumed(struct pypy_cjk_enc_s* d);
-RPY_EXPORTED_FOR_TESTS
+RPY_EXTERN
 Py_ssize_t pypy_cjk_enc_replace_on_error(struct pypy_cjk_enc_s* d,
                                          char *, Py_ssize_t, Py_ssize_t);
-RPY_EXPORTED_FOR_TESTS
+RPY_EXTERN
 const MultibyteCodec *pypy_cjk_enc_getcodec(struct pypy_cjk_enc_s *);
 
 /* list of codecs defined in the .c files */
 
 #define DEFINE_CODEC(name)                              \
-    RPY_EXPORTED_FOR_TESTS MultibyteCodec *pypy_cjkcodec_##name(void);
+    RPY_EXTERN MultibyteCodec *pypy_cjkcodec_##name(void);
 
 // _codecs_cn
 DEFINE_CODEC(gb2312)
diff --git a/pypy/module/_ssl/thread_lock.py b/pypy/module/_ssl/thread_lock.py
--- a/pypy/module/_ssl/thread_lock.py
+++ b/pypy/module/_ssl/thread_lock.py
@@ -65,7 +65,7 @@
 eci = rthread.eci.merge(ExternalCompilationInfo(
     separate_module_sources=[separate_module_source],
     post_include_bits=[
-        "RPY_EXPORTED_FOR_TESTS int _PyPy_SSL_SetupThreads(void);"],
+        "RPY_EXTERN int _PyPy_SSL_SetupThreads(void);"],
     libraries = libraries,
 ))
 
diff --git a/pypy/module/cppyy/src/dummy_backend.cxx b/pypy/module/cppyy/src/dummy_backend.cxx
--- a/pypy/module/cppyy/src/dummy_backend.cxx
+++ b/pypy/module/cppyy/src/dummy_backend.cxx
@@ -349,29 +349,29 @@
 
 
 /* name to opaque C++ scope representation -------------------------------- */
-RPY_EXPORTED_FOR_TESTS
+RPY_EXTERN
 int cppyy_num_scopes(cppyy_scope_t handle) {
     return 0;
 }
 
-RPY_EXPORTED_FOR_TESTS
+RPY_EXTERN
 char* cppyy_resolve_name(const char* cppitem_name) {
     return cppstring_to_cstring(cppitem_name);
 }
 
-RPY_EXPORTED_FOR_TESTS
+RPY_EXTERN
 cppyy_scope_t cppyy_get_scope(const char* scope_name) {
     return s_handles[scope_name];  // lookup failure will return 0 (== error)
 }
 
-RPY_EXPORTED_FOR_TESTS
+RPY_EXTERN
 cppyy_type_t cppyy_actual_class(cppyy_type_t klass, cppyy_object_t /* obj */) {
     return klass;
 }
 
 
 /* memory management ------------------------------------------------------ */
-RPY_EXPORTED_FOR_TESTS
+RPY_EXTERN
 void cppyy_destruct(cppyy_type_t handle, cppyy_object_t self) {
     if (handle == s_handles["example01"])
        delete (dummy::example01*)self;
@@ -379,7 +379,7 @@
 
 
 /* method/function dispatching -------------------------------------------- */
-RPY_EXPORTED_FOR_TESTS
+RPY_EXTERN
 void cppyy_call_v(cppyy_method_t method, cppyy_object_t self, int nargs, void* args) {
     long idx = (long)method;
     if (idx == s_methods["static_example01::staticSetPayload_payload*_double"]) {
@@ -469,7 +469,7 @@
     }
 }
 
-RPY_EXPORTED_FOR_TESTS
+RPY_EXTERN
 unsigned char cppyy_call_b(cppyy_method_t method, cppyy_object_t self, int nargs, void* args) {
     unsigned char result = 0;
     const long idx = (long)method;
@@ -482,7 +482,7 @@
     return result;
 }
 
-RPY_EXPORTED_FOR_TESTS
+RPY_EXTERN
 char cppyy_call_c(cppyy_method_t method, cppyy_object_t self, int nargs, void* args) {
     char result = 0;
     const long idx = (long)method;
@@ -498,7 +498,7 @@
     return result;
 }
 
-RPY_EXPORTED_FOR_TESTS
+RPY_EXTERN
 short cppyy_call_h(cppyy_method_t method, cppyy_object_t self, int nargs, void* args) {
     short result = 0;
     const long idx = (long)method; 
@@ -514,7 +514,7 @@
     return result;
 }
 
-RPY_EXPORTED_FOR_TESTS
+RPY_EXTERN
 int cppyy_call_i(cppyy_method_t method, cppyy_object_t self, int nargs, void* args) {
     int result = 0;
     const long idx = (long)method;
@@ -547,7 +547,7 @@
     return result;
 }
 
-RPY_EXPORTED_FOR_TESTS
+RPY_EXTERN
 long cppyy_call_l(cppyy_method_t method, cppyy_object_t self, int nargs, void* args) {
     long result = 0;
     const long idx = (long)method;
@@ -689,7 +689,7 @@
     return result;
 }
 
-RPY_EXPORTED_FOR_TESTS
+RPY_EXTERN
 long long cppyy_call_ll(cppyy_method_t method, cppyy_object_t self, int nargs, void* args) {
     long long result = 0;
     const long idx = (long)method;
@@ -705,7 +705,7 @@
     return result;
 }   
 
-RPY_EXPORTED_FOR_TESTS
+RPY_EXTERN
 float cppyy_call_f(cppyy_method_t method, cppyy_object_t self, int nargs, void* args) {
     float result = 0;
     const long idx = (long)method;
@@ -718,7 +718,7 @@
     return result;
 }   
 
-RPY_EXPORTED_FOR_TESTS
+RPY_EXTERN
 double cppyy_call_d(cppyy_method_t method, cppyy_object_t self, int nargs, void* args) {
     double result = 0.;
     const long idx = (long)method;
@@ -740,7 +740,7 @@
     return result;
 }
 
-RPY_EXPORTED_FOR_TESTS
+RPY_EXTERN
 char* cppyy_call_s(cppyy_method_t method, cppyy_object_t self, int nargs, void* args) {
     char* result = 0;
     const long idx = (long)method;
@@ -753,7 +753,7 @@
     return result;
 }
 
-RPY_EXPORTED_FOR_TESTS
+RPY_EXTERN
 cppyy_object_t cppyy_constructor(cppyy_method_t method, cppyy_type_t handle, int nargs, void* args) {
     void* result = 0;
     const long idx = (long)method;
@@ -776,14 +776,14 @@
     return (cppyy_object_t)result;
 }
 
-RPY_EXPORTED_FOR_TESTS
+RPY_EXTERN
 cppyy_methptrgetter_t cppyy_get_methptr_getter(cppyy_type_t /* handle */, cppyy_index_t /* method_index */) {
     return (cppyy_methptrgetter_t)0;
 }
 
 
 /* handling of function argument buffer ----------------------------------- */
-RPY_EXPORTED_FOR_TESTS
+RPY_EXTERN
 void* cppyy_allocate_function_args(int nargs) {
     CPPYY_G__value* args = (CPPYY_G__value*)malloc(nargs*sizeof(CPPYY_G__value));
     for (int i = 0; i < nargs; ++i)
@@ -793,36 +793,36 @@
 
 
 /* handling of function argument buffer ----------------------------------- */
-RPY_EXPORTED_FOR_TESTS
+RPY_EXTERN
 void cppyy_deallocate_function_args(void* args) {
     free(args);
 }
 
-RPY_EXPORTED_FOR_TESTS
+RPY_EXTERN
 size_t cppyy_function_arg_sizeof() {
     return sizeof(CPPYY_G__value);
 }
 
-RPY_EXPORTED_FOR_TESTS
+RPY_EXTERN
 size_t cppyy_function_arg_typeoffset() {
     return offsetof(CPPYY_G__value, type);
 }
 
 
 /* scope reflection information ------------------------------------------- */
-RPY_EXPORTED_FOR_TESTS
+RPY_EXTERN
 int cppyy_is_namespace(cppyy_scope_t /* handle */) {
     return 0;
 }   
 
-RPY_EXPORTED_FOR_TESTS
+RPY_EXTERN
 int cppyy_is_enum(const char* /* type_name */) {
     return 0;
 }
     
     
 /* class reflection information ------------------------------------------- */
-RPY_EXPORTED_FOR_TESTS
+RPY_EXTERN
 char* cppyy_final_name(cppyy_type_t handle) {
     for (Handles_t::iterator isp = s_handles.begin(); isp != s_handles.end(); ++isp) {
         if (isp->second == handle)
@@ -831,75 +831,75 @@
     return cppstring_to_cstring("");
 }
 
-RPY_EXPORTED_FOR_TESTS
+RPY_EXTERN
 char* cppyy_scoped_final_name(cppyy_type_t handle) {
     return cppyy_final_name(handle);
 }   
 
-RPY_EXPORTED_FOR_TESTS
+RPY_EXTERN
 int cppyy_has_complex_hierarchy(cppyy_type_t /* handle */) {
     return 0;
 }
 
-RPY_EXPORTED_FOR_TESTS
+RPY_EXTERN
 int cppyy_num_bases(cppyy_type_t /*handle*/) {
    return 0;
 }
 
 
 /* method/function reflection information --------------------------------- */
-RPY_EXPORTED_FOR_TESTS
+RPY_EXTERN
 int cppyy_num_methods(cppyy_scope_t handle) {
     return s_scopes[handle].m_methods.size();
 }
 
-RPY_EXPORTED_FOR_TESTS
+RPY_EXTERN
 cppyy_index_t cppyy_method_index_at(cppyy_scope_t /* scope */, int imeth) {
     return (cppyy_index_t)imeth;
 }
 
-RPY_EXPORTED_FOR_TESTS
+RPY_EXTERN
 char* cppyy_method_name(cppyy_scope_t handle, cppyy_index_t method_index) {
     return cppstring_to_cstring(s_scopes[handle].m_methods[(int)method_index].m_name);
 }
 
-RPY_EXPORTED_FOR_TESTS
+RPY_EXTERN
 char* cppyy_method_result_type(cppyy_scope_t handle, cppyy_index_t method_index) {
     return cppstring_to_cstring(s_scopes[handle].m_methods[method_index].m_returntype);
 }
     
-RPY_EXPORTED_FOR_TESTS
+RPY_EXTERN
 int cppyy_method_num_args(cppyy_scope_t handle, cppyy_index_t method_index) {
     return s_scopes[handle].m_methods[method_index].m_argtypes.size();
 }
 
-RPY_EXPORTED_FOR_TESTS
+RPY_EXTERN
 int cppyy_method_req_args(cppyy_scope_t handle, cppyy_index_t method_index) {
     return cppyy_method_num_args(handle, method_index);
 }
 
-RPY_EXPORTED_FOR_TESTS
+RPY_EXTERN
 char* cppyy_method_arg_type(cppyy_scope_t handle, cppyy_index_t method_index, int arg_index) {
     return cppstring_to_cstring(s_scopes[handle].m_methods[method_index].m_argtypes[arg_index]);
 }
 
-RPY_EXPORTED_FOR_TESTS
+RPY_EXTERN
 char* cppyy_method_arg_default(
         cppyy_scope_t /* handle */, cppyy_index_t /* method_index */, int /* arg_index */) {
     return cppstring_to_cstring("");
 }
 
-RPY_EXPORTED_FOR_TESTS
+RPY_EXTERN
 char* cppyy_method_signature(cppyy_scope_t /* handle */, cppyy_index_t /* method_index */) {
     return cppstring_to_cstring("");
 }
 
-RPY_EXPORTED_FOR_TESTS
+RPY_EXTERN
 int cppyy_method_is_template(cppyy_scope_t /* handle */, cppyy_index_t /* method_index */) {
     return 0;
 }
     
-RPY_EXPORTED_FOR_TESTS
+RPY_EXTERN
 cppyy_method_t cppyy_get_method(cppyy_scope_t handle, cppyy_index_t method_index) {
     if (s_scopes.find(handle) != s_scopes.end()) {
         long id = s_scopes[handle].m_method_offset + (long)method_index;
@@ -911,7 +911,7 @@
 
 
 /* method properties -----------------------------------------------------  */
-RPY_EXPORTED_FOR_TESTS
+RPY_EXTERN
 int cppyy_is_constructor(cppyy_type_t handle, cppyy_index_t method_index) {
     if (s_scopes.find(handle) != s_scopes.end())
         return s_scopes[handle].m_methods[method_index].m_type == kConstructor;
@@ -919,7 +919,7 @@
     return 0;
 }
 
-RPY_EXPORTED_FOR_TESTS
+RPY_EXTERN
 int cppyy_is_staticmethod(cppyy_type_t handle, cppyy_index_t method_index) {
     if (s_scopes.find(handle) != s_scopes.end())
         return s_scopes[handle].m_methods[method_index].m_type == kStatic;
@@ -929,34 +929,34 @@
 
 
 /* data member reflection information ------------------------------------- */
-RPY_EXPORTED_FOR_TESTS
+RPY_EXTERN
 int cppyy_num_datamembers(cppyy_scope_t handle) {
     return s_scopes[handle].m_datambrs.size();
 }
 
-RPY_EXPORTED_FOR_TESTS
+RPY_EXTERN
 char* cppyy_datamember_name(cppyy_scope_t handle, int idatambr) {
     return cppstring_to_cstring(s_scopes[handle].m_datambrs[idatambr].m_name);
 }
 
-RPY_EXPORTED_FOR_TESTS
+RPY_EXTERN
 char* cppyy_datamember_type(cppyy_scope_t handle, int idatambr) {
     return cppstring_to_cstring(s_scopes[handle].m_datambrs[idatambr].m_type);
 }
 
-RPY_EXPORTED_FOR_TESTS
+RPY_EXTERN
 ptrdiff_t cppyy_datamember_offset(cppyy_scope_t handle, int idatambr) {
     return s_scopes[handle].m_datambrs[idatambr].m_offset;
 }
 
 
 /* data member properties ------------------------------------------------  */
-RPY_EXPORTED_FOR_TESTS
+RPY_EXTERN
 int cppyy_is_publicdata(cppyy_scope_t handle, int idatambr) {
     return 1;
 }
 
-RPY_EXPORTED_FOR_TESTS
+RPY_EXTERN
 int cppyy_is_staticdata(cppyy_scope_t handle, int idatambr) {
     return s_scopes[handle].m_datambrs[idatambr].m_isstatic;
 }
@@ -964,44 +964,44 @@
 
 /* misc helpers ----------------------------------------------------------- */
 #if defined(_MSC_VER)
-RPY_EXPORTED_FOR_TESTS
+RPY_EXTERN
 long long cppyy_strtoll(const char* str) {
     return _strtoi64(str, NULL, 0);
 }
 
 extern "C" {
-RPY_EXPORTED_FOR_TESTS
+RPY_EXTERN
 unsigned long long cppyy_strtoull(const char* str) {
     return _strtoui64(str, NULL, 0);
 }
 }
 #else
-RPY_EXPORTED_FOR_TESTS
+RPY_EXTERN
 long long cppyy_strtoll(const char* str) {
     return strtoll(str, NULL, 0);
 }
 
 extern "C" {
-RPY_EXPORTED_FOR_TESTS
+RPY_EXTERN
 unsigned long long cppyy_strtoull(const char* str) {
     return strtoull(str, NULL, 0);
 }
 }
 #endif
 
-RPY_EXPORTED_FOR_TESTS
+RPY_EXTERN
 void cppyy_free(void* ptr) {
     free(ptr);
 }
 
-RPY_EXPORTED_FOR_TESTS
+RPY_EXTERN
 cppyy_object_t cppyy_charp2stdstring(const char* str) {
     void* arena = new char[sizeof(std::string)];
     new (arena) std::string(str);
     return (cppyy_object_t)arena;
 }
 
-RPY_EXPORTED_FOR_TESTS
+RPY_EXTERN
 cppyy_object_t cppyy_stdstring2stdstring(cppyy_object_t ptr) {
     void* arena = new char[sizeof(std::string)];
     new (arena) std::string(*(std::string*)ptr);
diff --git a/pypy/module/cppyy/test/conftest.py b/pypy/module/cppyy/test/conftest.py
--- a/pypy/module/cppyy/test/conftest.py
+++ b/pypy/module/cppyy/test/conftest.py
@@ -50,7 +50,7 @@
             eci = ExternalCompilationInfo(
                 separate_module_files=[srcpath.join('dummy_backend.cxx')],
                 include_dirs=[incpath, tstpath, cdir],
-                compile_extra=['-DRPY_EXPORTED_FOR_TESTS=RPY_EXPORTED'],
+                compile_extra=['-DRPY_EXTERN=RPY_EXPORTED'],
                 use_cpp_linker=True,
             )
 
diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -775,8 +775,7 @@
     struct PyPyAPI {
     %(members)s
     } _pypyAPI;
-    RPY_EXPORTED_FOR_TESTS
-    struct PyPyAPI* pypyAPI = &_pypyAPI;
+    RPY_EXTERN struct PyPyAPI* pypyAPI = &_pypyAPI;
     """ % dict(members=structmembers)
 
     functions = generate_decls_and_callbacks(db, export_symbols)
@@ -947,7 +946,7 @@
         name_no_star = process_va_name(name)
         header = ('%s pypy_va_get_%s(va_list* vp)' %
                   (name, name_no_star))
-        pypy_decls.append('RPY_EXPORTED_FOR_TESTS ' + header + ';')
+        pypy_decls.append('RPY_EXTERN ' + header + ';')
         functions.append(header + '\n{return va_arg(*vp, %s);}\n' % name)
 
     for name, (typ, expr) in GLOBALS.iteritems():
@@ -1007,7 +1006,7 @@
     if sys.platform == 'win32':
         get_pythonapi_source = '''
         #include 
-        RPY_EXPORTED_FOR_TESTS
+        RPY_EXTERN
         HANDLE pypy_get_pythonapi_handle() {
             MEMORY_BASIC_INFORMATION  mi;
             memset(&mi, 0, sizeof(mi));
diff --git a/pypy/module/operator/tscmp.h b/pypy/module/operator/tscmp.h
--- a/pypy/module/operator/tscmp.h
+++ b/pypy/module/operator/tscmp.h
@@ -1,2 +1,2 @@
-RPY_EXPORTED_FOR_TESTS int pypy_tscmp(const char *, const char *, long, long);
-RPY_EXPORTED_FOR_TESTS int pypy_tscmp_wide(const wchar_t *, const wchar_t *, long, long);
+RPY_EXTERN int pypy_tscmp(const char *, const char *, long, long);
+RPY_EXTERN int pypy_tscmp_wide(const wchar_t *, const wchar_t *, long, long);
diff --git a/pypy/module/rctime/interp_time.py b/pypy/module/rctime/interp_time.py
--- a/pypy/module/rctime/interp_time.py
+++ b/pypy/module/rctime/interp_time.py
@@ -35,7 +35,7 @@
     eci = ExternalCompilationInfo(
         includes = ['windows.h'],
         post_include_bits = [
-            "RPY_EXPORTED_FOR_TESTS\n"
+            "RPY_EXTERN\n"
             "BOOL pypy_timemodule_setCtrlHandler(HANDLE event);"],
         separate_module_sources=['''
             static HANDLE interrupt_event;
@@ -178,13 +178,13 @@
 if _WIN:
     win_eci = ExternalCompilationInfo(
         includes = ["time.h"],
-        post_include_bits = ["RPY_EXPORTED_FOR_TESTS "
+        post_include_bits = ["RPY_EXTERN "
                              "long pypy_get_timezone();\n"
-                             "RPY_EXPORTED_FOR_TESTS "
+                             "RPY_EXTERN "
                              "int pypy_get_daylight();\n"
-                             "RPY_EXPORTED_FOR_TESTS "
+                             "RPY_EXTERN "
                              "char** pypy_get_tzname();\n"
-                             "RPY_EXPORTED_FOR_TESTS "
+                             "RPY_EXTERN "
                              "void pypy__tzset();"],
         separate_module_sources = ["""
         long pypy_get_timezone() { return timezone; }

From noreply at buildbot.pypy.org  Sun Nov  9 17:47:23 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Sun,  9 Nov 2014 17:47:23 +0100 (CET)
Subject: [pypy-commit] pypy default: merge heads
Message-ID: <20141109164723.A13741C3526@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r74417:e82fcc48ffa6
Date: 2014-11-09 17:47 +0100
http://bitbucket.org/pypy/pypy/changeset/e82fcc48ffa6/

Log:	merge heads

diff --git a/rpython/translator/simplify.py b/rpython/translator/simplify.py
--- a/rpython/translator/simplify.py
+++ b/rpython/translator/simplify.py
@@ -5,6 +5,7 @@
 simplify_graph() applies all simplifications defined in this file.
 """
 import py
+from collections import defaultdict
 
 from rpython.flowspace.model import (Variable, Constant,
                                      c_last_exception, checkgraph, mkentrymap)
@@ -401,8 +402,8 @@
 def transform_dead_op_vars_in_blocks(blocks, graphs, translator=None):
     """Remove dead operations and variables that are passed over a link
     but not used in the target block. Input is a set of blocks"""
-    read_vars = {}  # set of variables really used
-    variable_flow = {}  # map {Var: list-of-Vars-it-depends-on}
+    read_vars = set()  # set of variables really used
+    dependencies = defaultdict(set) # map {Var: list-of-Vars-it-depends-on}
     set_of_blocks = set(blocks)
     start_blocks = find_start_blocks(graphs)
 
@@ -414,53 +415,48 @@
         # cannot remove the exc-raising operation
         return op is not block.operations[-1]
 
-    # compute variable_flow and an initial read_vars
+    # compute dependencies and an initial read_vars
     for block in blocks:
         # figure out which variables are ever read
         for op in block.operations:
-            if not canremove(op, block):   # mark the inputs as really needed
-                for arg in op.args:
-                    read_vars[arg] = True
+            if not canremove(op, block):   # the inputs are always needed
+                read_vars.update(op.args)
             else:
-                # if CanRemove, only mark dependencies of the result
-                # on the input variables
-                deps = variable_flow.setdefault(op.result, [])
-                deps.extend(op.args)
+                dependencies[op.result].update(op.args)
 
         if isinstance(block.exitswitch, Variable):
-            read_vars[block.exitswitch] = True
+            read_vars.add(block.exitswitch)
 
         if block.exits:
             for link in block.exits:
                 if link.target not in set_of_blocks:
                     for arg, targetarg in zip(link.args, link.target.inputargs):
-                        read_vars[arg] = True
-                        read_vars[targetarg] = True
+                        read_vars.add(arg)
+                        read_vars.add(targetarg)
                 else:
                     for arg, targetarg in zip(link.args, link.target.inputargs):
-                        deps = variable_flow.setdefault(targetarg, [])
-                        deps.append(arg)
+                        dependencies[targetarg].add(arg)
         else:
             # return and except blocks implicitely use their input variable(s)
             for arg in block.inputargs:
-                read_vars[arg] = True
-        # an input block's inputargs should not be modified, even if some
+                read_vars.add(arg)
+        # a start block's inputargs should not be modified, even if some
         # of the function's input arguments are not actually used
         if block in start_blocks:
             for arg in block.inputargs:
-                read_vars[arg] = True
+                read_vars.add(arg)
 
     # flow read_vars backwards so that any variable on which a read_vars
     # depends is also included in read_vars
     def flow_read_var_backward(pending):
-        pending = list(pending)
-        for var in pending:
-            for prevvar in variable_flow.get(var, []):
+        while pending:
+            var = pending.pop()
+            for prevvar in dependencies[var]:
                 if prevvar not in read_vars:
-                    read_vars[prevvar] = True
-                    pending.append(prevvar)
+                    read_vars.add(prevvar)
+                    pending.add(prevvar)
 
-    flow_read_var_backward(read_vars)
+    flow_read_var_backward(set(read_vars))
 
     for block in blocks:
 

From noreply at buildbot.pypy.org  Sun Nov  9 19:44:03 2014
From: noreply at buildbot.pypy.org (mattip)
Date: Sun,  9 Nov 2014 19:44:03 +0100 (CET)
Subject: [pypy-commit] pypy default: more RPY_ETERN definitions
Message-ID: <20141109184403.E0F041C33C0@cobra.cs.uni-duesseldorf.de>

Author: Matti Picus 
Branch: 
Changeset: r74418:e0ee57865d3d
Date: 2014-11-09 12:39 -0600
http://bitbucket.org/pypy/pypy/changeset/e0ee57865d3d/

Log:	more RPY_ETERN definitions

diff --git a/rpython/translator/c/src/ll_math.c b/rpython/translator/c/src/ll_math.c
--- a/rpython/translator/c/src/ll_math.c
+++ b/rpython/translator/c/src/ll_math.c
@@ -4,12 +4,12 @@
 #include 
 
 /* The following macros are copied from CPython header files */
+#include "src/precommondefs.h"
 
 #ifdef _WIN32
 #include 
 #include 
 #endif
-
 #ifdef _MSC_VER
 #define PyPy_IS_NAN _isnan
 #define PyPy_IS_INFINITY(X) (!_finite(X) && !_isnan(X))
@@ -57,7 +57,7 @@
  *      acosh(NaN) is NaN without signal.
  */
 
-double
+RPY_EXTERN double
 _pypy_math_acosh(double x)
 {
     if (PyPy_IS_NAN(x)) {
@@ -103,7 +103,7 @@
  *               := sign(x)*log1p(|x| + x^2/(1 + sqrt(1+x^2)))
  */
 
-double
+RPY_EXTERN double
 _pypy_math_asinh(double x)
 {
     double w;
@@ -146,7 +146,7 @@
  *
  */
 
-double
+RPY_EXTERN double
 _pypy_math_atanh(double x)
 {
     double absx;
@@ -181,7 +181,7 @@
    to avoid the significant loss of precision that arises from direct
    evaluation of the expression exp(x) - 1, for x near 0. */
 
-double
+RPY_EXTERN double
 _pypy_math_expm1(double x)
 {
     /* For abs(x) >= log(2), it's safe to evaluate exp(x) - 1 directly; this
@@ -207,7 +207,7 @@
    significant loss of precision that arises from direct evaluation when x is
    small. */
 
-double
+RPY_EXTERN double
 _pypy_math_log1p(double x)
 {
     /* For x small, we use the following approach.  Let y be the nearest float

From noreply at buildbot.pypy.org  Sun Nov  9 21:19:02 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Sun,  9 Nov 2014 21:19:02 +0100 (CET)
Subject: [pypy-commit] pypy default: Ooops. The gc-incminimark-pinning
 branch introduced a call to
Message-ID: <20141109201902.1EF7C1C354A@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r74419:40953f097e0e
Date: 2014-11-09 21:18 +0100
http://bitbucket.org/pypy/pypy/changeset/40953f097e0e/

Log:	Ooops. The gc-incminimark-pinning branch introduced a call to
	collect_and_reserve() from *every* malloc, rather than only when the
	nursery is full (which is comparatively very rare). Fix. I suspect
	this explains the 6% slow-down on a few benchmarks.

diff --git a/rpython/memory/gc/incminimark.py b/rpython/memory/gc/incminimark.py
--- a/rpython/memory/gc/incminimark.py
+++ b/rpython/memory/gc/incminimark.py
@@ -593,7 +593,10 @@
             #
             # Get the memory from the nursery.  If there is not enough space
             # there, do a collect first.
-            result = self.collect_and_reserve(rawtotalsize)
+            result = self.nursery_free
+            self.nursery_free = new_free = result + totalsize
+            if new_free > self.nursery_top:
+                result = self.collect_and_reserve(totalsize)
             #
             # Build the object.
             llarena.arena_reserve(result, totalsize)
@@ -649,7 +652,10 @@
             #
             # Get the memory from the nursery.  If there is not enough space
             # there, do a collect first.
-            result = self.collect_and_reserve(raw_malloc_usage(totalsize))
+            result = self.nursery_free
+            self.nursery_free = new_free = result + totalsize
+            if new_free > self.nursery_top:
+                result = self.collect_and_reserve(totalsize)
             #
             # Build the object.
             llarena.arena_reserve(result, totalsize)
@@ -682,13 +688,11 @@
         and finally reserve 'totalsize' bytes at the start of the
         now-empty nursery.
         """
-        if self.nursery_free + totalsize <= self.nursery_top:
-            result = self.nursery_free
-            self.nursery_free = result + totalsize
-            return result
 
         minor_collection_count = 0
         while True:
+            self.nursery_free = llmemory.NULL      # debug: don't use me
+
             if self.nursery_barriers.non_empty():
                 size_gc_header = self.gcheaderbuilder.size_gc_header
                 pinned_obj_size = size_gc_header + self.get_size(

From noreply at buildbot.pypy.org  Sun Nov  9 21:41:48 2014
From: noreply at buildbot.pypy.org (mattip)
Date: Sun,  9 Nov 2014 21:41:48 +0100 (CET)
Subject: [pypy-commit] pypy default: backed out changeset: e0ee57865d3d,
 fix windows own tests differently
Message-ID: <20141109204148.DB7231C33C0@cobra.cs.uni-duesseldorf.de>

Author: Matti Picus 
Branch: 
Changeset: r74420:b3b01191e99b
Date: 2014-11-09 14:32 -0600
http://bitbucket.org/pypy/pypy/changeset/b3b01191e99b/

Log:	backed out changeset: e0ee57865d3d, fix windows own tests
	differently

diff --git a/rpython/translator/c/src/ll_math.c b/rpython/translator/c/src/ll_math.c
--- a/rpython/translator/c/src/ll_math.c
+++ b/rpython/translator/c/src/ll_math.c
@@ -4,12 +4,12 @@
 #include 
 
 /* The following macros are copied from CPython header files */
-#include "src/precommondefs.h"
 
 #ifdef _WIN32
 #include 
 #include 
 #endif
+
 #ifdef _MSC_VER
 #define PyPy_IS_NAN _isnan
 #define PyPy_IS_INFINITY(X) (!_finite(X) && !_isnan(X))
@@ -57,7 +57,7 @@
  *      acosh(NaN) is NaN without signal.
  */
 
-RPY_EXTERN double
+double
 _pypy_math_acosh(double x)
 {
     if (PyPy_IS_NAN(x)) {
@@ -103,7 +103,7 @@
  *               := sign(x)*log1p(|x| + x^2/(1 + sqrt(1+x^2)))
  */
 
-RPY_EXTERN double
+double
 _pypy_math_asinh(double x)
 {
     double w;
@@ -146,7 +146,7 @@
  *
  */
 
-RPY_EXTERN double
+double
 _pypy_math_atanh(double x)
 {
     double absx;
@@ -181,7 +181,7 @@
    to avoid the significant loss of precision that arises from direct
    evaluation of the expression exp(x) - 1, for x near 0. */
 
-RPY_EXTERN double
+double
 _pypy_math_expm1(double x)
 {
     /* For abs(x) >= log(2), it's safe to evaluate exp(x) - 1 directly; this
@@ -207,7 +207,7 @@
    significant loss of precision that arises from direct evaluation when x is
    small. */
 
-RPY_EXTERN double
+double
 _pypy_math_log1p(double x)
 {
     /* For x small, we use the following approach.  Let y be the nearest float

From noreply at buildbot.pypy.org  Sun Nov  9 21:41:50 2014
From: noreply at buildbot.pypy.org (mattip)
Date: Sun,  9 Nov 2014 21:41:50 +0100 (CET)
Subject: [pypy-commit] pypy default: add RPY_EXTERN to function declarations
	via h file
Message-ID: <20141109204150.0F36B1C33C0@cobra.cs.uni-duesseldorf.de>

Author: Matti Picus 
Branch: 
Changeset: r74421:451f9a4e1dd6
Date: 2014-11-09 14:41 -0600
http://bitbucket.org/pypy/pypy/changeset/451f9a4e1dd6/

Log:	add RPY_EXTERN to function declarations via h file

diff --git a/rpython/translator/c/src/ll_math.c b/rpython/translator/c/src/ll_math.c
--- a/rpython/translator/c/src/ll_math.c
+++ b/rpython/translator/c/src/ll_math.c
@@ -10,6 +10,8 @@
 #include 
 #endif
 
+#include "src/ll_math.h"
+
 #ifdef _MSC_VER
 #define PyPy_IS_NAN _isnan
 #define PyPy_IS_INFINITY(X) (!_finite(X) && !_isnan(X))

From noreply at buildbot.pypy.org  Sun Nov  9 22:03:45 2014
From: noreply at buildbot.pypy.org (amauryfa)
Date: Sun,  9 Nov 2014 22:03:45 +0100 (CET)
Subject: [pypy-commit] pypy more-rposix: Move os.ttyname()
Message-ID: <20141109210345.E50D41C33C0@cobra.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc 
Branch: more-rposix
Changeset: r74422:12a878a11f79
Date: 2014-11-09 20:54 +0100
http://bitbucket.org/pypy/pypy/changeset/12a878a11f79/

Log:	Move os.ttyname()

diff --git a/rpython/rlib/rposix.py b/rpython/rlib/rposix.py
--- a/rpython/rlib/rposix.py
+++ b/rpython/rlib/rposix.py
@@ -826,7 +826,16 @@
     if not is_valid_fd(fd):
         return False
     return c_isatty(fd) != 0
-    
+
+c_ttyname = external('ttyname', [lltype.Signed], rffi.CCHARP, releasegil=False)
+
+ at replace_os_function('ttyname')
+def ttyname(fd):
+    l_name = os_ttyname(fd)
+    if not l_name:
+        raise OSError(get_errno(), "ttyname raised")
+    return rffi.charp2str(l_name)
+
 c_strerror = external('strerror', [rffi.INT], rffi.CCHARP,
                       releasegil=False)
 
diff --git a/rpython/rtyper/module/ll_os.py b/rpython/rtyper/module/ll_os.py
--- a/rpython/rtyper/module/ll_os.py
+++ b/rpython/rtyper/module/ll_os.py
@@ -102,8 +102,6 @@
 else:
     includes += ['sys/utime.h', 'sys/types.h']
 
-_CYGWIN = sys.platform == 'cygwin'
-
 class CConfig:
     """
     Definitions for platform integration.
@@ -119,19 +117,6 @@
     _compilation_info_ = ExternalCompilationInfo(
         includes=includes
     )
-    if not _WIN32:
-        CLOCK_T = platform.SimpleType('clock_t', rffi.INT)
-
-        TMS = platform.Struct(
-            'struct tms', [('tms_utime', rffi.INT),
-                           ('tms_stime', rffi.INT),
-                           ('tms_cutime', rffi.INT),
-                           ('tms_cstime', rffi.INT)])
-
-
-    SEEK_SET = platform.DefinedConstantInteger('SEEK_SET')
-    SEEK_CUR = platform.DefinedConstantInteger('SEEK_CUR')
-    SEEK_END = platform.DefinedConstantInteger('SEEK_END')
 
 
 class RegisterOs(BaseLazyRegistering):
@@ -168,21 +153,6 @@
             return ll_os_stat.register_statvfs_variant('statvfs', traits)
 
 
-    # ------------------------------- os.W* ---------------------------------
-
-    @registering_if(os, 'ttyname')
-    def register_os_ttyname(self):
-        os_ttyname = self.llexternal('ttyname', [lltype.Signed], rffi.CCHARP, releasegil=False)
-
-        def ttyname_llimpl(fd):
-            l_name = os_ttyname(fd)
-            if not l_name:
-                raise OSError(rposix.get_errno(), "ttyname raised")
-            return rffi.charp2str(l_name)
-
-        return extdef([int], str, "ll_os.ttyname",
-                      llimpl=ttyname_llimpl)
-
 # ____________________________________________________________
 # Support for os.environ
 

From noreply at buildbot.pypy.org  Sun Nov  9 22:18:42 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Sun,  9 Nov 2014 22:18:42 +0100 (CET)
Subject: [pypy-commit] pypy default: Fix the Makefile so that "make tests"
 can be run here. For some reason
Message-ID: <20141109211842.43E2F1C354A@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r74423:5aa672cd2f3c
Date: 2014-11-09 22:18 +0100
http://bitbucket.org/pypy/pypy/changeset/5aa672cd2f3c/

Log:	Fix the Makefile so that "make tests" can be run here. For some
	reason it seems never to have worked in the dynamic cases, but it's
	still useful for the static cases.

diff --git a/rpython/translator/c/src/stacklet/Makefile b/rpython/translator/c/src/stacklet/Makefile
--- a/rpython/translator/c/src/stacklet/Makefile
+++ b/rpython/translator/c/src/stacklet/Makefile
@@ -1,11 +1,13 @@
 
 all: stacklet.so
 
+INC = -I../..
+
 stacklet.so: stacklet.c stacklet.h
-	gcc -fPIC -shared -O2 -o $@ stacklet.c
+	gcc ${INC} -fPIC -shared -O2 -o $@ stacklet.c
 
 stacklet_g.so: stacklet.c stacklet.h
-	gcc -fPIC -shared -g -o $@ stacklet.c -DDEBUG_DUMP
+	gcc ${INC} -fPIC -shared -g -o $@ stacklet.c -DDEBUG_DUMP
 
 clean:
 	rm -fr stacklet.so stacklet_g.so
@@ -19,31 +21,30 @@
 
 ALL_TESTS = tests-static-g \
             tests-static-o \
-            tests-dynamic-g \
-            tests-dynamic-o
+            #tests-dynamic-g tests-dynamic-o
 
 run-all-tests: $(ALL_TESTS)
 	@echo "*** All test suites passed ***"
 
 tests-static-g: stacklet.c stacklet.h tests.c
-	gcc -Wall -g -o run_tests_static_g stacklet.c tests.c ${DEBUG}
+	gcc ${INC} -Wall -g -o run_tests_static_g stacklet.c tests.c ${DEBUG}
 	run_tests_static_g
 
 tests-static-o: stacklet.c stacklet.h tests.c
-	gcc -Wall -g -O2 -o run_tests_static_o stacklet.c tests.c ${DEBUG}
+	gcc ${INC} -Wall -g -O2 -o run_tests_static_o stacklet.c tests.c ${DEBUG}
 	run_tests_static_o
 
 tests-dynamic-g: stacklet_g.so tests.c
-	gcc -Wall -g -o run_tests_dynamic_g stacklet_g.so tests.c ${DEBUG}
+	gcc ${INC} -Wall -g -o run_tests_dynamic_g stacklet_g.so tests.c ${DEBUG}
 	LD_LIBRARY_PATH=. run_tests_dynamic_g
 
 tests-dynamic-o: stacklet.so tests.c
-	gcc -Wall -g -O2 -o run_tests_dynamic_o stacklet.so tests.c ${DEBUG}
+	gcc ${INC} -Wall -g -O2 -o run_tests_dynamic_o stacklet.so tests.c ${DEBUG}
 	LD_LIBRARY_PATH=. run_tests_dynamic_o
 
 tests-repeat: tests
 	python runtests.py run_tests_static_g > /dev/null
 	python runtests.py run_tests_static_o > /dev/null
-	LD_LIBRARY_PATH=. python runtests.py run_tests_dynamic_g > /dev/null
-	LD_LIBRARY_PATH=. python runtests.py run_tests_dynamic_o > /dev/null
+	#LD_LIBRARY_PATH=. python runtests.py run_tests_dynamic_g > /dev/null
+	#LD_LIBRARY_PATH=. python runtests.py run_tests_dynamic_o > /dev/null
 	@echo "*** All tests passed repeatedly ***"

From noreply at buildbot.pypy.org  Mon Nov 10 02:36:38 2014
From: noreply at buildbot.pypy.org (rlamy)
Date: Mon, 10 Nov 2014 02:36:38 +0100 (CET)
Subject: [pypy-commit] pypy default: rename all the things in
	annotator.follow_link()
Message-ID: <20141110013638.E4BB01C354A@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy 
Branch: 
Changeset: r74424:3b3f07bcbd47
Date: 2014-11-07 19:31 +0000
http://bitbucket.org/pypy/pypy/changeset/3b3f07bcbd47/

Log:	rename all the things in annotator.follow_link()

diff --git a/rpython/annotator/annrpython.py b/rpython/annotator/annrpython.py
--- a/rpython/annotator/annrpython.py
+++ b/rpython/annotator/annrpython.py
@@ -481,87 +481,84 @@
 
     def follow_link(self, graph, link, knowntypedata):
         in_except_block = False
-        last_exception_var = link.last_exception  # may be None for non-exception link
-        last_exc_value_var = link.last_exc_value  # may be None for non-exception link
+        v_last_exc_type = link.last_exception  # may be None for non-exception link
+        v_last_exc_value = link.last_exc_value  # may be None for non-exception link
 
-        if isinstance(link.exitcase, (types.ClassType, type)) \
-                and issubclass(link.exitcase, py.builtin.BaseException):
-            assert last_exception_var and last_exc_value_var
-            last_exc_value_object = self.bookkeeper.valueoftype(link.exitcase)
-            last_exception_object = annmodel.SomeType()
-            if isinstance(last_exception_var, Constant):
-                last_exception_object.const = last_exception_var.value
-            last_exception_object.is_type_of = [last_exc_value_var]
+        if (isinstance(link.exitcase, (types.ClassType, type)) and
+                issubclass(link.exitcase, BaseException)):
+            assert v_last_exc_type and v_last_exc_value
+            s_last_exc_value = self.bookkeeper.valueoftype(link.exitcase)
+            s_last_exc_type = annmodel.SomeType()
+            if isinstance(v_last_exc_type, Constant):
+                s_last_exc_type.const = v_last_exc_type.value
+            s_last_exc_type.is_type_of = [v_last_exc_value]
 
-            if isinstance(last_exception_var, Variable):
-                self.setbinding(last_exception_var, last_exception_object)
-            if isinstance(last_exc_value_var, Variable):
-                self.setbinding(last_exc_value_var, last_exc_value_object)
+            if isinstance(v_last_exc_type, Variable):
+                self.setbinding(v_last_exc_type, s_last_exc_type)
+            if isinstance(v_last_exc_value, Variable):
+                self.setbinding(v_last_exc_value, s_last_exc_value)
 
-            last_exception_object = annmodel.SomeType()
-            if isinstance(last_exception_var, Constant):
-                last_exception_object.const = last_exception_var.value
-            #if link.exitcase is Exception:
-            #    last_exc_value_object = annmodel.SomeObject()
-            #else:
+            s_last_exc_type = annmodel.SomeType()
+            if isinstance(v_last_exc_type, Constant):
+                s_last_exc_type.const = v_last_exc_type.value
             last_exc_value_vars = []
             in_except_block = True
 
         ignore_link = False
-        cells = []
+        inputs_s = []
         renaming = {}
-        for a, v in zip(link.args, link.target.inputargs):
-            renaming.setdefault(a, []).append(v)
-        for a, v in zip(link.args, link.target.inputargs):
-            if a == last_exception_var:
+        for v_out, v_input in zip(link.args, link.target.inputargs):
+            renaming.setdefault(v_out, []).append(v_input)
+        for v_out, v_input in zip(link.args, link.target.inputargs):
+            if v_out == v_last_exc_type:
                 assert in_except_block
-                cells.append(last_exception_object)
-            elif a == last_exc_value_var:
+                inputs_s.append(s_last_exc_type)
+            elif v_out == v_last_exc_value:
                 assert in_except_block
-                cells.append(last_exc_value_object)
-                last_exc_value_vars.append(v)
+                inputs_s.append(s_last_exc_value)
+                last_exc_value_vars.append(v_input)
             else:
-                cell = self.binding(a)
-                if (link.exitcase, a) in knowntypedata:
-                    knownvarvalue = knowntypedata[(link.exitcase, a)]
-                    cell = pair(cell, knownvarvalue).improve()
+                s_out = self.annotation(v_out)
+                if (link.exitcase, v_out) in knowntypedata:
+                    knownvarvalue = knowntypedata[(link.exitcase, v_out)]
+                    s_out = pair(s_out, knownvarvalue).improve()
                     # ignore links that try to pass impossible values
-                    if cell == annmodel.s_ImpossibleValue:
+                    if s_out == annmodel.s_ImpossibleValue:
                         ignore_link = True
 
-                if hasattr(cell,'is_type_of'):
+                if hasattr(s_out,'is_type_of'):
                     renamed_is_type_of = []
-                    for v in cell.is_type_of:
-                        new_vs = renaming.get(v,[])
+                    for v in s_out.is_type_of:
+                        new_vs = renaming.get(v, [])
                         renamed_is_type_of += new_vs
-                    assert cell.knowntype is type
+                    assert s_out.knowntype is type
                     newcell = annmodel.SomeType()
-                    if cell.is_constant():
-                        newcell.const = cell.const
-                    cell = newcell
-                    cell.is_type_of = renamed_is_type_of
+                    if s_out.is_constant():
+                        newcell.const = s_out.const
+                    s_out = newcell
+                    s_out.is_type_of = renamed_is_type_of
 
-                if hasattr(cell, 'knowntypedata'):
+                if hasattr(s_out, 'knowntypedata'):
                     renamed_knowntypedata = {}
-                    for (value, v), s in cell.knowntypedata.items():
+                    for (value, v), s in s_out.knowntypedata.items():
                         new_vs = renaming.get(v, [])
                         for new_v in new_vs:
                             renamed_knowntypedata[value, new_v] = s
-                    assert isinstance(cell, annmodel.SomeBool)
+                    assert isinstance(s_out, annmodel.SomeBool)
                     newcell = annmodel.SomeBool()
-                    if cell.is_constant():
-                        newcell.const = cell.const
-                    cell = newcell
-                    cell.set_knowntypedata(renamed_knowntypedata)
+                    if s_out.is_constant():
+                        newcell.const = s_out.const
+                    s_out = newcell
+                    s_out.set_knowntypedata(renamed_knowntypedata)
 
-                cells.append(cell)
+                inputs_s.append(s_out)
         if ignore_link:
             return
 
         if in_except_block:
-            last_exception_object.is_type_of = last_exc_value_vars
+            s_last_exc_type.is_type_of = last_exc_value_vars
         self.links_followed[link] = True
-        self.addpendingblock(graph, link.target, cells)
+        self.addpendingblock(graph, link.target, inputs_s)
 
     #___ creating the annotations based on operations ______
 

From noreply at buildbot.pypy.org  Mon Nov 10 13:29:55 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 10 Nov 2014 13:29:55 +0100 (CET)
Subject: [pypy-commit] pypy default: If we don't enable the debug prints,
 don't call get_location_str().
Message-ID: <20141110122955.C22891C33C0@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r74425:e9b9afa6edb7
Date: 2014-11-10 13:29 +0100
http://bitbucket.org/pypy/pypy/changeset/e9b9afa6edb7/

Log:	If we don't enable the debug prints, don't call get_location_str().

diff --git a/rpython/jit/metainterp/pyjitpl.py b/rpython/jit/metainterp/pyjitpl.py
--- a/rpython/jit/metainterp/pyjitpl.py
+++ b/rpython/jit/metainterp/pyjitpl.py
@@ -14,7 +14,8 @@
 from rpython.jit.metainterp.optimizeopt.util import args_dict
 from rpython.jit.metainterp.resoperation import rop
 from rpython.rlib import nonconst, rstack
-from rpython.rlib.debug import debug_start, debug_stop, debug_print, make_sure_not_resized
+from rpython.rlib.debug import debug_start, debug_stop, debug_print
+from rpython.rlib.debug import have_debug_prints, make_sure_not_resized
 from rpython.rlib.jit import Counters
 from rpython.rlib.objectmodel import we_are_translated, specialize
 from rpython.rlib.unroll import unrolling_iterable
@@ -1122,8 +1123,9 @@
 
     def debug_merge_point(self, jitdriver_sd, jd_index, portal_call_depth, current_call_id, greenkey):
         # debugging: produce a DEBUG_MERGE_POINT operation
-        loc = jitdriver_sd.warmstate.get_location_str(greenkey)
-        debug_print(loc)
+        if have_debug_prints():
+            loc = jitdriver_sd.warmstate.get_location_str(greenkey)
+            debug_print(loc)
         args = [ConstInt(jd_index), ConstInt(portal_call_depth), ConstInt(current_call_id)] + greenkey
         self.metainterp.history.record(rop.DEBUG_MERGE_POINT, args, None)
 

From noreply at buildbot.pypy.org  Mon Nov 10 14:56:47 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 10 Nov 2014 14:56:47 +0100 (CET)
Subject: [pypy-commit] pypy stmgc-c7: Start a built-in module "_stm".
Message-ID: <20141110135647.08E3D1C07BB@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: stmgc-c7
Changeset: r74426:a94a7b675578
Date: 2014-11-10 14:56 +0100
http://bitbucket.org/pypy/pypy/changeset/a94a7b675578/

Log:	Start a built-in module "_stm".

diff --git a/pypy/interpreter/executioncontext.py b/pypy/interpreter/executioncontext.py
--- a/pypy/interpreter/executioncontext.py
+++ b/pypy/interpreter/executioncontext.py
@@ -34,7 +34,7 @@
         self.w_profilefuncarg = None
         #
         if self.space.config.translation.stm:
-            from pypy.module.thread.stm import initialize_execution_context
+            from pypy.module._stm.ec import initialize_execution_context
             initialize_execution_context(self)
 
     def gettopframe(self):
diff --git a/pypy/module/_stm/__init__.py b/pypy/module/_stm/__init__.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/_stm/__init__.py
@@ -0,0 +1,11 @@
+
+# Package initialisation
+from pypy.interpreter.mixedmodule import MixedModule
+
+class Module(MixedModule):
+    appleveldefs = {
+    }
+
+    interpleveldefs = {
+        'local': 'local.STMLocal',
+    }
diff --git a/pypy/module/_stm/ec.py b/pypy/module/_stm/ec.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/_stm/ec.py
@@ -0,0 +1,29 @@
+"""
+Attach extra STM-only attributes to the ExecutionContext.
+"""
+
+from pypy.interpreter.executioncontext import ExecutionContext
+from pypy.interpreter.gateway import W_Root
+
+
+class FakeWeakKeyDictionary:
+    # Only used if we don't have weakrefs.
+    # Then thread._local instances will leak, but too bad.
+    def __init__(self):
+        self.d = {}
+    def get(self, key):
+        return self.d.get(key, None)
+    def set(self, key, value):
+        self.d[key] = value
+
+def initialize_execution_context(ec):
+    """Called from ExecutionContext.__init__()."""
+    if ec.space.config.translation.rweakref:
+        from rpython.rlib import rweakref
+        from pypy.module._stm.local import STMLocal
+        ec._thread_local_dicts = rweakref.RWeakKeyDictionary(STMLocal, W_Root)
+    else:
+        ec._thread_local_dicts = FakeWeakKeyDictionary()
+    if ec.space.config.objspace.std.withmethodcache:
+        from pypy.objspace.std.typeobject import MethodCache
+        ec._methodcache = MethodCache(ec.space)
diff --git a/pypy/module/_stm/local.py b/pypy/module/_stm/local.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/_stm/local.py
@@ -0,0 +1,75 @@
+"""
+The '_stm.local' class, used for 'thread._local' with STM.
+"""
+
+from pypy.interpreter.gateway import W_Root, interp2app
+from pypy.interpreter.typedef import TypeDef, GetSetProperty, descr_get_dict
+from rpython.rlib import jit
+from rpython.rlib.objectmodel import we_are_translated
+
+
+def _fill_untranslated(ec):
+    if not we_are_translated() and not hasattr(ec, '_thread_local_dicts'):
+        from pypy.module._stm.ec import initialize_execution_context
+        initialize_execution_context(ec)
+
+
+class STMLocal(W_Root):
+    """Thread-local data"""
+
+    @jit.dont_look_inside
+    def __init__(self, space, initargs):
+        self.space = space
+        self.initargs = initargs
+        # The app-level __init__() will be called by the general
+        # instance-creation logic.  It causes getdict() to be
+        # immediately called.  If we don't prepare and set a w_dict
+        # for the current thread, then this would in cause getdict()
+        # to call __init__() a second time.
+        ec = space.getexecutioncontext()
+        _fill_untranslated(ec)
+        w_dict = space.newdict(instance=True)
+        ec._thread_local_dicts.set(self, w_dict)
+
+    @jit.dont_look_inside
+    def create_new_dict(self, ec):
+        # create a new dict for this thread
+        space = self.space
+        w_dict = space.newdict(instance=True)
+        ec._thread_local_dicts.set(self, w_dict)
+        # call __init__
+        try:
+            w_self = space.wrap(self)
+            w_type = space.type(w_self)
+            w_init = space.getattr(w_type, space.wrap("__init__"))
+            space.call_obj_args(w_init, w_self, self.initargs)
+        except:
+            # failed, forget w_dict and propagate the exception
+            ec._thread_local_dicts.set(self, None)
+            raise
+        # ready
+        return w_dict
+
+    def getdict(self, space):
+        ec = space.getexecutioncontext()
+        _fill_untranslated(ec)
+        w_dict = ec._thread_local_dicts.get(self)
+        if w_dict is None:
+            w_dict = self.create_new_dict(ec)
+        return w_dict
+
+    def descr_local__new__(space, w_subtype, __args__):
+        local = space.allocate_instance(STMLocal, w_subtype)
+        STMLocal.__init__(local, space, __args__)
+        return space.wrap(local)
+
+    def descr_local__init__(self, space):
+        # No arguments allowed
+        pass
+
+STMLocal.typedef = TypeDef("_stm.local",
+                     __doc__ = "Thread-local data",
+                     __new__ = interp2app(STMLocal.descr_local__new__.im_func),
+                     __init__ = interp2app(STMLocal.descr_local__init__),
+                     __dict__ = GetSetProperty(descr_get_dict, cls=STMLocal),
+                     )
diff --git a/pypy/module/_stm/lock.py b/pypy/module/_stm/lock.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/_stm/lock.py
@@ -0,0 +1,21 @@
+from pypy.module.thread.error import wrap_thread_error
+
+
+class STMLock(rthread.Lock):
+    def __init__(self, space, ll_lock):
+        rthread.Lock.__init__(self, ll_lock)
+        self.space = space
+
+    def acquire(self, flag):
+        if rstm.is_atomic():
+            acquired = rthread.Lock.acquire(self, False)
+            if flag and not acquired:
+                raise wrap_thread_error(self.space,
+                    "deadlock: an atomic transaction tries to acquire "
+                    "a lock that is already acquired.  See pypy/doc/stm.rst.")
+        else:
+            acquired = rthread.Lock.acquire(self, flag)
+        return acquired
+
+def allocate_stm_lock(space):
+    return STMLock(space, rthread.allocate_ll_lock())
diff --git a/pypy/module/thread/test/__init__.py b/pypy/module/_stm/test/__init__.py
copy from pypy/module/thread/test/__init__.py
copy to pypy/module/_stm/test/__init__.py
diff --git a/pypy/module/thread/test/test_stm.py b/pypy/module/_stm/test/test_local.py
rename from pypy/module/thread/test/test_stm.py
rename to pypy/module/_stm/test/test_local.py
--- a/pypy/module/thread/test/test_stm.py
+++ b/pypy/module/_stm/test/test_local.py
@@ -2,10 +2,12 @@
 
 
 class AppTestSTMLocal(test_local.AppTestLocal):
+    spaceconfig = test_local.AppTestLocal.spaceconfig.copy()
+    spaceconfig['usemodules'] += ('_stm',)
 
     def setup_class(cls):
         test_local.AppTestLocal.setup_class.im_func(cls)
         cls.w__local = cls.space.appexec([], """():
-            import thread
-            return thread._untranslated_stmlocal
+            import _stm
+            return _stm.local
         """)
diff --git a/pypy/module/_stm/threadlocals.py b/pypy/module/_stm/threadlocals.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/_stm/threadlocals.py
@@ -0,0 +1,42 @@
+"""
+An STM-friendly subclass of OSThreadLocals.
+"""
+
+from pypy.module.thread.threadlocals import OSThreadLocals
+from rpython.rlib import rstm
+from rpython.rlib.objectmodel import invoke_around_extcall
+
+
+class STMThreadLocals(OSThreadLocals):
+    threads_running = False
+    _immutable_fields_ = ['threads_running?']
+
+    def initialize(self, space):
+        """NOT_RPYTHON: set up a mechanism to send to the C code the value
+        set by space.actionflag.setcheckinterval()."""
+        #
+        def setcheckinterval_callback():
+            self.configure_transaction_length(space)
+        #
+        assert space.actionflag.setcheckinterval_callback is None
+        space.actionflag.setcheckinterval_callback = setcheckinterval_callback
+
+    # XXX?
+    #def getallvalues(self):
+    #    raise ValueError
+
+    def setup_threads(self, space):
+        if not self.threads_running:
+            # invalidate quasi-immutable if we have threads:
+            self.threads_running = True
+        self.configure_transaction_length(space)
+        invoke_around_extcall(rstm.before_external_call,
+                              rstm.after_external_call,
+                              rstm.enter_callback_call,
+                              rstm.leave_callback_call)
+
+    def configure_transaction_length(self, space):
+        if self.threads_running:
+            interval = space.actionflag.getcheckinterval()
+            rstm.set_transaction_length(interval / 10000.0)
+
diff --git a/pypy/module/thread/__init__.py b/pypy/module/thread/__init__.py
--- a/pypy/module/thread/__init__.py
+++ b/pypy/module/thread/__init__.py
@@ -45,5 +45,3 @@
             self.extra_interpdef('_local', 'stm.STMLocal')
         else:
             self.extra_interpdef('_local', 'os_local.Local')
-            if not self.space.config.translating:
-                self.extra_interpdef('_untranslated_stmlocal', 'stm.STMLocal')
diff --git a/pypy/module/thread/stm.py b/pypy/module/thread/stm.py
--- a/pypy/module/thread/stm.py
+++ b/pypy/module/thread/stm.py
@@ -1,161 +1,8 @@
 """
-Software Transactional Memory emulation of the GIL.
-
-XXX this module may contain near-duplicated code from the non-STM variants
+Redirect some classes from pypy.module._stm.
 """
 
-from pypy.module.thread.threadlocals import OSThreadLocals
-from pypy.module.thread.error import wrap_thread_error
-from pypy.interpreter.executioncontext import ExecutionContext
-from pypy.interpreter.gateway import W_Root, interp2app
-from pypy.interpreter.typedef import TypeDef, GetSetProperty, descr_get_dict
-from rpython.rlib import rthread
-from rpython.rlib import rstm
-from rpython.rlib import jit
-from rpython.rlib.objectmodel import invoke_around_extcall, we_are_translated
+from pypy.module._stm import threadlocals, local
 
-
-class FakeWeakKeyDictionary:
-    # Only used if we don't have weakrefs.
-    # Then thread._local instances will leak, but too bad.
-    def __init__(self):
-        self.d = {}
-    def get(self, key):
-        return self.d.get(key, None)
-    def set(self, key, value):
-        self.d[key] = value
-
-
-def initialize_execution_context(ec):
-    """Called from ExecutionContext.__init__()."""
-    if ec.space.config.translation.rweakref:
-        from rpython.rlib import rweakref
-        ec._thread_local_dicts = rweakref.RWeakKeyDictionary(STMLocal, W_Root)
-    else:
-        ec._thread_local_dicts = FakeWeakKeyDictionary()
-    if ec.space.config.objspace.std.withmethodcache:
-        from pypy.objspace.std.typeobject import MethodCache
-        ec._methodcache = MethodCache(ec.space)
-
-def _fill_untranslated(ec):
-    if not we_are_translated() and not hasattr(ec, '_thread_local_dicts'):
-        initialize_execution_context(ec)
-
-
-class STMThreadLocals(OSThreadLocals):
-    threads_running = False
-    _immutable_fields_ = ['threads_running?']
-
-    def initialize(self, space):
-        """NOT_RPYTHON: set up a mechanism to send to the C code the value
-        set by space.actionflag.setcheckinterval()."""
-        #
-        def setcheckinterval_callback():
-            self.configure_transaction_length(space)
-        #
-        assert space.actionflag.setcheckinterval_callback is None
-        space.actionflag.setcheckinterval_callback = setcheckinterval_callback
-
-    # XXX?
-    #def getallvalues(self):
-    #    raise ValueError
-
-    def setup_threads(self, space):
-        if not self.threads_running:
-            # invalidate quasi-immutable if we have threads:
-            self.threads_running = True
-        self.configure_transaction_length(space)
-        invoke_around_extcall(rstm.before_external_call,
-                              rstm.after_external_call,
-                              rstm.enter_callback_call,
-                              rstm.leave_callback_call)
-
-    def configure_transaction_length(self, space):
-        if self.threads_running:
-            interval = space.actionflag.getcheckinterval()
-            rstm.set_transaction_length(interval / 10000.0)
-
-# ____________________________________________________________
-
-
-class STMLock(rthread.Lock):
-    def __init__(self, space, ll_lock):
-        rthread.Lock.__init__(self, ll_lock)
-        self.space = space
-
-    def acquire(self, flag):
-        if rstm.is_atomic():
-            acquired = rthread.Lock.acquire(self, False)
-            if flag and not acquired:
-                raise wrap_thread_error(self.space,
-                    "deadlock: an atomic transaction tries to acquire "
-                    "a lock that is already acquired.  See pypy/doc/stm.rst.")
-        else:
-            acquired = rthread.Lock.acquire(self, flag)
-        return acquired
-
-def allocate_stm_lock(space):
-    return STMLock(space, rthread.allocate_ll_lock())
-
-# ____________________________________________________________
-
-
-class STMLocal(W_Root):
-    """Thread-local data"""
-
-    @jit.dont_look_inside
-    def __init__(self, space, initargs):
-        self.space = space
-        self.initargs = initargs
-        # The app-level __init__() will be called by the general
-        # instance-creation logic.  It causes getdict() to be
-        # immediately called.  If we don't prepare and set a w_dict
-        # for the current thread, then this would in cause getdict()
-        # to call __init__() a second time.
-        ec = space.getexecutioncontext()
-        _fill_untranslated(ec)
-        w_dict = space.newdict(instance=True)
-        ec._thread_local_dicts.set(self, w_dict)
-
-    @jit.dont_look_inside
-    def create_new_dict(self, ec):
-        # create a new dict for this thread
-        space = self.space
-        w_dict = space.newdict(instance=True)
-        ec._thread_local_dicts.set(self, w_dict)
-        # call __init__
-        try:
-            w_self = space.wrap(self)
-            w_type = space.type(w_self)
-            w_init = space.getattr(w_type, space.wrap("__init__"))
-            space.call_obj_args(w_init, w_self, self.initargs)
-        except:
-            # failed, forget w_dict and propagate the exception
-            ec._thread_local_dicts.set(self, None)
-            raise
-        # ready
-        return w_dict
-
-    def getdict(self, space):
-        ec = space.getexecutioncontext()
-        _fill_untranslated(ec)
-        w_dict = ec._thread_local_dicts.get(self)
-        if w_dict is None:
-            w_dict = self.create_new_dict(ec)
-        return w_dict
-
-    def descr_local__new__(space, w_subtype, __args__):
-        local = space.allocate_instance(STMLocal, w_subtype)
-        STMLocal.__init__(local, space, __args__)
-        return space.wrap(local)
-
-    def descr_local__init__(self, space):
-        # No arguments allowed
-        pass
-
-STMLocal.typedef = TypeDef("thread._local",
-                     __doc__ = "Thread-local data",
-                     __new__ = interp2app(STMLocal.descr_local__new__.im_func),
-                     __init__ = interp2app(STMLocal.descr_local__init__),
-                     __dict__ = GetSetProperty(descr_get_dict, cls=STMLocal),
-                     )
+STMThreadLocals = threadlocals.STMThreadLocals
+STMLocal = local.STMLocal

From noreply at buildbot.pypy.org  Mon Nov 10 15:09:57 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 10 Nov 2014 15:09:57 +0100 (CET)
Subject: [pypy-commit] stmgc hashtable: Export stm_hashtable_lookup() too
Message-ID: <20141110140957.8D15D1C07BB@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: hashtable
Changeset: r1502:194265597fad
Date: 2014-11-10 15:09 +0100
http://bitbucket.org/pypy/stmgc/changeset/194265597fad/

Log:	Export stm_hashtable_lookup() too

diff --git a/c7/stm/hashtable.c b/c7/stm/hashtable.c
--- a/c7/stm/hashtable.c
+++ b/c7/stm/hashtable.c
@@ -25,8 +25,6 @@
 */
 
 
-typedef TLPREFIX struct stm_hashtable_entry_s stm_hashtable_entry_t;
-
 uint32_t stm_hashtable_entry_userdata;
 
 
@@ -170,9 +168,9 @@
     VOLATILE_HASHTABLE(hashtable)->table = biggertable;
 }
 
-static stm_hashtable_entry_t *_stm_hashtable_lookup(object_t *hashtableobj,
-                                                    stm_hashtable_t *hashtable,
-                                                    uintptr_t index)
+stm_hashtable_entry_t *stm_hashtable_lookup(object_t *hashtableobj,
+                                            stm_hashtable_t *hashtable,
+                                            uintptr_t index)
 {
     stm_hashtable_table_t *table;
     uintptr_t mask;
@@ -305,22 +303,20 @@
     }
 }
 
-object_t *stm_hashtable_read(object_t *hashtableobj, stm_hashtable_t *hashtable,
-                             uintptr_t index)
+object_t *stm_hashtable_read(object_t *hobj, stm_hashtable_t *hashtable,
+                             uintptr_t key)
 {
-    stm_hashtable_entry_t *e = _stm_hashtable_lookup(hashtableobj, hashtable,
-                                                     index);
+    stm_hashtable_entry_t *e = stm_hashtable_lookup(hobj, hashtable, key);
     stm_read((object_t *)e);
     return e->object;
 }
 
-void stm_hashtable_write(object_t *hashtableobj, stm_hashtable_t *hashtable,
-                         uintptr_t index, object_t *nvalue,
+void stm_hashtable_write(object_t *hobj, stm_hashtable_t *hashtable,
+                         uintptr_t key, object_t *nvalue,
                          stm_thread_local_t *tl)
 {
     STM_PUSH_ROOT(*tl, nvalue);
-    stm_hashtable_entry_t *e = _stm_hashtable_lookup(hashtableobj, hashtable,
-                                                     index);
+    stm_hashtable_entry_t *e = stm_hashtable_lookup(hobj, hashtable, key);
     stm_write((object_t *)e);
     STM_POP_ROOT(*tl, nvalue);
     e->object = nvalue;
diff --git a/c7/stmgc.h b/c7/stmgc.h
--- a/c7/stmgc.h
+++ b/c7/stmgc.h
@@ -534,8 +534,12 @@
    If you want to embed the hashtable inside an 'object_t' you
    probably need a light finalizer to do the freeing. */
 typedef struct stm_hashtable_s stm_hashtable_t;
+typedef TLPREFIX struct stm_hashtable_entry_s stm_hashtable_entry_t;
+
 stm_hashtable_t *stm_hashtable_create(void);
 void stm_hashtable_free(stm_hashtable_t *);
+stm_hashtable_entry_t *stm_hashtable_lookup(object_t *, stm_hashtable_t *,
+                                            uintptr_t key);
 object_t *stm_hashtable_read(object_t *, stm_hashtable_t *, uintptr_t key);
 void stm_hashtable_write(object_t *, stm_hashtable_t *, uintptr_t key,
                          object_t *nvalue, stm_thread_local_t *);

From noreply at buildbot.pypy.org  Mon Nov 10 16:15:15 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 10 Nov 2014 16:15:15 +0100 (CET)
Subject: [pypy-commit] pypy stmgc-c7: Add _stm.count(),
 which returns a new integer every time it is called.
Message-ID: <20141110151515.432861C07BB@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: stmgc-c7
Changeset: r74427:3dc9224b99dc
Date: 2014-11-10 16:13 +0100
http://bitbucket.org/pypy/pypy/changeset/3dc9224b99dc/

Log:	Add _stm.count(), which returns a new integer every time it is
	called.

diff --git a/pypy/module/_stm/__init__.py b/pypy/module/_stm/__init__.py
--- a/pypy/module/_stm/__init__.py
+++ b/pypy/module/_stm/__init__.py
@@ -8,4 +8,5 @@
 
     interpleveldefs = {
         'local': 'local.STMLocal',
+        'count': 'count.count',
     }
diff --git a/pypy/module/_stm/count.py b/pypy/module/_stm/count.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/_stm/count.py
@@ -0,0 +1,12 @@
+"""
+_stm.count()
+"""
+
+from rpython.rlib import rstm
+
+
+def count(space):
+    """Return a new integer every time it is called,
+without generating conflicts."""
+    count = rstm.stm_count()
+    return space.wrap(count)
diff --git a/pypy/module/_stm/test/test_count.py b/pypy/module/_stm/test/test_count.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/_stm/test/test_count.py
@@ -0,0 +1,10 @@
+
+
+class AppTestCount:
+    spaceconfig = dict(usemodules=['_stm'])
+
+    def test_count(self):
+        import _stm
+        x = _stm.count()
+        y = _stm.count()
+        assert y == x + 1
diff --git a/rpython/rlib/rstm.py b/rpython/rlib/rstm.py
--- a/rpython/rlib/rstm.py
+++ b/rpython/rlib/rstm.py
@@ -156,7 +156,7 @@
 def pop_marker():
     llop.stm_pop_marker(lltype.Void)
 
-def stm_count():     # for tests
+def stm_count():
     return llop.stm_count(lltype.Signed)
 
 # ____________________________________________________________
diff --git a/rpython/rtyper/lltypesystem/lloperation.py b/rpython/rtyper/lltypesystem/lloperation.py
--- a/rpython/rtyper/lltypesystem/lloperation.py
+++ b/rpython/rtyper/lltypesystem/lloperation.py
@@ -458,7 +458,7 @@
     'stm_expand_marker':      LLOp(),
     'stm_setup_expand_marker_for_pypy': LLOp(),
 
-    'stm_count':                 LLOp(),
+    'stm_count':                 LLOp(canrun=True),
     'stm_really_force_cast_ptr': LLOp(),
 
     # __________ address operations __________
diff --git a/rpython/rtyper/lltypesystem/opimpl.py b/rpython/rtyper/lltypesystem/opimpl.py
--- a/rpython/rtyper/lltypesystem/opimpl.py
+++ b/rpython/rtyper/lltypesystem/opimpl.py
@@ -724,6 +724,13 @@
 def op_stm_hint_commit_soon():
     pass
 
+def op_stm_count():
+    global _stm_counter
+    x = _stm_counter
+    _stm_counter = x + 1
+    return x
+_stm_counter = 0
+
 
 # ____________________________________________________________
 
diff --git a/rpython/translator/stm/src_stm/stmgcintf.c b/rpython/translator/stm/src_stm/stmgcintf.c
--- a/rpython/translator/stm/src_stm/stmgcintf.c
+++ b/rpython/translator/stm/src_stm/stmgcintf.c
@@ -224,7 +224,6 @@
 
 long _pypy_stm_count(void)
 {
-    /* for tests */
     static long value = 0;
-    return value++;
+    return __sync_fetch_and_add(&value, 1);
 }

From noreply at buildbot.pypy.org  Mon Nov 10 16:15:16 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 10 Nov 2014 16:15:16 +0100 (CET)
Subject: [pypy-commit] pypy stmgc-c7: import stmgc/194265597fad [hashtable]
Message-ID: <20141110151516.70D261C07BB@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: stmgc-c7
Changeset: r74428:f3d738ba20db
Date: 2014-11-10 16:14 +0100
http://bitbucket.org/pypy/pypy/changeset/f3d738ba20db/

Log:	import stmgc/194265597fad [hashtable]

diff --git a/rpython/translator/stm/src_stm/revision b/rpython/translator/stm/src_stm/revision
--- a/rpython/translator/stm/src_stm/revision
+++ b/rpython/translator/stm/src_stm/revision
@@ -1,1 +1,1 @@
-b27bcdd6bc87
+194265597fad
diff --git a/rpython/translator/stm/src_stm/stm/hashtable.c b/rpython/translator/stm/src_stm/stm/hashtable.c
new file mode 100644
--- /dev/null
+++ b/rpython/translator/stm/src_stm/stm/hashtable.c
@@ -0,0 +1,380 @@
+/* Imported by rpython/translator/stm/import_stmgc.py */
+/*
+Design of stmgc's "hashtable" objects
+=====================================
+
+A "hashtable" is theoretically a lazily-filled array of objects of
+length 2**64.  Initially it is full of NULLs.  It's obviously
+implemented as a dictionary in which NULL objects are not needed.
+
+The only operations on a hashtable are reading or writing an object at
+a given index.
+
+There are two markers for every index (a read and a write marker).
+This is unlike regular arrays, which have only two markers in total.
+
+
+Implementation
+--------------
+
+First idea: have the hashtable in raw memory, pointing to "entry"
+objects.  The entry objects themselves point to the user-specified
+objects, and they have the read/write markers.  Every entry object
+itself, once created, stays around.  It is only removed by the next
+major GC if it points to NULL and its read/write markers are not set
+in any currently-running transaction.
+*/
+
+
+uint32_t stm_hashtable_entry_userdata;
+
+
+#define INITIAL_HASHTABLE_SIZE   8
+#define PERTURB_SHIFT            5
+#define RESIZING_LOCK            0
+
+typedef struct {
+    uintptr_t mask;
+
+    /* 'resize_counter' start at an odd value, and is decremented (by
+       6) for every new item put in 'items'.  When it crosses 0, we
+       instead allocate a bigger table and change 'resize_counter' to
+       be a regular pointer to it (which is then even).  The whole
+       structure is immutable then.
+
+       The field 'resize_counter' also works as a write lock: changes
+       go via the intermediate value RESIZING_LOCK (0).
+    */
+    uintptr_t resize_counter;
+
+    stm_hashtable_entry_t *items[INITIAL_HASHTABLE_SIZE];
+} stm_hashtable_table_t;
+
+#define IS_EVEN(p) (((p) & 1) == 0)
+
+struct stm_hashtable_s {
+    stm_hashtable_table_t *table;
+    stm_hashtable_table_t initial_table;
+    uint64_t additions;
+};
+
+
+static inline void init_table(stm_hashtable_table_t *table, uintptr_t itemcount)
+{
+    table->mask = itemcount - 1;
+    table->resize_counter = itemcount * 4 + 1;
+    memset(table->items, 0, itemcount * sizeof(stm_hashtable_entry_t *));
+}
+
+stm_hashtable_t *stm_hashtable_create(void)
+{
+    stm_hashtable_t *hashtable = malloc(sizeof(stm_hashtable_t));
+    assert(hashtable);
+    hashtable->table = &hashtable->initial_table;
+    hashtable->additions = 0;
+    init_table(&hashtable->initial_table, INITIAL_HASHTABLE_SIZE);
+    return hashtable;
+}
+
+void stm_hashtable_free(stm_hashtable_t *hashtable)
+{
+    uintptr_t rc = hashtable->initial_table.resize_counter;
+    free(hashtable);
+    while (IS_EVEN(rc)) {
+        assert(rc != RESIZING_LOCK);
+
+        stm_hashtable_table_t *table = (stm_hashtable_table_t *)rc;
+        rc = table->resize_counter;
+        free(table);
+    }
+}
+
+static bool _stm_was_read_by_anybody(object_t *obj)
+{
+    long i;
+    for (i = 1; i <= NB_SEGMENTS; i++) {
+        char *remote_base = get_segment_base(i);
+        uint8_t remote_version = get_segment(i)->transaction_read_version;
+        if (was_read_remote(remote_base, obj, remote_version))
+            return true;
+    }
+    return false;
+}
+
+#define VOLATILE_HASHTABLE(p)    ((volatile stm_hashtable_t *)(p))
+#define VOLATILE_TABLE(p)  ((volatile stm_hashtable_table_t *)(p))
+
+static void _insert_clean(stm_hashtable_table_t *table,
+                          stm_hashtable_entry_t *entry)
+{
+    uintptr_t mask = table->mask;
+    uintptr_t i = entry->index & mask;
+    if (table->items[i] == NULL) {
+        table->items[i] = entry;
+        return;
+    }
+
+    uintptr_t perturb = entry->index;
+    while (1) {
+        i = (i << 2) + i + perturb + 1;
+        i &= mask;
+        if (table->items[i] == NULL) {
+            table->items[i] = entry;
+            return;
+        }
+
+        perturb >>= PERTURB_SHIFT;
+    }
+}
+
+static void _stm_rehash_hashtable(stm_hashtable_t *hashtable,
+                                  uintptr_t biggercount,
+                                  bool remove_unread)
+{
+    dprintf(("rehash %p to %ld, remove_unread=%d\n",
+             hashtable, biggercount, (int)remove_unread));
+
+    size_t size = (offsetof(stm_hashtable_table_t, items)
+                   + biggercount * sizeof(stm_hashtable_entry_t *));
+    stm_hashtable_table_t *biggertable = malloc(size);
+    assert(biggertable);   // XXX
+
+    stm_hashtable_table_t *table = hashtable->table;
+    table->resize_counter = (uintptr_t)biggertable;
+    /* ^^^ this unlocks the table by writing a non-zero value to
+       table->resize_counter, but the new value is a pointer to the
+       new bigger table, so IS_EVEN() is still true */
+
+    init_table(biggertable, biggercount);
+
+    uintptr_t j, mask = table->mask;
+    uintptr_t rc = biggertable->resize_counter;
+    for (j = 0; j <= mask; j++) {
+        stm_hashtable_entry_t *entry = table->items[j];
+        if (entry == NULL)
+            continue;
+        if (remove_unread) {
+            if (entry->object == NULL &&
+                   !_stm_was_read_by_anybody((object_t *)entry))
+                continue;
+        }
+        _insert_clean(biggertable, entry);
+        rc -= 6;
+    }
+    biggertable->resize_counter = rc;
+
+    write_fence();   /* make sure that 'biggertable' is valid here,
+                        and make sure 'table->resize_counter' is updated
+                        ('table' must be immutable from now on). */
+    VOLATILE_HASHTABLE(hashtable)->table = biggertable;
+}
+
+stm_hashtable_entry_t *stm_hashtable_lookup(object_t *hashtableobj,
+                                            stm_hashtable_t *hashtable,
+                                            uintptr_t index)
+{
+    stm_hashtable_table_t *table;
+    uintptr_t mask;
+    uintptr_t i;
+    stm_hashtable_entry_t *entry;
+
+ restart:
+    /* classical dict lookup logic */
+    table = VOLATILE_HASHTABLE(hashtable)->table;
+    mask = table->mask;      /* read-only field */
+    i = index & mask;
+    entry = VOLATILE_TABLE(table)->items[i];
+    if (entry != NULL) {
+        if (entry->index == index)
+            return entry;           /* found at the first try */
+
+        uintptr_t perturb = index;
+        while (1) {
+            i = (i << 2) + i + perturb + 1;
+            i &= mask;
+            entry = VOLATILE_TABLE(table)->items[i];
+            if (entry != NULL) {
+                if (entry->index == index)
+                    return entry;    /* found */
+            }
+            else
+                break;
+            perturb >>= PERTURB_SHIFT;
+        }
+    }
+    /* here, we didn't find the 'entry' with the correct index. */
+
+    uintptr_t rc = VOLATILE_TABLE(table)->resize_counter;
+
+    /* if rc is RESIZING_LOCK (which is 0, so even), a concurrent thread
+       is writing to the hashtable.  Or, if rc is another even number, it is
+       actually a pointer to the next version of the table, installed
+       just now.  In both cases, this thread must simply spin loop.
+    */
+    if (IS_EVEN(rc)) {
+        spin_loop();
+        goto restart;
+    }
+    /* in the other cases, we need to grab the RESIZING_LOCK.
+     */
+    if (!__sync_bool_compare_and_swap(&table->resize_counter,
+                                      rc, RESIZING_LOCK)) {
+        goto restart;
+    }
+    /* we now have the lock.  The only table with a non-even value of
+       'resize_counter' should be the last one in the chain, so if we
+       succeeded in locking it, check this. */
+    assert(table == hashtable->table);
+
+    /* Check that 'table->items[i]' is still NULL,
+       i.e. hasn't been populated under our feet.
+    */
+    if (table->items[i] != NULL) {
+        table->resize_counter = rc;    /* unlock */
+        goto restart;
+    }
+    /* if rc is greater than 6, there is enough room for a new
+       item in the current table.
+    */
+    if (rc > 6) {
+        /* we can only enter here once!  If we allocate stuff, we may
+           run the GC, and so 'hashtableobj' might move afterwards. */
+        if (_is_in_nursery(hashtableobj)) {
+            entry = (stm_hashtable_entry_t *)
+                stm_allocate(sizeof(stm_hashtable_entry_t));
+            entry->userdata = stm_hashtable_entry_userdata;
+            entry->index = index;
+            entry->object = NULL;
+        }
+        else {
+            /* for a non-nursery 'hashtableobj', we pretend that the
+               'entry' object we're about to return was already
+               existing all along, with NULL in all segments.  If the
+               caller of this function is going to modify the 'object'
+               field, it will call stm_write(entry) first, which will
+               correctly schedule 'entry' for write propagation.  We
+               do that even if 'hashtableobj' was created by the
+               running transaction: the new 'entry' object is created
+               as if it was older than the transaction.
+
+               Note the following difference: if 'hashtableobj' is
+               still in the nursery (case above), the 'entry' object
+               is also allocated from the nursery, and after a minor
+               collection it ages as an old-but-created-by-the-
+               current-transaction object.  We could try to emulate
+               this here, or to create young 'entry' objects, but
+               doing either of these would require careful
+               synchronization with other pieces of the code that may
+               change.
+            */
+            acquire_privatization_lock();
+            char *p = allocate_outside_nursery_large(
+                          sizeof(stm_hashtable_entry_t));
+            entry = (stm_hashtable_entry_t *)(p - stm_object_pages);
+
+            long j;
+            for (j = 0; j <= NB_SEGMENTS; j++) {
+                struct stm_hashtable_entry_s *e;
+                e = (struct stm_hashtable_entry_s *)
+                        REAL_ADDRESS(get_segment_base(j), entry);
+                e->header.stm_flags = GCFLAG_WRITE_BARRIER;
+                e->userdata = stm_hashtable_entry_userdata;
+                e->index = index;
+                e->object = NULL;
+            }
+            release_privatization_lock();
+        }
+        write_fence();     /* make sure 'entry' is fully initialized here */
+        table->items[i] = entry;
+        hashtable->additions += 1;
+        write_fence();     /* make sure 'table->items' is written here */
+        VOLATILE_TABLE(table)->resize_counter = rc - 6;    /* unlock */
+        return entry;
+    }
+    else {
+        /* if rc is smaller than 6, we must allocate a new bigger table.
+         */
+        uintptr_t biggercount = table->mask + 1;
+        if (biggercount < 50000)
+            biggercount *= 4;
+        else
+            biggercount *= 2;
+        _stm_rehash_hashtable(hashtable, biggercount, /*remove_unread=*/false);
+        goto restart;
+    }
+}
+
+object_t *stm_hashtable_read(object_t *hobj, stm_hashtable_t *hashtable,
+                             uintptr_t key)
+{
+    stm_hashtable_entry_t *e = stm_hashtable_lookup(hobj, hashtable, key);
+    stm_read((object_t *)e);
+    return e->object;
+}
+
+void stm_hashtable_write(object_t *hobj, stm_hashtable_t *hashtable,
+                         uintptr_t key, object_t *nvalue,
+                         stm_thread_local_t *tl)
+{
+    STM_PUSH_ROOT(*tl, nvalue);
+    stm_hashtable_entry_t *e = stm_hashtable_lookup(hobj, hashtable, key);
+    stm_write((object_t *)e);
+    STM_POP_ROOT(*tl, nvalue);
+    e->object = nvalue;
+}
+
+static void _stm_compact_hashtable(stm_hashtable_t *hashtable)
+{
+    stm_hashtable_table_t *table = hashtable->table;
+    assert(!IS_EVEN(table->resize_counter));
+
+    if (hashtable->additions * 4 > table->mask) {
+        hashtable->additions = 0;
+        uintptr_t initial_rc = (table->mask + 1) * 4 + 1;
+        uintptr_t num_entries_times_6 = initial_rc - table->resize_counter;
+        uintptr_t count = INITIAL_HASHTABLE_SIZE;
+        while (count * 4 < num_entries_times_6)
+            count *= 2;
+        /* sanity-check: 'num_entries_times_6 < initial_rc', and so 'count'
+           can never grow larger than the current table size. */
+        assert(count <= table->mask + 1);
+
+        _stm_rehash_hashtable(hashtable, count, /*remove_unread=*/true);
+    }
+
+    table = hashtable->table;
+    assert(!IS_EVEN(table->resize_counter));
+
+    if (table != &hashtable->initial_table) {
+        uintptr_t rc = hashtable->initial_table.resize_counter;
+        while (1) {
+            assert(IS_EVEN(rc));
+            assert(rc != RESIZING_LOCK);
+
+            stm_hashtable_table_t *old_table = (stm_hashtable_table_t *)rc;
+            if (old_table == table)
+                break;
+            rc = old_table->resize_counter;
+            free(old_table);
+        }
+        hashtable->initial_table.resize_counter = (uintptr_t)table;
+    }
+}
+
+void stm_hashtable_tracefn(stm_hashtable_t *hashtable, void trace(object_t **))
+{
+    if (trace == TRACE_FOR_MAJOR_COLLECTION)
+        _stm_compact_hashtable(hashtable);
+
+    stm_hashtable_table_t *table;
+    table = VOLATILE_HASHTABLE(hashtable)->table;
+
+    uintptr_t j, mask = table->mask;
+    for (j = 0; j <= mask; j++) {
+        stm_hashtable_entry_t *volatile *pentry;
+        pentry = &VOLATILE_TABLE(table)->items[j];
+        if (*pentry != NULL) {
+            trace((object_t **)pentry);
+        }
+    }
+}
diff --git a/rpython/translator/stm/src_stm/stmgc.c b/rpython/translator/stm/src_stm/stmgc.c
--- a/rpython/translator/stm/src_stm/stmgc.c
+++ b/rpython/translator/stm/src_stm/stmgc.c
@@ -17,7 +17,6 @@
 #include "stm/weakref.h"
 #include "stm/marker.h"
 #include "stm/finalizer.h"
-#include "stm/hashtable.h"
 
 #include "stm/misc.c"
 #include "stm/list.c"
diff --git a/rpython/translator/stm/src_stm/stmgc.h b/rpython/translator/stm/src_stm/stmgc.h
--- a/rpython/translator/stm/src_stm/stmgc.h
+++ b/rpython/translator/stm/src_stm/stmgc.h
@@ -535,8 +535,12 @@
    If you want to embed the hashtable inside an 'object_t' you
    probably need a light finalizer to do the freeing. */
 typedef struct stm_hashtable_s stm_hashtable_t;
+typedef TLPREFIX struct stm_hashtable_entry_s stm_hashtable_entry_t;
+
 stm_hashtable_t *stm_hashtable_create(void);
 void stm_hashtable_free(stm_hashtable_t *);
+stm_hashtable_entry_t *stm_hashtable_lookup(object_t *, stm_hashtable_t *,
+                                            uintptr_t key);
 object_t *stm_hashtable_read(object_t *, stm_hashtable_t *, uintptr_t key);
 void stm_hashtable_write(object_t *, stm_hashtable_t *, uintptr_t key,
                          object_t *nvalue, stm_thread_local_t *);

From noreply at buildbot.pypy.org  Mon Nov 10 16:18:13 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 10 Nov 2014 16:18:13 +0100 (CET)
Subject: [pypy-commit] pypy stmgc-c7: Enable the _stm module when
	translating with stm
Message-ID: <20141110151813.0F5DF1C07BB@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: stmgc-c7
Changeset: r74429:88991c2dc871
Date: 2014-11-10 16:17 +0100
http://bitbucket.org/pypy/pypy/changeset/88991c2dc871/

Log:	Enable the _stm module when translating with stm

diff --git a/pypy/goal/targetpypystandalone.py b/pypy/goal/targetpypystandalone.py
--- a/pypy/goal/targetpypystandalone.py
+++ b/pypy/goal/targetpypystandalone.py
@@ -245,6 +245,7 @@
 
         if config.translation.stm:
             config.translation.thread = True
+            config.objspace.usemodules._stm = True
 
         if config.objspace.allworkingmodules:
             from pypy.config.pypyoption import enable_allworkingmodules

From noreply at buildbot.pypy.org  Mon Nov 10 18:05:14 2014
From: noreply at buildbot.pypy.org (amauryfa)
Date: Mon, 10 Nov 2014 18:05:14 +0100 (CET)
Subject: [pypy-commit] pypy more-rposix: Fix translation
Message-ID: <20141110170514.9F7791C33C0@cobra.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc 
Branch: more-rposix
Changeset: r74430:6527530048c9
Date: 2014-11-10 18:04 +0100
http://bitbucket.org/pypy/pypy/changeset/6527530048c9/

Log:	Fix translation

diff --git a/pypy/module/posix/__init__.py b/pypy/module/posix/__init__.py
--- a/pypy/module/posix/__init__.py
+++ b/pypy/module/posix/__init__.py
@@ -1,5 +1,4 @@
 from pypy.interpreter.mixedmodule import MixedModule
-from rpython.rtyper.module.ll_os import RegisterOs
 
 import os
 exec 'import %s as posix' % os.name
@@ -172,7 +171,9 @@
     if hasattr(os, 'chroot'):
         interpleveldefs['chroot'] = 'interp_posix.chroot'
 
-    for name in RegisterOs.w_star:
+    for name in ['WCOREDUMP', 'WIFCONTINUED', 'WIFSTOPPED',
+                 'WIFSIGNALED', 'WIFEXITED',
+                 'WEXITSTATUS', 'WSTOPSIG', 'WTERMSIG']:
         if hasattr(os, name):
             interpleveldefs[name] = 'interp_posix.' + name
 
diff --git a/pypy/module/posix/interp_posix.py b/pypy/module/posix/interp_posix.py
--- a/pypy/module/posix/interp_posix.py
+++ b/pypy/module/posix/interp_posix.py
@@ -6,7 +6,6 @@
 from rpython.rlib.rarithmetic import r_longlong
 from rpython.rlib.unroll import unrolling_iterable
 from rpython.rtyper.module import ll_os_stat
-from rpython.rtyper.module.ll_os import RegisterOs
 
 from pypy.interpreter.gateway import unwrap_spec
 from pypy.interpreter.error import OperationError, wrap_oserror, wrap_oserror2
@@ -1203,7 +1202,7 @@
         raise wrap_oserror(space, e)
 
 def declare_new_w_star(name):
-    if name in RegisterOs.w_star_returning_int:
+    if name in ('WEXITSTATUS', 'WSTOPSIG', 'WTERMSIG'):
         @unwrap_spec(status=c_int)
         def WSTAR(space, status):
             return space.wrap(getattr(os, name)(status))
@@ -1215,7 +1214,9 @@
     WSTAR.func_name = name
     return WSTAR
 
-for name in RegisterOs.w_star:
+for name in ['WCOREDUMP', 'WIFCONTINUED', 'WIFSTOPPED',
+             'WIFSIGNALED', 'WIFEXITED',
+             'WEXITSTATUS', 'WSTOPSIG', 'WTERMSIG']:
     if hasattr(os, name):
         func = declare_new_w_star(name)
         globals()[name] = func
diff --git a/pypy/module/posix/test/test_posix2.py b/pypy/module/posix/test/test_posix2.py
--- a/pypy/module/posix/test/test_posix2.py
+++ b/pypy/module/posix/test/test_posix2.py
@@ -6,7 +6,6 @@
 from rpython.tool.udir import udir
 from pypy.tool.pytest.objspace import gettestobjspace
 from pypy.conftest import pypydir
-from rpython.rtyper.module.ll_os import RegisterOs
 from rpython.translator.c.test.test_extfunc import need_sparse_files
 import os
 import py
@@ -576,7 +575,9 @@
         raises(TypeError, "os.utime('xxx', 3)")
         raises(OSError, "os.utime('somefilewhichihopewouldneverappearhere', None)")
 
-    for name in RegisterOs.w_star:
+    for name in ['WCOREDUMP', 'WIFCONTINUED', 'WIFSTOPPED',
+                 'WIFSIGNALED', 'WIFEXITED',
+                 'WEXITSTATUS', 'WSTOPSIG', 'WTERMSIG']:
         if hasattr(os, name):
             values = [0, 1, 127, 128, 255]
             code = py.code.Source("""
diff --git a/rpython/rlib/rposix.py b/rpython/rlib/rposix.py
--- a/rpython/rlib/rposix.py
+++ b/rpython/rlib/rposix.py
@@ -831,7 +831,7 @@
 
 @replace_os_function('ttyname')
 def ttyname(fd):
-    l_name = os_ttyname(fd)
+    l_name = c_ttyname(fd)
     if not l_name:
         raise OSError(get_errno(), "ttyname raised")
     return rffi.charp2str(l_name)

From noreply at buildbot.pypy.org  Mon Nov 10 19:45:58 2014
From: noreply at buildbot.pypy.org (amauryfa)
Date: Mon, 10 Nov 2014 19:45:58 +0100 (CET)
Subject: [pypy-commit] pypy more-rposix: Fix tests (and translation of small
	targets)
Message-ID: <20141110184558.D6A9C1C134B@cobra.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc 
Branch: more-rposix
Changeset: r74431:77ea270bb7a5
Date: 2014-11-10 19:45 +0100
http://bitbucket.org/pypy/pypy/changeset/77ea270bb7a5/

Log:	Fix tests (and translation of small targets)

diff --git a/rpython/flowspace/specialcase.py b/rpython/flowspace/specialcase.py
--- a/rpython/flowspace/specialcase.py
+++ b/rpython/flowspace/specialcase.py
@@ -59,7 +59,7 @@
         return SPECIAL_CASES[fn]   # TypeError if 'fn' not hashable
     except (KeyError, TypeError):
         # Try to import modules containing special cases
-        for modname in SPECIAL_MODULES.get(fn.__module__, []):
+        for modname in SPECIAL_MODULES.get(getattr(fn, '__module__', None), []):
             __import__(modname)
         try:
             return SPECIAL_CASES[fn]
diff --git a/rpython/rlib/rfile.py b/rpython/rlib/rfile.py
--- a/rpython/rlib/rfile.py
+++ b/rpython/rlib/rfile.py
@@ -160,7 +160,6 @@
 
 def create_fdopen_rfile(fd, mode="r", buffering=-1):
     newmode = _sanitize_mode(mode)
-    fd = rffi.cast(rffi.INT, fd)
     rposix.validate_fd(fd)
     ll_mode = rffi.str2charp(newmode)
     try:
diff --git a/rpython/rlib/rposix.py b/rpython/rlib/rposix.py
--- a/rpython/rlib/rposix.py
+++ b/rpython/rlib/rposix.py
@@ -335,6 +335,7 @@
                    releasegil=False)
 
 @replace_os_function('read')
+ at enforceargs(int, int)
 def read(fd, count):
     if count < 0:
         raise OSError(errno.EINVAL, None)
@@ -345,6 +346,7 @@
         return buf.str(got)
 
 @replace_os_function('write')
+ at enforceargs(int, None)
 def write(fd, data):
     count = len(data)
     validate_fd(fd)

From noreply at buildbot.pypy.org  Mon Nov 10 21:24:59 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 10 Nov 2014 21:24:59 +0100 (CET)
Subject: [pypy-commit] pypy default: Test and fix: produce a user-readable
 jit_debug() output in the logs
Message-ID: <20141110202459.978F91C362C@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r74432:a5e7edf73a08
Date: 2014-11-10 21:24 +0100
http://bitbucket.org/pypy/pypy/changeset/a5e7edf73a08/

Log:	Test and fix: produce a user-readable jit_debug() output in the logs
	too, not just when tracing it the first time. It was half-pointless
	to record the jit_debug operation.

diff --git a/rpython/jit/metainterp/logger.py b/rpython/jit/metainterp/logger.py
--- a/rpython/jit/metainterp/logger.py
+++ b/rpython/jit/metainterp/logger.py
@@ -137,6 +137,14 @@
             s = jd_sd.warmstate.get_location_str(op.getarglist()[3:])
             s = s.replace(',', '.') # we use comma for argument splitting
             return "debug_merge_point(%d, %d, '%s')" % (op.getarg(1).getint(), op.getarg(2).getint(), s)
+        if op.getopnum() == rop.JIT_DEBUG:
+            args = op.getarglist()
+            s = args[0]._get_str()
+            s = s.replace(',', '.') # we use comma for argument splitting
+            s2 = ''
+            for box in args[1:]:
+                s2 += ', %d' % box.getint()
+            return "jit_debug('%s'%s)" % (s, s2)
         if ops_offset is None:
             offset = -1
         else:
diff --git a/rpython/jit/metainterp/pyjitpl.py b/rpython/jit/metainterp/pyjitpl.py
--- a/rpython/jit/metainterp/pyjitpl.py
+++ b/rpython/jit/metainterp/pyjitpl.py
@@ -1178,10 +1178,7 @@
 
     @arguments("box", "box", "box", "box", "box")
     def opimpl_jit_debug(self, stringbox, arg1box, arg2box, arg3box, arg4box):
-        from rpython.rtyper.lltypesystem import rstr
-        from rpython.rtyper.annlowlevel import hlstr
-        msg = stringbox.getref(lltype.Ptr(rstr.STR))
-        debug_print('jit_debug:', hlstr(msg),
+        debug_print('jit_debug:', stringbox._get_str(),
                     arg1box.getint(), arg2box.getint(),
                     arg3box.getint(), arg4box.getint())
         args = [stringbox, arg1box, arg2box, arg3box, arg4box]
diff --git a/rpython/jit/metainterp/test/test_logger.py b/rpython/jit/metainterp/test/test_logger.py
--- a/rpython/jit/metainterp/test/test_logger.py
+++ b/rpython/jit/metainterp/test/test_logger.py
@@ -137,6 +137,17 @@
         assert loop.operations[0].getarg(2).getint() == 0
         assert oloop.operations[0].getarg(2)._get_str() == "dupa"
 
+    def test_jit_debug(self):
+        inp = '''
+        []
+        jit_debug('foobar', -1, 5)
+        '''
+        _, loop, oloop = self.reparse(inp)
+        assert loop.operations[0].getarg(0)._get_str() == "foobar"
+        assert loop.operations[0].getarg(1).getint() == -1
+        assert oloop.operations[0].getarg(0)._get_str() == "foobar"
+        assert oloop.operations[0].getarg(1).getint() == -1
+
     def test_floats(self):
         inp = '''
         [f0]

From noreply at buildbot.pypy.org  Mon Nov 10 21:45:09 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 10 Nov 2014 21:45:09 +0100 (CET)
Subject: [pypy-commit] pypy default: minor fixes for jit_debug
Message-ID: <20141110204509.6AC5A1C134B@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r74433:95244fa2d0d2
Date: 2014-11-10 21:44 +0100
http://bitbucket.org/pypy/pypy/changeset/95244fa2d0d2/

Log:	minor fixes for jit_debug

diff --git a/rpython/jit/metainterp/optimizeopt/heap.py b/rpython/jit/metainterp/optimizeopt/heap.py
--- a/rpython/jit/metainterp/optimizeopt/heap.py
+++ b/rpython/jit/metainterp/optimizeopt/heap.py
@@ -273,6 +273,7 @@
             opnum == rop.STRSETITEM or           # no effect on GC struct/array
             opnum == rop.UNICODESETITEM or       # no effect on GC struct/array
             opnum == rop.DEBUG_MERGE_POINT or    # no effect whatsoever
+            opnum == rop.JIT_DEBUG or            # no effect whatsoever
             opnum == rop.COPYSTRCONTENT or       # no effect on GC struct/array
             opnum == rop.COPYUNICODECONTENT):    # no effect on GC struct/array
             return
diff --git a/rpython/jit/metainterp/optimizeopt/optimizer.py b/rpython/jit/metainterp/optimizeopt/optimizer.py
--- a/rpython/jit/metainterp/optimizeopt/optimizer.py
+++ b/rpython/jit/metainterp/optimizeopt/optimizer.py
@@ -660,6 +660,9 @@
     def optimize_DEBUG_MERGE_POINT(self, op):
         self.emit_operation(op)
 
+    def optimize_JIT_DEBUG(self, op):
+        self.emit_operation(op)
+
     def optimize_STRGETITEM(self, op):
         indexvalue = self.getvalue(op.getarg(1))
         if indexvalue.is_constant():
diff --git a/rpython/jit/tool/oparser.py b/rpython/jit/tool/oparser.py
--- a/rpython/jit/tool/oparser.py
+++ b/rpython/jit/tool/oparser.py
@@ -298,8 +298,9 @@
         first_comment = None
         for line in lines:
             # for simplicity comments are not allowed on
-            # debug_merge_point lines
-            if '#' in line and 'debug_merge_point(' not in line:
+            # debug_merge_point or jit_debug lines
+            if '#' in line and ('debug_merge_point(' not in line and
+                                'jit_debug(' not in line):
                 if line.lstrip()[0] == '#': # comment only
                     if first_comment is None:
                         first_comment = line

From noreply at buildbot.pypy.org  Tue Nov 11 02:39:06 2014
From: noreply at buildbot.pypy.org (rlamy)
Date: Tue, 11 Nov 2014 02:39:06 +0100 (CET)
Subject: [pypy-commit] pypy expressions: Make sure that inputargs contains
	only Variables
Message-ID: <20141111013906.1B94C1C354A@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy 
Branch: expressions
Changeset: r74439:5e50ef071407
Date: 2014-11-09 05:27 +0000
http://bitbucket.org/pypy/pypy/changeset/5e50ef071407/

Log:	Make sure that inputargs contains only Variables

diff --git a/rpython/flowspace/model.py b/rpython/flowspace/model.py
--- a/rpython/flowspace/model.py
+++ b/rpython/flowspace/model.py
@@ -163,7 +163,7 @@
                 exits blockcolor""".split()
 
     def __init__(self, inputargs):
-        self.inputargs = list(inputargs)  # mixed list of variable/const XXX
+        self.inputargs = list(inputargs)  # list of Variable
         self.operations = []              # list of SpaceOperation(s)
         self.exitswitch = None            # a variable or
                                           #  Constant(last_exception), see below
@@ -197,7 +197,8 @@
         "Return all variables mentioned in this Block."
         result = self.inputargs[:]
         for op in self.operations:
-            result += op.args
+            for arg in op.args:
+                result.extend(arg.dependencies)
             result.append(op.result)
         return uniqueitems([w for w in result if isinstance(w, Variable)])
 
@@ -566,7 +567,7 @@
             assert block.exits == ()
 
         def definevar(v, only_in_link=None):
-            assert isinstance(v, Variable)
+            assert type(v) is Variable
             assert v not in vars, "duplicate variable %r" % (v,)
             assert v not in vars_previous_blocks, (
                 "variable %r used in more than one block" % (v,))

From noreply at buildbot.pypy.org  Tue Nov 11 02:38:59 2014
From: noreply at buildbot.pypy.org (rlamy)
Date: Tue, 11 Nov 2014 02:38:59 +0100 (CET)
Subject: [pypy-commit] pypy expressions: Create expression object V_Type and
 use it during flowing
Message-ID: <20141111013859.D78AA1C354A@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy 
Branch: expressions
Changeset: r74434:a5c6b9a6c89d
Date: 2014-11-08 01:57 +0000
http://bitbucket.org/pypy/pypy/changeset/a5c6b9a6c89d/

Log:	Create expression object V_Type and use it during flowing

diff --git a/rpython/annotator/expression.py b/rpython/annotator/expression.py
new file mode 100644
--- /dev/null
+++ b/rpython/annotator/expression.py
@@ -0,0 +1,17 @@
+from rpython.flowspace.model import Variable
+from rpython.flowspace.operation import op
+from rpython.annotator.model import SomeType
+
+class V_Type(Variable):
+    def __init__(self, v_obj):
+        Variable.__init__(self)
+        self.arg = v_obj
+        s = SomeType()
+        s.is_type_of = [v_obj]
+        self.annotation = s
+
+    def as_operation(self):
+        return op.type(self.arg)
+
+    def __eq__(self, other):
+        return isinstance(other, V_Type) and other.arg == self.arg
diff --git a/rpython/annotator/unaryop.py b/rpython/annotator/unaryop.py
--- a/rpython/annotator/unaryop.py
+++ b/rpython/annotator/unaryop.py
@@ -8,7 +8,7 @@
 from rpython.annotator.model import (SomeObject, SomeInteger, SomeBool,
     SomeString, SomeChar, SomeList, SomeDict, SomeTuple, SomeImpossibleValue,
     SomeUnicodeCodePoint, SomeInstance, SomeBuiltin, SomeBuiltinMethod,
-    SomeFloat, SomeIterator, SomePBC, SomeNone, SomeType, s_ImpossibleValue,
+    SomeFloat, SomeIterator, SomePBC, SomeNone, s_ImpossibleValue,
     s_Bool, s_None, unionof, add_knowntypedata,
     HarmlesslyBlocked, SomeWeakRef, SomeUnicodeString, SomeByteArray)
 from rpython.annotator.bookkeeper import getbookkeeper, immutablevalue
@@ -21,11 +21,9 @@
                         if oper.dispatch == 1])
 UNARY_OPERATIONS.remove('contains')
 
- at op.type.register(SomeObject)
-def type_SomeObject(annotator, arg):
-    r = SomeType()
-    r.is_type_of = [arg]
-    return r
+ at op.assign.register(SomeObject)
+def assign(annotator, v_obj):
+    return annotator.annotation(v_obj)
 
 @op.bool.register(SomeObject)
 def bool_SomeObject(annotator, obj):
diff --git a/rpython/flowspace/model.py b/rpython/flowspace/model.py
--- a/rpython/flowspace/model.py
+++ b/rpython/flowspace/model.py
@@ -250,7 +250,7 @@
 
 
 class Variable(object):
-    __slots__ = ["_name", "_nr", "annotation", "concretetype"]
+    __slots__ = ["_name", "_nr", "annotation", "concretetype", "equals"]
 
     dummyname = 'v'
     namesdict = {dummyname: (dummyname, 0)}
@@ -578,7 +578,8 @@
                 for v in op.args:
                     assert isinstance(v, (Constant, Variable))
                     if isinstance(v, Variable):
-                        usevar(v)
+                        if type(v) is Variable:
+                            usevar(v)
                     else:
                         assert v.value is not last_exception
                         #assert v.value != last_exc_value
@@ -648,14 +649,10 @@
                     assert link.last_exc_value is None
                 for v in link.args:
                     assert isinstance(v, (Constant, Variable))
-                    if isinstance(v, Variable):
+                    if type(v) is Variable:
                         usevar(v, in_link=link)
                         if exc_link:
                             assert v != block.operations[-1].result
-                    #else:
-                    #    if not exc_link:
-                    #        assert v.value is not last_exception
-                    #        #assert v.value != last_exc_value
                 allexitcases[link.exitcase] = True
             assert len(allexitcases) == len(block.exits)
             vars_previous_blocks.update(vars)
diff --git a/rpython/flowspace/operation.py b/rpython/flowspace/operation.py
--- a/rpython/flowspace/operation.py
+++ b/rpython/flowspace/operation.py
@@ -356,7 +356,7 @@
 
 add_operator('is_', 2, dispatch=2, pure=True)
 add_operator('id', 1, dispatch=1, pyfunc=id)
-add_operator('type', 1, dispatch=1, pyfunc=new_style_type, pure=True)
+#add_operator('type', 1, dispatch=1, pyfunc=new_style_type, pure=True)
 add_operator('issubtype', 2, dispatch=1, pyfunc=issubclass, pure=True)  # not for old-style classes
 add_operator('repr', 1, dispatch=1, pyfunc=repr, pure=True)
 add_operator('str', 1, dispatch=1, pyfunc=str, pure=True)
@@ -431,6 +431,7 @@
 add_operator('yield_', 1)
 add_operator('newslice', 3)
 add_operator('hint', None, dispatch=1)
+add_operator('assign', 1, dispatch=1)
 
 class Contains(SingleDispatchMixin, PureOperation):
     opname = 'contains'
@@ -442,6 +443,21 @@
     def get_specialization(cls, s_seq, s_elem):
         return cls._dispatch(type(s_seq))
 
+class Type(SingleDispatchMixin, PureOperation):
+    opname = 'type'
+    arity = 1
+    canraise = []
+    pyfunc = staticmethod(new_style_type)
+
+    def eval(self, ctx):
+        result = self.constfold()
+        if result is not None:
+            return result
+        from rpython.annotator.expression import V_Type
+        v_instance, = self.args
+        result = V_Type(v_instance)
+        return ctx.do_op(op.assign(result))
+
 
 class NewDict(HLOperation):
     opname = 'newdict'
diff --git a/rpython/translator/transform.py b/rpython/translator/transform.py
--- a/rpython/translator/transform.py
+++ b/rpython/translator/transform.py
@@ -134,6 +134,16 @@
                     s_dict = self.annotation(op.args[0])
                     s_dict.dictdef.generalize_key(self.binding(op.args[1]))
 
+def remove_assign(ann, block_subset):
+    for block in block_subset:
+        for i in range(len(block.operations)):
+            op = block.operations[i]
+            if op.opname == 'assign':
+                new_op = op.args[0].as_operation()
+                new_op.result = op.result
+                block.operations[i] = new_op
+
+
 
 def transform_dead_op_vars(self, block_subset):
     # we redo the same simplification from simplify.py,
@@ -249,6 +259,7 @@
     transform_extend_with_str_slice,
     transform_extend_with_char_count,
     transform_list_contains,
+    remove_assign,
     ]
 
 def transform_graph(ann, extra_passes=None, block_subset=None):

From noreply at buildbot.pypy.org  Tue Nov 11 02:39:01 2014
From: noreply at buildbot.pypy.org (rlamy)
Date: Tue, 11 Nov 2014 02:39:01 +0100 (CET)
Subject: [pypy-commit] pypy expressions: apply variable renaming to V_Type
Message-ID: <20141111013901.188441C354A@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy 
Branch: expressions
Changeset: r74435:fd557eb28c87
Date: 2014-11-08 18:31 +0000
http://bitbucket.org/pypy/pypy/changeset/fd557eb28c87/

Log:	apply variable renaming to V_Type

diff --git a/rpython/annotator/expression.py b/rpython/annotator/expression.py
--- a/rpython/annotator/expression.py
+++ b/rpython/annotator/expression.py
@@ -15,3 +15,9 @@
 
     def __eq__(self, other):
         return isinstance(other, V_Type) and other.arg == self.arg
+
+    def replace(self, mapping):
+        if self.arg in mapping:
+            return V_Type(mapping[self.arg])
+        else:
+            return self
diff --git a/rpython/flowspace/flowcontext.py b/rpython/flowspace/flowcontext.py
--- a/rpython/flowspace/flowcontext.py
+++ b/rpython/flowspace/flowcontext.py
@@ -465,7 +465,7 @@
             msg = "implicit %s shouldn't occur" % exc_cls.__name__
             w_type = Constant(AssertionError)
             w_value = Constant(AssertionError(msg))
-            link = Link([w_type, w_value], self.graph.exceptblock)
+            link = self.graph.new_raise_link(w_value, w_type)
             self.recorder.crnt_block.closeblock(link)
 
         except Raise as e:
@@ -473,7 +473,7 @@
             if w_exc.w_type == const(ImportError):
                 msg = 'import statement always raises %s' % e
                 raise ImportError(msg)
-            link = Link([w_exc.w_type, w_exc.w_value], self.graph.exceptblock)
+            link = self.graph.new_raise_link(w_exc.w_value, w_exc.w_type)
             self.recorder.crnt_block.closeblock(link)
 
         except StopFlowing:
diff --git a/rpython/flowspace/model.py b/rpython/flowspace/model.py
--- a/rpython/flowspace/model.py
+++ b/rpython/flowspace/model.py
@@ -31,6 +31,9 @@
     def getreturnvar(self):
         return self.returnblock.inputargs[0]
 
+    def new_raise_link(self, v_exception, v_exc_type):
+        return Link([v_exc_type, v_exception], self.exceptblock)
+
     @property
     def source(self):
         if hasattr(self, "_source"):
@@ -208,13 +211,12 @@
     def renamevariables(self, mapping):
         for a in mapping:
             assert isinstance(a, Variable), a
-        self.inputargs = [mapping.get(a, a) for a in self.inputargs]
-        for op in self.operations:
-            op.args = [mapping.get(a, a) for a in op.args]
-            op.result = mapping.get(op.result, op.result)
-        self.exitswitch = mapping.get(self.exitswitch, self.exitswitch)
+        self.inputargs = [a.replace(mapping) for a in self.inputargs]
+        self.operations = [op.replace(mapping) for op in self.operations]
+        if self.exitswitch is not None:
+            self.exitswitch = self.exitswitch.replace(mapping)
         for link in self.exits:
-            link.args = [mapping.get(a, a) for a in link.args]
+            link.args = [a.replace(mapping) for a in link.args]
 
     def closeblock(self, *exits):
         assert self.exits == [], "block already closed"
@@ -320,6 +322,8 @@
             newvar.concretetype = self.concretetype
         return newvar
 
+    def replace(self, mapping):
+        return mapping.get(self, self)
 
 
 class Constant(Hashable):
@@ -349,6 +353,9 @@
             # cannot count on it not mutating at runtime!
             return False
 
+    def replace(self, mapping):
+        return self
+
 
 class FSException(object):
     def __init__(self, w_type, w_value):
@@ -424,8 +431,8 @@
                                 ", ".join(map(repr, self.args)))
 
     def replace(self, mapping):
-        newargs = [mapping.get(arg, arg) for arg in self.args]
-        newresult = mapping.get(self.result, self.result)
+        newargs = [arg.replace(mapping) for arg in self.args]
+        newresult = self.result.replace(mapping)
         return type(self)(self.opname, newargs, newresult, self.offset)
 
 class Atom(object):
@@ -580,6 +587,8 @@
                     if isinstance(v, Variable):
                         if type(v) is Variable:
                             usevar(v)
+                        else:
+                            usevar(v.arg)
                     else:
                         assert v.value is not last_exception
                         #assert v.value != last_exc_value
diff --git a/rpython/flowspace/operation.py b/rpython/flowspace/operation.py
--- a/rpython/flowspace/operation.py
+++ b/rpython/flowspace/operation.py
@@ -74,8 +74,8 @@
         self.offset = -1
 
     def replace(self, mapping):
-        newargs = [mapping.get(arg, arg) for arg in self.args]
-        newresult = mapping.get(self.result, self.result)
+        newargs = [arg.replace(mapping) for arg in self.args]
+        newresult = self.result.replace(mapping)
         newop = type(self)(*newargs)
         newop.result = newresult
         newop.offset = self.offset

From noreply at buildbot.pypy.org  Tue Nov 11 02:39:07 2014
From: noreply at buildbot.pypy.org (rlamy)
Date: Tue, 11 Nov 2014 02:39:07 +0100 (CET)
Subject: [pypy-commit] pypy expressions: create ExceptBlock class
Message-ID: <20141111013907.2DEF21C354A@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy 
Branch: expressions
Changeset: r74440:2328e7a3f341
Date: 2014-11-10 01:23 +0000
http://bitbucket.org/pypy/pypy/changeset/2328e7a3f341/

Log:	create ExceptBlock class

diff --git a/rpython/flowspace/model.py b/rpython/flowspace/model.py
--- a/rpython/flowspace/model.py
+++ b/rpython/flowspace/model.py
@@ -18,11 +18,7 @@
         self.returnblock = Block([return_var or Variable()])
         self.returnblock.operations = ()
         self.returnblock.exits = ()
-        # block corresponding to exception results
-        self.exceptblock = Block([Variable('etype'),   # exception class
-                                  Variable('evalue')])  # exception value
-        self.exceptblock.operations = ()
-        self.exceptblock.exits = ()
+        self.exceptblock = ExceptBlock()
         self.tag = None
 
     def getargs(self):
@@ -181,8 +177,6 @@
         else:
             if (not self.exits) and len(self.inputargs) == 1:
                 txt = "return block"
-            elif (not self.exits) and len(self.inputargs) == 2:
-                txt = "raise block"
             else:
                 txt = "codeless block"
         return txt
@@ -251,6 +245,17 @@
 
     view = show
 
+class ExceptBlock(Block):
+    def __init__(self):
+        self.inputargs = [Variable('etype'),   # exception class
+                          Variable('evalue')]  # exception value
+        self.operations = ()
+        self.exits = ()
+        self.exitswitch = None
+
+    def __str__(self):
+        return "raise block"
+
 
 class Variable(object):
     __slots__ = ["_name", "_nr", "annotation", "concretetype", "equals"]

From noreply at buildbot.pypy.org  Tue Nov 11 02:39:02 2014
From: noreply at buildbot.pypy.org (rlamy)
Date: Tue, 11 Nov 2014 02:39:02 +0100 (CET)
Subject: [pypy-commit] pypy expressions: fix
	transform_dead_op_vars_in_blocks()
Message-ID: <20141111013902.50F171C354A@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy 
Branch: expressions
Changeset: r74436:b499e643c6d3
Date: 2014-11-09 01:11 +0000
http://bitbucket.org/pypy/pypy/changeset/b499e643c6d3/

Log:	fix transform_dead_op_vars_in_blocks()

diff --git a/rpython/annotator/expression.py b/rpython/annotator/expression.py
--- a/rpython/annotator/expression.py
+++ b/rpython/annotator/expression.py
@@ -21,3 +21,7 @@
             return V_Type(mapping[self.arg])
         else:
             return self
+
+    @property
+    def dependencies(self):
+        return self.arg.dependencies
diff --git a/rpython/flowspace/model.py b/rpython/flowspace/model.py
--- a/rpython/flowspace/model.py
+++ b/rpython/flowspace/model.py
@@ -325,6 +325,10 @@
     def replace(self, mapping):
         return mapping.get(self, self)
 
+    @property
+    def dependencies(self):
+        return set([self])
+
 
 class Constant(Hashable):
     __slots__ = ["concretetype"]
@@ -356,6 +360,10 @@
     def replace(self, mapping):
         return self
 
+    @property
+    def dependencies(self):
+        return set()
+
 
 class FSException(object):
     def __init__(self, w_type, w_value):
diff --git a/rpython/translator/simplify.py b/rpython/translator/simplify.py
--- a/rpython/translator/simplify.py
+++ b/rpython/translator/simplify.py
@@ -420,9 +420,11 @@
         # figure out which variables are ever read
         for op in block.operations:
             if not canremove(op, block):   # the inputs are always needed
-                read_vars.update(op.args)
+                for arg in op.args:
+                    read_vars.update(arg.dependencies)
             else:
-                dependencies[op.result].update(op.args)
+                for arg in op.args:
+                    dependencies[op.result].update(arg.dependencies)
 
         if isinstance(block.exitswitch, Variable):
             read_vars.add(block.exitswitch)

From noreply at buildbot.pypy.org  Tue Nov 11 02:39:08 2014
From: noreply at buildbot.pypy.org (rlamy)
Date: Tue, 11 Nov 2014 02:39:08 +0100 (CET)
Subject: [pypy-commit] pypy expressions: refactor join_blocks()
Message-ID: <20141111013908.5A6651C354A@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy 
Branch: expressions
Changeset: r74441:320c084c002c
Date: 2014-11-10 18:51 +0000
http://bitbucket.org/pypy/pypy/changeset/320c084c002c/

Log:	refactor join_blocks()

diff --git a/rpython/translator/simplify.py b/rpython/translator/simplify.py
--- a/rpython/translator/simplify.py
+++ b/rpython/translator/simplify.py
@@ -63,26 +63,19 @@
     When this happens, we need to replace the preceeding link with the
     following link.  Arguments of the links should be updated."""
     for link in list(graph.iterlinks()):
-            while not link.target.operations:
-                block1 = link.target
-                if block1.exitswitch is not None:
-                    break
-                if not block1.exits:
-                    break
-                exit = block1.exits[0]
-                assert block1 is not exit.target, (
-                    "the graph contains an empty infinite loop")
-                outputargs = []
-                for v in exit.args:
-                    if isinstance(v, Variable):
-                        # this variable is valid in the context of block1
-                        # but it must come from 'link'
-                        i = block1.inputargs.index(v)
-                        v = link.args[i]
-                    outputargs.append(v)
-                link.args = outputargs
-                link.target = exit.target
-                # the while loop above will simplify recursively the new link
+        while not link.target.operations:
+            block1 = link.target
+            if block1.exitswitch is not None:
+                break
+            if not block1.exits:
+                break
+            exit = block1.exits[0]
+            assert block1 is not exit.target, (
+                "the graph contains an empty infinite loop")
+            subst = dict(zip(block1.inputargs, link.args))
+            link.args = [v.replace(subst) for v in exit.args]
+            link.target = exit.target
+            # the while loop above will simplify recursively the new link
 
 def transform_ovfcheck(graph):
     """The special function calls ovfcheck needs to

From noreply at buildbot.pypy.org  Tue Nov 11 02:39:03 2014
From: noreply at buildbot.pypy.org (rlamy)
Date: Tue, 11 Nov 2014 02:39:03 +0100 (CET)
Subject: [pypy-commit] pypy expressions: fix generators
Message-ID: <20141111013903.9B2541C3A61@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy 
Branch: expressions
Changeset: r74437:ebe48f5de764
Date: 2014-11-09 01:48 +0000
http://bitbucket.org/pypy/pypy/changeset/ebe48f5de764/

Log:	fix generators

diff --git a/rpython/flowspace/generator.py b/rpython/flowspace/generator.py
--- a/rpython/flowspace/generator.py
+++ b/rpython/flowspace/generator.py
@@ -106,6 +106,7 @@
 def tweak_generator_body_graph(Entry, graph):
     # First, always run simplify_graph in order to reduce the number of
     # variables passed around
+    from rpython.annotator.expression import V_Type
     simplify_graph(graph)
     insert_empty_startblock(None, graph)
     _insert_reads(graph.startblock, Entry.varnames)
@@ -115,7 +116,7 @@
     #
     stopblock = Block([])
     op0 = op.simple_call(const(StopIteration))
-    op1 = op.type(op0.result)
+    op1 = op.assign(V_Type(op0.result))
     stopblock.operations = [op0, op1]
     stopblock.closeblock(Link([op1.result, op0.result], graph.exceptblock))
     #

From noreply at buildbot.pypy.org  Tue Nov 11 02:39:09 2014
From: noreply at buildbot.pypy.org (rlamy)
Date: Tue, 11 Nov 2014 02:39:09 +0100 (CET)
Subject: [pypy-commit] pypy expressions: remove op.assign hack
Message-ID: <20141111013909.7B2E41C354A@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy 
Branch: expressions
Changeset: r74442:052a0a7e7232
Date: 2014-11-11 00:51 +0000
http://bitbucket.org/pypy/pypy/changeset/052a0a7e7232/

Log:	remove op.assign hack

diff --git a/rpython/annotator/unaryop.py b/rpython/annotator/unaryop.py
--- a/rpython/annotator/unaryop.py
+++ b/rpython/annotator/unaryop.py
@@ -21,10 +21,6 @@
                         if oper.dispatch == 1])
 UNARY_OPERATIONS.remove('contains')
 
- at op.assign.register(SomeObject)
-def assign(annotator, v_obj):
-    return annotator.annotation(v_obj)
-
 @op.bool.register(SomeObject)
 def bool_SomeObject(annotator, obj):
     r = SomeBool()
diff --git a/rpython/flowspace/flowcontext.py b/rpython/flowspace/flowcontext.py
--- a/rpython/flowspace/flowcontext.py
+++ b/rpython/flowspace/flowcontext.py
@@ -93,20 +93,10 @@
     for link in list(graph.iterlinks()):
         block = link.target
         if isinstance(block, EggBlock):
-            if (not block.operations and len(block.exits) == 1 and
-                link.args == block.inputargs):   # not renamed
-                # if the variables are not renamed across this link
-                # (common case for EggBlocks) then it's easy enough to
-                # get rid of the empty EggBlock.
-                link2 = block.exits[0]
-                link.args = list(link2.args)
-                link.target = link2.target
-                assert link2.exitcase is None
-            else:
-                mapping = {}
-                for a in block.inputargs:
-                    mapping[a] = Variable(a)
-                block.renamevariables(mapping)
+            mapping = {}
+            for a in block.inputargs:
+                mapping[a] = Variable(a)
+            block.renamevariables(mapping)
 
 # ____________________________________________________________
 
@@ -413,15 +403,19 @@
             return w_condition.value
         return self.recorder.guessbool(self, w_condition)
 
-    def record(self, spaceop):
+    def merge_point(self):
         recorder = self.recorder
         if getattr(recorder, 'final_state', None) is not None:
             self.mergeblock(recorder.crnt_block, recorder.final_state)
             raise StopFlowing
+
+    def record(self, spaceop):
+        recorder = self.recorder
         spaceop.offset = self.last_instr
         recorder.append(spaceop)
 
     def do_op(self, op):
+        self.merge_point()
         self.record(op)
         self.guessexception(op.canraise)
         return op.result
diff --git a/rpython/flowspace/generator.py b/rpython/flowspace/generator.py
--- a/rpython/flowspace/generator.py
+++ b/rpython/flowspace/generator.py
@@ -116,9 +116,8 @@
     #
     stopblock = Block([])
     op0 = op.simple_call(const(StopIteration))
-    op1 = op.assign(V_Type(op0.result))
-    stopblock.operations = [op0, op1]
-    stopblock.closeblock(Link([op1.result, op0.result], graph.exceptblock))
+    stopblock.operations = [op0]
+    stopblock.closeblock(Link([V_Type(op0.result), op0.result], graph.exceptblock))
     #
     for block in list(graph.iterblocks()):
         for exit in block.exits:
diff --git a/rpython/flowspace/operation.py b/rpython/flowspace/operation.py
--- a/rpython/flowspace/operation.py
+++ b/rpython/flowspace/operation.py
@@ -431,7 +431,6 @@
 add_operator('yield_', 1)
 add_operator('newslice', 3)
 add_operator('hint', None, dispatch=1)
-add_operator('assign', 1, dispatch=1)
 
 class Contains(SingleDispatchMixin, PureOperation):
     opname = 'contains'
@@ -453,10 +452,10 @@
         result = self.constfold()
         if result is not None:
             return result
+        ctx.merge_point()
         from rpython.annotator.expression import V_Type
         v_instance, = self.args
-        result = V_Type(v_instance)
-        return ctx.do_op(op.assign(result))
+        return V_Type(v_instance)
 
 
 class NewDict(HLOperation):
diff --git a/rpython/translator/simplify.py b/rpython/translator/simplify.py
--- a/rpython/translator/simplify.py
+++ b/rpython/translator/simplify.py
@@ -10,6 +10,7 @@
 from rpython.flowspace.model import (Variable, Constant,
                                      c_last_exception, checkgraph, mkentrymap)
 from rpython.flowspace.operation import OverflowingOperation, op
+from rpython.annotator.expression import V_Type
 from rpython.rlib import rarithmetic
 from rpython.translator import unsimplify
 from rpython.translator.backendopt import ssa
@@ -70,6 +71,8 @@
             if not block1.exits:
                 break
             exit = block1.exits[0]
+            if any(isinstance(arg, V_Type) for arg in exit.args):
+                break
             assert block1 is not exit.target, (
                 "the graph contains an empty infinite loop")
             subst = dict(zip(block1.inputargs, link.args))
@@ -256,7 +259,8 @@
             for vprev, vtarg in zip(link.args, link.target.inputargs):
                 renaming[vtarg] = vprev
             def rename(v):
-                return renaming.get(v, v)
+                if v is not None:
+                    return v.replace(renaming)
             def rename_op(op):
                 op = op.replace(renaming)
                 # special case...
diff --git a/rpython/translator/transform.py b/rpython/translator/transform.py
--- a/rpython/translator/transform.py
+++ b/rpython/translator/transform.py
@@ -8,6 +8,7 @@
 from rpython.flowspace.model import SpaceOperation
 from rpython.flowspace.model import Variable, Constant, Link
 from rpython.flowspace.model import c_last_exception, checkgraph
+from rpython.annotator.expression import V_Type
 from rpython.annotator import model as annmodel
 from rpython.rtyper.lltypesystem import lltype
 
@@ -134,15 +135,43 @@
                     s_dict = self.annotation(op.args[0])
                     s_dict.dictdef.generalize_key(self.binding(op.args[1]))
 
-def remove_assign(ann, block_subset):
+def remove_expr(ann, block_subset):
     for block in block_subset:
-        for i in range(len(block.operations)):
+        i = 0
+        while i < len(block.operations):
             op = block.operations[i]
-            if op.opname == 'assign':
-                new_op = op.args[0].as_operation()
-                new_op.result = op.result
-                block.operations[i] = new_op
-
+            if any(isinstance(arg, V_Type) for arg in op.args):
+                new_args = []
+                new_ops = []
+                for arg in op.args:
+                    if isinstance(arg, V_Type):
+                        new_op = arg.as_operation()
+                        new_op.result.annotation = arg.annotation
+                        new_ops.append(new_op)
+                        new_args.append(new_op.result)
+                    else:
+                        new_args.append(arg)
+                op.args = new_args
+                block.operations[i:i] = new_ops
+            else:
+                i += 1
+        for exit in block.exits:
+            if any(isinstance(arg, V_Type) for arg in exit.args):
+                new_args = []
+                new_ops = []
+                for arg in exit.args:
+                    if isinstance(arg, V_Type):
+                        new_op = arg.as_operation()
+                        new_op.result.annotation = arg.annotation
+                        new_ops.append(new_op)
+                        new_args.append(new_op.result)
+                    else:
+                        new_args.append(arg)
+                exit.args = new_args
+                if block.exitswitch == c_last_exception:
+                    block.operations[-1:-1] = new_ops
+                else:
+                    block.operations.extend(new_ops)
 
 
 def transform_dead_op_vars(self, block_subset):
@@ -259,7 +288,7 @@
     transform_extend_with_str_slice,
     transform_extend_with_char_count,
     transform_list_contains,
-    remove_assign,
+    remove_expr,
     ]
 
 def transform_graph(ann, extra_passes=None, block_subset=None):

From noreply at buildbot.pypy.org  Tue Nov 11 02:39:04 2014
From: noreply at buildbot.pypy.org (rlamy)
Date: Tue, 11 Nov 2014 02:39:04 +0100 (CET)
Subject: [pypy-commit] pypy expressions: fix checkgraph()
Message-ID: <20141111013904.E11951C354A@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy 
Branch: expressions
Changeset: r74438:d811881dde09
Date: 2014-11-09 03:04 +0000
http://bitbucket.org/pypy/pypy/changeset/d811881dde09/

Log:	fix checkgraph()

diff --git a/rpython/flowspace/model.py b/rpython/flowspace/model.py
--- a/rpython/flowspace/model.py
+++ b/rpython/flowspace/model.py
@@ -572,10 +572,11 @@
                 "variable %r used in more than one block" % (v,))
             vars[v] = only_in_link
 
-        def usevar(v, in_link=None):
-            assert v in vars
-            if in_link is not None:
-                assert vars[v] is None or vars[v] is in_link
+        def usevar(var, in_link=None):
+            for v in var.dependencies:
+                assert v in vars
+                if in_link is not None:
+                    assert vars[v] is None or vars[v] is in_link
 
 
         for block in graph.iterblocks():
@@ -593,10 +594,7 @@
                 for v in op.args:
                     assert isinstance(v, (Constant, Variable))
                     if isinstance(v, Variable):
-                        if type(v) is Variable:
-                            usevar(v)
-                        else:
-                            usevar(v.arg)
+                        usevar(v)
                     else:
                         assert v.value is not last_exception
                         #assert v.value != last_exc_value
@@ -666,7 +664,7 @@
                     assert link.last_exc_value is None
                 for v in link.args:
                     assert isinstance(v, (Constant, Variable))
-                    if type(v) is Variable:
+                    if isinstance(v, Variable):
                         usevar(v, in_link=link)
                         if exc_link:
                             assert v != block.operations[-1].result

From noreply at buildbot.pypy.org  Tue Nov 11 13:56:23 2014
From: noreply at buildbot.pypy.org (Raemi)
Date: Tue, 11 Nov 2014 13:56:23 +0100 (CET)
Subject: [pypy-commit] stmgc c8-new-page-handling: fix deadlock
Message-ID: <20141111125623.118D81D2859@cobra.cs.uni-duesseldorf.de>

Author: Remi Meier 
Branch: c8-new-page-handling
Changeset: r1503:15d0affd8675
Date: 2014-11-11 13:56 +0100
http://bitbucket.org/pypy/stmgc/changeset/15d0affd8675/

Log:	fix deadlock

diff --git a/c8/stm/core.c b/c8/stm/core.c
--- a/c8/stm/core.c
+++ b/c8/stm/core.c
@@ -157,6 +157,8 @@
     page_privatize_in(my_segnum, pagenum);
     assert(get_page_status_in(my_segnum, pagenum) == PAGE_PRIVATE);
 
+    /* before copying anything, acquire modification locks from our and
+       the other segment */
     uint64_t to_lock = (1UL << copy_from_segnum)| (1UL << my_segnum);
     acquire_modification_lock_set(to_lock);
     pagecopy((char*)(get_virt_page_of(my_segnum, pagenum) * 4096UL),
@@ -169,7 +171,6 @@
        might need a walk into the past. */
     struct stm_commit_log_entry_s *src_version, *target_version;
     src_version = get_priv_segment(copy_from_segnum)->last_commit_log_entry;
-    OPT_ASSERT(src_version->rev_num == most_recent_rev);
     target_version = STM_PSEGMENT->last_commit_log_entry;
 
 
@@ -229,7 +230,7 @@
 
     fprintf(stderr, "commit log:\n");
     while ((cl = cl->next)) {
-        if (cl == (void *)-1) {
+        if (cl == INEV_RUNNING) {
             fprintf(stderr, "  INEVITABLE\n");
             return;
         }
@@ -262,7 +263,7 @@
     /* Don't check this 'cl'. This entry is already checked */
 
     if (STM_PSEGMENT->transaction_state == TS_INEVITABLE) {
-        assert(first_cl->next == (void*)-1);
+        assert(first_cl->next == INEV_RUNNING);
         return;
     }
 
@@ -270,13 +271,13 @@
     uint64_t segments_to_lock = 1UL << my_segnum;
     cl = first_cl;
     while ((next_cl = cl->next) != NULL) {
-        if (next_cl == (void *)-1) {
+        if (next_cl == INEV_RUNNING) {
 #if STM_TESTS
             if (free_if_abort != (void *)-1)
                 free(free_if_abort);
             stm_abort_transaction();
 #endif
-            /* only go this far when validating */
+            /* only validate entries up to INEV */
             break;
         }
         assert(next_cl->rev_num > cl->rev_num);
@@ -324,7 +325,10 @@
 
             /* last fully validated entry */
             STM_PSEGMENT->last_commit_log_entry = cl;
+            if (cl == last_cl)
+                break;
         }
+        assert(cl == last_cl);
 
         OPT_ASSERT(segment_really_copied_from < (1 << NB_SEGMENTS));
         int segnum;
@@ -346,6 +350,8 @@
     if (needs_abort) {
         if (free_if_abort != (void *)-1)
             free(free_if_abort);
+        /* pages may be inconsistent */
+
         stm_abort_transaction();
     }
 }
@@ -381,18 +387,25 @@
         /* try to attach to commit log: */
         old = STM_PSEGMENT->last_commit_log_entry;
         if (old->next == NULL) {
-            if ((uintptr_t)new != -1) /* INEVITABLE */
+            if (new != INEV_RUNNING) /* INEVITABLE */
                 new->rev_num = old->rev_num + 1;
 
             if (__sync_bool_compare_and_swap(&old->next, NULL, new))
                 break;   /* success! */
+        } else if (old->next == INEV_RUNNING) {
+            /* we failed because there is an INEV transaction running */
+            usleep(10);
         }
+
+        /* check for requested safe point. otherwise an INEV transaction
+           may try to commit but cannot because of the busy-loop here. */
+        _stm_collectable_safe_point();
     }
 }
 
 static void _validate_and_turn_inevitable(void)
 {
-    _validate_and_attach((struct stm_commit_log_entry_s *)-1);
+    _validate_and_attach((struct stm_commit_log_entry_s *)INEV_RUNNING);
 }
 
 static void _validate_and_add_to_commit_log(void)
@@ -403,9 +416,9 @@
     if (STM_PSEGMENT->transaction_state == TS_INEVITABLE) {
         old = STM_PSEGMENT->last_commit_log_entry;
         new->rev_num = old->rev_num + 1;
-        OPT_ASSERT(old->next == (void *)-1);
+        OPT_ASSERT(old->next == INEV_RUNNING);
 
-        bool yes = __sync_bool_compare_and_swap(&old->next, (void*)-1, new);
+        bool yes = __sync_bool_compare_and_swap(&old->next, INEV_RUNNING, new);
         OPT_ASSERT(yes);
     }
     else {
diff --git a/c8/stm/core.h b/c8/stm/core.h
--- a/c8/stm/core.h
+++ b/c8/stm/core.h
@@ -123,6 +123,7 @@
    data from 'written[n].backup' is the content of (slices of) the
    object as they were *before* that commit occurred.
 */
+#define INEV_RUNNING ((void*)-1)
 struct stm_commit_log_entry_s {
     struct stm_commit_log_entry_s *volatile next;
     int segment_num;

From noreply at buildbot.pypy.org  Tue Nov 11 14:32:49 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Tue, 11 Nov 2014 14:32:49 +0100 (CET)
Subject: [pypy-commit] pypy stmgc-c7: tweak _stm.count()
Message-ID: <20141111133249.602AC1C129D@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: stmgc-c7
Changeset: r74443:9cebce388452
Date: 2014-11-11 14:32 +0100
http://bitbucket.org/pypy/pypy/changeset/9cebce388452/

Log:	tweak _stm.count()

diff --git a/pypy/module/_stm/count.py b/pypy/module/_stm/count.py
--- a/pypy/module/_stm/count.py
+++ b/pypy/module/_stm/count.py
@@ -6,7 +6,8 @@
 
 
 def count(space):
-    """Return a new integer every time it is called,
-without generating conflicts."""
+    """Return a different positive integer every time it is called.
+This works without generating conflicts.  The returned integers are
+only roughly in increasing order; this should not be relied upon."""
     count = rstm.stm_count()
     return space.wrap(count)
diff --git a/rpython/translator/stm/src_stm/stmgcintf.c b/rpython/translator/stm/src_stm/stmgcintf.c
--- a/rpython/translator/stm/src_stm/stmgcintf.c
+++ b/rpython/translator/stm/src_stm/stmgcintf.c
@@ -224,6 +224,6 @@
 
 long _pypy_stm_count(void)
 {
-    static long value = 0;
+    static long value = 1;
     return __sync_fetch_and_add(&value, 1);
 }

From noreply at buildbot.pypy.org  Tue Nov 11 14:47:36 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Tue, 11 Nov 2014 14:47:36 +0100 (CET)
Subject: [pypy-commit] pypy stmgc-c7: Light finalizers seem to be missing
 nothing more than being enabled.
Message-ID: <20141111134736.846C31C07BB@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: stmgc-c7
Changeset: r74444:b72126002e43
Date: 2014-11-11 14:47 +0100
http://bitbucket.org/pypy/pypy/changeset/b72126002e43/

Log:	Light finalizers seem to be missing nothing more than being enabled.
	We'll see if this theory breaks down.

diff --git a/rpython/memory/gc/stmgc.py b/rpython/memory/gc/stmgc.py
--- a/rpython/memory/gc/stmgc.py
+++ b/rpython/memory/gc/stmgc.py
@@ -78,8 +78,8 @@
         if contains_weakptr:    # check constant-folded
             return llop.stm_allocate_weakref(llmemory.GCREF, size, typeid16)
         if needs_finalizer:
-            #if is_finalizer_light:   XXX implement me
-            #   return llop.stm_allocate_f_light(llmemory.GCREF, size, typeid16)
+            if is_finalizer_light:
+                return llop.stm_allocate_f_light(llmemory.GCREF, size, typeid16)
             return llop.stm_allocate_finalizer(llmemory.GCREF, size, typeid16)
         return llop.stm_allocate_tid(llmemory.GCREF, size, typeid16)
 
diff --git a/rpython/memory/gctransform/framework.py b/rpython/memory/gctransform/framework.py
--- a/rpython/memory/gctransform/framework.py
+++ b/rpython/memory/gctransform/framework.py
@@ -1289,11 +1289,8 @@
                 [llmemory.Address], lltype.Void)
         try:
             g = destrptr._obj.graph
-            if self.translator.config.translation.stm:
-                light = False  # XXX no working light finalizers with STM so far
-            else:
-                analyzer = FinalizerAnalyzer(self.translator)
-                light = not analyzer.analyze_light_finalizer(g)
+            analyzer = FinalizerAnalyzer(self.translator)
+            light = not analyzer.analyze_light_finalizer(g)
         except lltype.DelayedPointer:
             light = False    # XXX bah, too bad
         return fptr, light
diff --git a/rpython/translator/stm/funcgen.py b/rpython/translator/stm/funcgen.py
--- a/rpython/translator/stm/funcgen.py
+++ b/rpython/translator/stm/funcgen.py
@@ -106,7 +106,7 @@
     return ('%s = (rpygcchar_t *)stm_allocate(%s); ' % (
         result, arg_size) +
             '((rpyobj_t *)%s)->tid = %s;\n' % (result, arg_type_id) +
-            'stm_enable_light_finalizer(%s);' % (result,))
+            'stm_enable_light_finalizer((object_t *)%s);' % (result,))
 
 def stm_get_from_obj(funcgen, op):
     assert op.args[0].concretetype == llmemory.GCREF
diff --git a/rpython/translator/stm/src_stm/extracode.h b/rpython/translator/stm/src_stm/extracode.h
--- a/rpython/translator/stm/src_stm/extracode.h
+++ b/rpython/translator/stm/src_stm/extracode.h
@@ -1,6 +1,7 @@
 
 static void _stm_call_finalizer(object_t *obj)
 {
+    /* this function works for both regular and light finalizers */
     void *funcptr = pypy_stmcb_fetch_finalizer(((rpyobj_t *)obj)->tid);
     ((void(*)(object_t *))funcptr)(obj);
 }
@@ -31,6 +32,7 @@
     }
 
     stmcb_finalizer = &_stm_call_finalizer;
+    stmcb_light_finalizer = &_stm_call_finalizer;
 }
 
 void pypy_stm_register_thread_local(void)
diff --git a/rpython/translator/stm/test/test_ztranslated.py b/rpython/translator/stm/test/test_ztranslated.py
--- a/rpython/translator/stm/test/test_ztranslated.py
+++ b/rpython/translator/stm/test/test_ztranslated.py
@@ -535,7 +535,6 @@
         assert 'destructors called: 1\n' in data
 
     def test_light_finalizer(self):
-        py.test.skip("XXX implement me")
         class X:
             @rgc.must_be_light_finalizer
             def __del__(self):

From noreply at buildbot.pypy.org  Tue Nov 11 16:21:25 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Tue, 11 Nov 2014 16:21:25 +0100 (CET)
Subject: [pypy-commit] pypy default: Add 'cast_gcref_to_instance' here. The
 JIT has code that does
Message-ID: <20141111152125.11BAF1C03DC@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r74445:5171dd85f03b
Date: 2014-11-11 15:15 +0100
http://bitbucket.org/pypy/pypy/changeset/5171dd85f03b/

Log:	Add 'cast_gcref_to_instance' here. The JIT has code that does
	indirectly the same thing, but it's cleaner to have a direct inverse
	of 'cast_instance_to_gcref'.

diff --git a/rpython/rtyper/annlowlevel.py b/rpython/rtyper/annlowlevel.py
--- a/rpython/rtyper/annlowlevel.py
+++ b/rpython/rtyper/annlowlevel.py
@@ -513,6 +513,13 @@
                                   % (ptr, Class))
     return ptr
 
+ at specialize.arg(0)
+def cast_gcref_to_instance(Class, ptr):
+    """Reverse the hacking done in cast_instance_to_gcref()."""
+    from rpython.rtyper.rclass import OBJECTPTR
+    ptr = lltype.cast_opaque_ptr(OBJECTPTR, ptr)
+    return cast_base_ptr_to_instance(Class, ptr)
+
 class CastBasePtrToInstanceEntry(extregistry.ExtRegistryEntry):
     _about_ = cast_base_ptr_to_instance
 
diff --git a/rpython/rtyper/test/test_annlowlevel.py b/rpython/rtyper/test/test_annlowlevel.py
--- a/rpython/rtyper/test/test_annlowlevel.py
+++ b/rpython/rtyper/test/test_annlowlevel.py
@@ -4,7 +4,7 @@
 
 from rpython.rtyper.test.tool import BaseRtypingTest
 from rpython.rtyper.lltypesystem.rstr import mallocstr, mallocunicode
-from rpython.rtyper.lltypesystem import lltype
+from rpython.rtyper.lltypesystem import lltype, llmemory
 from rpython.rtyper.annlowlevel import hlstr, llstr
 from rpython.rtyper.annlowlevel import hlunicode, llunicode
 from rpython.rtyper import annlowlevel
@@ -73,6 +73,15 @@
         y = annlowlevel.cast_base_ptr_to_instance(X, ptr)
         assert y is x
 
+    def test_cast_instance_to_gcref(self):
+        class X(object):
+            pass
+        x = X()
+        ptr = annlowlevel.cast_instance_to_gcref(x)
+        assert lltype.typeOf(ptr) == llmemory.GCREF
+        y = annlowlevel.cast_gcref_to_instance(X, ptr)
+        assert y is x
+
     def test_delayedptr(self):
         FUNCTYPE = lltype.FuncType([], lltype.Signed)
         name = "delayed!myfunc"

From noreply at buildbot.pypy.org  Tue Nov 11 16:21:26 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Tue, 11 Nov 2014 16:21:26 +0100 (CET)
Subject: [pypy-commit] pypy stmgc-c7: Add 'cast_gcref_to_instance' here. The
 JIT has code that does
Message-ID: <20141111152126.595D31C03DC@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: stmgc-c7
Changeset: r74446:d93251afcbd4
Date: 2014-11-11 15:15 +0100
http://bitbucket.org/pypy/pypy/changeset/d93251afcbd4/

Log:	Add 'cast_gcref_to_instance' here. The JIT has code that does
	indirectly the same thing, but it's cleaner to have a direct inverse
	of 'cast_instance_to_gcref'. (grafted from
	5171dd85f03b3b2eea33c6402e12a4a9d6129d9d)

diff --git a/rpython/rtyper/annlowlevel.py b/rpython/rtyper/annlowlevel.py
--- a/rpython/rtyper/annlowlevel.py
+++ b/rpython/rtyper/annlowlevel.py
@@ -509,6 +509,13 @@
                                   % (ptr, Class))
     return ptr
 
+ at specialize.arg(0)
+def cast_gcref_to_instance(Class, ptr):
+    """Reverse the hacking done in cast_instance_to_gcref()."""
+    from rpython.rtyper.rclass import OBJECTPTR
+    ptr = lltype.cast_opaque_ptr(OBJECTPTR, ptr)
+    return cast_base_ptr_to_instance(Class, ptr)
+
 class CastBasePtrToInstanceEntry(extregistry.ExtRegistryEntry):
     _about_ = cast_base_ptr_to_instance
 
diff --git a/rpython/rtyper/test/test_annlowlevel.py b/rpython/rtyper/test/test_annlowlevel.py
--- a/rpython/rtyper/test/test_annlowlevel.py
+++ b/rpython/rtyper/test/test_annlowlevel.py
@@ -4,7 +4,7 @@
 
 from rpython.rtyper.test.tool import BaseRtypingTest
 from rpython.rtyper.lltypesystem.rstr import mallocstr, mallocunicode
-from rpython.rtyper.lltypesystem import lltype
+from rpython.rtyper.lltypesystem import lltype, llmemory
 from rpython.rtyper.annlowlevel import hlstr, llstr
 from rpython.rtyper.annlowlevel import hlunicode, llunicode
 from rpython.rtyper import annlowlevel
@@ -65,6 +65,15 @@
         y = annlowlevel.cast_base_ptr_to_instance(X, ptr)
         assert y is x
 
+    def test_cast_instance_to_gcref(self):
+        class X(object):
+            pass
+        x = X()
+        ptr = annlowlevel.cast_instance_to_gcref(x)
+        assert lltype.typeOf(ptr) == llmemory.GCREF
+        y = annlowlevel.cast_gcref_to_instance(X, ptr)
+        assert y is x
+
     def test_delayedptr(self):
         FUNCTYPE = lltype.FuncType([], lltype.Signed)
         name = "delayed!myfunc"

From noreply at buildbot.pypy.org  Tue Nov 11 16:21:27 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Tue, 11 Nov 2014 16:21:27 +0100 (CET)
Subject: [pypy-commit] pypy stmgc-c7: Mostly implement stmgc's hashtables in
	RPython.
Message-ID: <20141111152127.8A28A1C03DC@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: stmgc-c7
Changeset: r74447:55d1033012f1
Date: 2014-11-11 16:20 +0100
http://bitbucket.org/pypy/pypy/changeset/55d1033012f1/

Log:	Mostly implement stmgc's hashtables in RPython.

diff --git a/rpython/memory/gc/stmgc.py b/rpython/memory/gc/stmgc.py
--- a/rpython/memory/gc/stmgc.py
+++ b/rpython/memory/gc/stmgc.py
@@ -102,8 +102,8 @@
 
 
     def can_move(self, obj):
-        """Means the reference will stay valid, except if not
-        seen by the GC, then it can get collected."""
+        """An object that cannot move means the pointer to it will stay valid,
+        as long as it is not actually collected."""
         return llop.stm_can_move(lltype.Bool, obj)
 
 
diff --git a/rpython/rlib/rstm.py b/rpython/rlib/rstm.py
--- a/rpython/rlib/rstm.py
+++ b/rpython/rlib/rstm.py
@@ -1,6 +1,7 @@
 from rpython.rlib.objectmodel import we_are_translated, specialize
 from rpython.rlib.objectmodel import CDefinedIntSymbolic
-from rpython.rlib.rgc import stm_is_enabled
+from rpython.rlib.nonconst import NonConstant
+from rpython.rlib import rgc
 from rpython.rtyper.lltypesystem import lltype, rffi, rstr, llmemory
 from rpython.rtyper.lltypesystem.lloperation import llop
 from rpython.rtyper.extregistry import ExtRegistryEntry
@@ -54,7 +55,7 @@
     """ keep: should be True for checks that are absolutely
     needed. False means the JIT only keeps the check if it
     thinks that it helps """
-    if stm_is_enabled():
+    if rgc.stm_is_enabled():
         if llop.stm_should_break_transaction(lltype.Bool, keep):
             break_transaction()
 
@@ -171,3 +172,67 @@
         hop.exception_cannot_occur()
         if hop.rtyper.annotator.translator.config.translation.stm:
             hop.genop('stm_rewind_jmp_frame', [], resulttype=lltype.Void)
+
+# ____________________________________________________________
+
+_STM_HASHTABLE_P = rffi.COpaquePtr('stm_hashtable_t')
+
+_STM_HASHTABLE_ENTRY = lltype.GcStruct('HASHTABLE_ENTRY',
+                                       ('index', lltype.Unsigned),
+                                       ('object', llmemory.GCREF))
+
+
+class Hashtable(object):
+
+    def __new__(cls):
+        "NOT_RPYTHON: for tests, return a HashtableForTest instance"
+        return HashtableForTest()
+
+    def __init__(self):
+        # Pass a null pointer to _STM_HASHTABLE_ENTRY to stm_hashtable_create().
+        # Make sure we see a malloc() of it, so that its typeid is correctly
+        # initialized.  It can be done in a NonConstant(False) path so that
+        # the C compiler will actually drop it.
+        if NonConstant(False):
+            p = lltype.malloc(_STM_HASHTABLE_ENTRY)
+        else:
+            p = lltype.nullptr(_STM_HASHTABLE_ENTRY)
+        self.ll_raw_hashtable = llop.stm_hashtable_create(_STM_HASHTABLE_P, p)
+
+    @rgc.must_be_light_finalizer
+    def __del__(self):
+        llop.stm_hashtable_free(lltype.Void, self.ll_raw_hashtable)
+
+    def get(self, key):
+        # 'key' must be a plain integer.  Returns a GCREF.
+        return llop.stm_hashtable_read(llmemory.GCREF, self,
+                                       self.ll_raw_hashtable, key)
+
+    def set(self, key, value):
+        llop.stm_hashtable_write(lltype.Void, self,
+                                 self.ll_raw_hashtable, key, value)
+
+
+class HashtableForTest(object):
+    _NULL = lltype.nullptr(llmemory.GCREF.TO)
+
+    def __init__(self):
+        self._content = {}      # dict {integer: GCREF}
+
+    def _cleanup_(self):
+        raise Exception("cannot translate a prebuilt rstm.Hashtable object")
+
+    def get(self, key):
+        assert type(key) is int
+        return self._content.get(key, self._NULL)
+
+    def set(self, key, value):
+        assert type(key) is int
+        assert lltype.typeOf(value) == llmemory.GCREF
+        if value:
+            self._content[key] = value
+        else:
+            try:
+                del self._content[key]
+            except KeyError:
+                pass
diff --git a/rpython/rtyper/annlowlevel.py b/rpython/rtyper/annlowlevel.py
--- a/rpython/rtyper/annlowlevel.py
+++ b/rpython/rtyper/annlowlevel.py
@@ -512,7 +512,7 @@
 @specialize.arg(0)
 def cast_gcref_to_instance(Class, ptr):
     """Reverse the hacking done in cast_instance_to_gcref()."""
-    from rpython.rtyper.rclass import OBJECTPTR
+    from rpython.rtyper.lltypesystem.rclass import OBJECTPTR
     ptr = lltype.cast_opaque_ptr(OBJECTPTR, ptr)
     return cast_base_ptr_to_instance(Class, ptr)
 
diff --git a/rpython/rtyper/lltypesystem/lloperation.py b/rpython/rtyper/lltypesystem/lloperation.py
--- a/rpython/rtyper/lltypesystem/lloperation.py
+++ b/rpython/rtyper/lltypesystem/lloperation.py
@@ -461,6 +461,11 @@
     'stm_count':                 LLOp(canrun=True),
     'stm_really_force_cast_ptr': LLOp(),
 
+    'stm_hashtable_create':   LLOp(),
+    'stm_hashtable_free':     LLOp(),
+    'stm_hashtable_read':     LLOp(),
+    'stm_hashtable_write':    LLOp(),
+
     # __________ address operations __________
 
     'boehm_malloc':         LLOp(),
diff --git a/rpython/translator/backendopt/finalizer.py b/rpython/translator/backendopt/finalizer.py
--- a/rpython/translator/backendopt/finalizer.py
+++ b/rpython/translator/backendopt/finalizer.py
@@ -18,7 +18,7 @@
     """
     ok_operations = ['ptr_nonzero', 'ptr_eq', 'ptr_ne', 'free', 'same_as',
                      'direct_ptradd', 'force_cast', 'track_alloc_stop',
-                     'raw_free', 'debug_print']
+                     'raw_free', 'debug_print', 'stm_hashtable_free']
 
     def analyze_light_finalizer(self, graph):
         result = self.analyze_direct_call(graph)
diff --git a/rpython/translator/stm/funcgen.py b/rpython/translator/stm/funcgen.py
--- a/rpython/translator/stm/funcgen.py
+++ b/rpython/translator/stm/funcgen.py
@@ -283,3 +283,31 @@
     arg = funcgen.expr(op.args[0])
     typename = cdecl(funcgen.lltypename(op.result), '')
     return '%s = (%s)(uintptr_t)%s;' % (result, typename, arg)
+
+def stm_hashtable_create(funcgen, op):
+    _STM_HASHTABLE_ENTRY = op.args[0].concretetype.TO
+    type_id = funcgen.db.gctransformer.get_type_id(_STM_HASHTABLE_ENTRY)
+    expr_type_id = funcgen.expr(Constant(type_id, lltype.typeOf(type_id)))
+    result = funcgen.expr(op.result)
+    return ('stm_hashtable_entry_userdata = %s; '
+            '%s = stm_hashtable_create();' % (expr_type_id, result,))
+
+def stm_hashtable_free(funcgen, op):
+    arg = funcgen.expr(op.args[0])
+    return 'stm_hashtable_free(%s);' % (arg,)
+
+def stm_hashtable_read(funcgen, op):
+    arg0 = funcgen.expr(op.args[0])
+    arg1 = funcgen.expr(op.args[1])
+    arg2 = funcgen.expr(op.args[2])
+    result = funcgen.expr(op.result)
+    return '%s = (rpygcchar_t *)stm_hashtable_read((object_t *)%s, %s, %s);' % (
+        result, arg0, arg1, arg2)
+
+def stm_hashtable_write(funcgen, op):
+    arg0 = funcgen.expr(op.args[0])
+    arg1 = funcgen.expr(op.args[1])
+    arg2 = funcgen.expr(op.args[2])
+    arg3 = funcgen.expr(op.args[3])
+    return ('stm_hashtable_write((object_t *)%s, %s, %s, (object_t *)%s, '
+            '&stm_thread_local);' % (arg0, arg1, arg2, arg3))
diff --git a/rpython/translator/stm/test/test_ztranslated.py b/rpython/translator/stm/test/test_ztranslated.py
--- a/rpython/translator/stm/test/test_ztranslated.py
+++ b/rpython/translator/stm/test/test_ztranslated.py
@@ -4,6 +4,8 @@
 from rpython.rtyper.lltypesystem import lltype, llmemory, rffi
 from rpython.rtyper.lltypesystem.rclass import OBJECTPTR
 from rpython.rtyper.lltypesystem.lloperation import llop
+from rpython.rtyper.annlowlevel import cast_instance_to_gcref
+from rpython.rtyper.annlowlevel import cast_gcref_to_instance
 from rpython.translator.stm.test.support import CompiledSTMTests
 from rpython.translator.stm.test import targetdemo2
 
@@ -549,3 +551,38 @@
         t, cbuilder = self.compile(main)
         data, err = cbuilder.cmdexec('', err=True)
         assert '' in err
+
+    def test_hashtable(self):
+        py.test.skip("missing: custom tracer on Hashtable")
+
+        class X(object):
+            pass
+
+        def main(argv):
+            h = rstm.Hashtable()
+            p = h.get(-1234)
+            assert p == lltype.nullptr(llmemory.GCREF.TO)
+            #
+            x1 = X()
+            p1 = cast_instance_to_gcref(x1)
+            h.set(-1234, p1)
+            #
+            p2 = h.get(-1234)
+            x2 = cast_gcref_to_instance(X, p2)
+            assert x2 is x1
+            #
+            rgc.collect()
+            #
+            p2 = h.get(-1234)
+            x2 = cast_gcref_to_instance(X, p2)
+            assert x2 is x1
+            #
+            print "ok!"
+            return 0
+
+        res = main([])      # direct run
+        assert res == 0
+
+        t, cbuilder = self.compile(main)
+        data = cbuilder.cmdexec('')
+        assert 'ok!\n' in data

From noreply at buildbot.pypy.org  Tue Nov 11 17:18:42 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Tue, 11 Nov 2014 17:18:42 +0100 (CET)
Subject: [pypy-commit] pypy stmgc-c7: llify the class Hashtable,
 in preparation for a custom tracing hook.
Message-ID: <20141111161842.3D1C41C02CB@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: stmgc-c7
Changeset: r74448:76875c1d32f0
Date: 2014-11-11 17:18 +0100
http://bitbucket.org/pypy/pypy/changeset/76875c1d32f0/

Log:	llify the class Hashtable, in preparation for a custom tracing hook.

diff --git a/rpython/memory/gctransform/stmframework.py b/rpython/memory/gctransform/stmframework.py
--- a/rpython/memory/gctransform/stmframework.py
+++ b/rpython/memory/gctransform/stmframework.py
@@ -10,6 +10,15 @@
 from rpython.rlib import rstm
 
 
+def invokecallback(root, visit_fn):
+    """Used as a callback for gc.trace().  There is also a custom tracer
+    in rpython.rlib.rstm that checks if the callback it gets is exactly
+    'invokecallback', and if so, it knows that the 'arg' is actually a
+    C-level visit function.
+    """
+    visit_fn(root)
+
+
 class StmFrameworkGCTransformer(BaseFrameworkGCTransformer):
 
     def _declare_functions(self, GCClass, getfn, s_gc, s_typeid16):
@@ -24,8 +33,6 @@
             getfn(pypy_stmcb_size_rounded_up, [llannotation.SomeAddress()],
                   annmodel.SomeInteger()))
         #
-        def invokecallback(root, visit_fn):
-            visit_fn(root)
         def pypy_stmcb_trace(obj, visit_fn):
             gc.trace(obj, invokecallback, visit_fn)
         pypy_stmcb_trace.c_name = "pypy_stmcb_trace"
diff --git a/rpython/rlib/rstm.py b/rpython/rlib/rstm.py
--- a/rpython/rlib/rstm.py
+++ b/rpython/rlib/rstm.py
@@ -181,37 +181,34 @@
                                        ('index', lltype.Unsigned),
                                        ('object', llmemory.GCREF))
 
+def ll_hashtable_get(h, key):
+    # 'key' must be a plain integer.  Returns a GCREF.
+    return llop.stm_hashtable_read(llmemory.GCREF, h, h.ll_raw_hashtable, key)
 
-class Hashtable(object):
+def ll_hashtable_set(h, key, value):
+    llop.stm_hashtable_write(lltype.Void, h, h.ll_raw_hashtable, key, value)
 
-    def __new__(cls):
-        "NOT_RPYTHON: for tests, return a HashtableForTest instance"
-        return HashtableForTest()
+_HASHTABLE_OBJ = lltype.GcStruct('HASHTABLE_OBJ',
+                                 ('ll_raw_hashtable', _STM_HASHTABLE_P),
+                                 adtmeths={'get': ll_hashtable_get,
+                                           'set': ll_hashtable_set})
 
-    def __init__(self):
-        # Pass a null pointer to _STM_HASHTABLE_ENTRY to stm_hashtable_create().
-        # Make sure we see a malloc() of it, so that its typeid is correctly
-        # initialized.  It can be done in a NonConstant(False) path so that
-        # the C compiler will actually drop it.
-        if NonConstant(False):
-            p = lltype.malloc(_STM_HASHTABLE_ENTRY)
-        else:
-            p = lltype.nullptr(_STM_HASHTABLE_ENTRY)
-        self.ll_raw_hashtable = llop.stm_hashtable_create(_STM_HASHTABLE_P, p)
+#def ll_hashtable_trace(...)
 
-    @rgc.must_be_light_finalizer
-    def __del__(self):
-        llop.stm_hashtable_free(lltype.Void, self.ll_raw_hashtable)
-
-    def get(self, key):
-        # 'key' must be a plain integer.  Returns a GCREF.
-        return llop.stm_hashtable_read(llmemory.GCREF, self,
-                                       self.ll_raw_hashtable, key)
-
-    def set(self, key, value):
-        llop.stm_hashtable_write(lltype.Void, self,
-                                 self.ll_raw_hashtable, key, value)
-
+def create_hashtable():
+    if not we_are_translated():
+        return HashtableForTest()      # for tests
+    # Pass a null pointer to _STM_HASHTABLE_ENTRY to stm_hashtable_create().
+    # Make sure we see a malloc() of it, so that its typeid is correctly
+    # initialized.  It can be done in a NonConstant(False) path so that
+    # the C compiler will actually drop it.
+    if NonConstant(False):
+        p = lltype.malloc(_STM_HASHTABLE_ENTRY)
+    else:
+        p = lltype.nullptr(_STM_HASHTABLE_ENTRY)
+    h = lltype.malloc(_HASHTABLE_OBJ)
+    h.ll_raw_hashtable = llop.stm_hashtable_create(_STM_HASHTABLE_P, p)
+    return h
 
 class HashtableForTest(object):
     _NULL = lltype.nullptr(llmemory.GCREF.TO)
diff --git a/rpython/translator/stm/test/test_ztranslated.py b/rpython/translator/stm/test/test_ztranslated.py
--- a/rpython/translator/stm/test/test_ztranslated.py
+++ b/rpython/translator/stm/test/test_ztranslated.py
@@ -559,7 +559,7 @@
             pass
 
         def main(argv):
-            h = rstm.Hashtable()
+            h = rstm.create_hashtable()
             p = h.get(-1234)
             assert p == lltype.nullptr(llmemory.GCREF.TO)
             #

From noreply at buildbot.pypy.org  Tue Nov 11 18:09:13 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Tue, 11 Nov 2014 18:09:13 +0100 (CET)
Subject: [pypy-commit] pypy stmgc-c7: in-progress: prepare for merging
 improve-gc-tracing-hooks
Message-ID: <20141111170913.1544D1C02CB@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: stmgc-c7
Changeset: r74449:03d55ba443a0
Date: 2014-11-11 18:02 +0100
http://bitbucket.org/pypy/pypy/changeset/03d55ba443a0/

Log:	in-progress: prepare for merging improve-gc-tracing-hooks

diff --git a/rpython/memory/gctransform/stmframework.py b/rpython/memory/gctransform/stmframework.py
--- a/rpython/memory/gctransform/stmframework.py
+++ b/rpython/memory/gctransform/stmframework.py
@@ -5,19 +5,39 @@
 from rpython.memory.gctransform.framework import (TYPE_ID,
      BaseFrameworkGCTransformer, BaseRootWalker, sizeofaddr)
 from rpython.memory.gctypelayout import WEAKREF, WEAKREFPTR
+from rpython.memory.gc.stm import StmGC
 from rpython.rtyper import rmodel, llannotation
 from rpython.translator.backendopt.support import var_needsgc
+from rpython.rlib.objectmodel import specialize
 from rpython.rlib import rstm
 
 
+VISIT_FPTR = StmGC.VISIT_FPTR
+
 def invokecallback(root, visit_fn):
-    """Used as a callback for gc.trace().  There is also a custom tracer
-    in rpython.rlib.rstm that checks if the callback it gets is exactly
-    'invokecallback', and if so, it knows that the 'arg' is actually a
-    C-level visit function.
-    """
+    """Used as a callback for gc.trace()."""
     visit_fn(root)
 
+ at specialize.arg(0)
+def get_visit_function(callback, arg):
+    """Hack: take a 'callback, arg' pair received from RPython code
+    calling gc.trace(), and return a raw function pointer suitable for
+    calling the C code.  We hide the 'arg' in some global if needed."""
+    if callback is invokecallback:
+        return arg      # the arg is directly the 'visit_fn' in this case
+    raw_visit_glob = _get_raw_visit_glob(callback)
+    raw_visit_glob.arg = arg
+    return llhelper(VISIT_FPTR, raw_visit_glob.visit)
+
+ at specialize.memo()
+def _get_raw_visit_glob(callback):
+    class RawVisitGlob:
+        _alloc_flavor_ = "raw"
+    raw_visit_glob = RawVisitGlob()
+    raw_visit_glob.visit = lambda obj: callback(obj, raw_visit_glob.arg)
+    return _raw_visit_globs.setdefault(callback, raw_visit_glob)
+_raw_visit_globs = {}
+
 
 class StmFrameworkGCTransformer(BaseFrameworkGCTransformer):
 
@@ -38,7 +58,7 @@
         pypy_stmcb_trace.c_name = "pypy_stmcb_trace"
         self.autoregister_ptrs.append(
             getfn(pypy_stmcb_trace, [llannotation.SomeAddress(),
-                                     llannotation.SomePtr(GCClass.VISIT_FPTR)],
+                                     llannotation.SomePtr(VISIT_FPTR)],
                   annmodel.s_None))
         #
         def pypy_stmcb_obj_supports_cards(obj):
@@ -60,7 +80,7 @@
         self.autoregister_ptrs.append(
             getfn(pypy_stmcb_trace_cards,
                   [llannotation.SomeAddress(),
-                   llannotation.SomePtr(GCClass.VISIT_FPTR),
+                   llannotation.SomePtr(VISIT_FPTR),
                    annmodel.s_Int,
                    annmodel.s_Int],
                   annmodel.s_None))

From noreply at buildbot.pypy.org  Tue Nov 11 18:09:14 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Tue, 11 Nov 2014 18:09:14 +0100 (CET)
Subject: [pypy-commit] pypy default: Move this attribute to the Boehm
 subclass. For the Framework subclass
Message-ID: <20141111170914.4522B1C02CB@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r74450:a2aafac5b363
Date: 2014-11-11 18:08 +0100
http://bitbucket.org/pypy/pypy/changeset/a2aafac5b363/

Log:	Move this attribute to the Boehm subclass. For the Framework
	subclass it is overridden anyway from the GCClass.

diff --git a/rpython/jit/backend/llsupport/gc.py b/rpython/jit/backend/llsupport/gc.py
--- a/rpython/jit/backend/llsupport/gc.py
+++ b/rpython/jit/backend/llsupport/gc.py
@@ -55,7 +55,6 @@
 # ____________________________________________________________
 
 class GcLLDescription(GcCache):
-    malloc_zero_filled = True
 
     def __init__(self, gcdescr, translator=None, rtyper=None):
         GcCache.__init__(self, translator is not None, rtyper)
@@ -246,6 +245,7 @@
 
 class GcLLDescr_boehm(GcLLDescription):
     kind                  = 'boehm'
+    malloc_zero_filled    = True
     moving_gc             = False
     round_up              = False
     write_barrier_descr   = None

From noreply at buildbot.pypy.org  Tue Nov 11 18:37:10 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Tue, 11 Nov 2014 18:37:10 +0100 (CET)
Subject: [pypy-commit] pypy default: Use the correct GETFIELD_RAW instead of
 GETFIELD_GC, because they
Message-ID: <20141111173710.72DED1C129D@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r74451:014332b2ea4b
Date: 2014-11-11 18:28 +0100
http://bitbucket.org/pypy/pypy/changeset/014332b2ea4b/

Log:	Use the correct GETFIELD_RAW instead of GETFIELD_GC, because they
	are done on a raw address. Important for stm.

diff --git a/rpython/jit/backend/llsupport/rewrite.py b/rpython/jit/backend/llsupport/rewrite.py
--- a/rpython/jit/backend/llsupport/rewrite.py
+++ b/rpython/jit/backend/llsupport/rewrite.py
@@ -263,7 +263,7 @@
     def gen_malloc_frame(self, frame_info, frame, size_box):
         descrs = self.gc_ll_descr.getframedescrs(self.cpu)
         if self.gc_ll_descr.kind == 'boehm':
-            op0 = ResOperation(rop.GETFIELD_GC, [history.ConstInt(frame_info)],
+            op0 = ResOperation(rop.GETFIELD_RAW, [history.ConstInt(frame_info)],
                                size_box,
                                descr=descrs.jfi_frame_depth)
             self.newops.append(op0)
@@ -272,7 +272,7 @@
             self.handle_new_array(descrs.arraydescr, op1)
         else:
             # we read size in bytes here, not the length
-            op0 = ResOperation(rop.GETFIELD_GC, [history.ConstInt(frame_info)],
+            op0 = ResOperation(rop.GETFIELD_RAW, [history.ConstInt(frame_info)],
                                size_box,
                                descr=descrs.jfi_frame_size)
             self.newops.append(op0)
@@ -282,7 +282,7 @@
             # we need to explicitely zero all the gc fields, because
             # of the unusal malloc pattern
             extra_ops = [
-                ResOperation(rop.GETFIELD_GC, [history.ConstInt(frame_info)],
+                ResOperation(rop.GETFIELD_RAW, [history.ConstInt(frame_info)],
                              length_box, descr=descrs.jfi_frame_depth),
                 ResOperation(rop.SETFIELD_GC, [frame, self.c_zero],
                              None, descr=descrs.jf_extra_stack_depth),
diff --git a/rpython/jit/backend/llsupport/test/test_rewrite.py b/rpython/jit/backend/llsupport/test/test_rewrite.py
--- a/rpython/jit/backend/llsupport/test/test_rewrite.py
+++ b/rpython/jit/backend/llsupport/test/test_rewrite.py
@@ -981,10 +981,10 @@
         i2 = call_assembler(i0, f0, descr=casmdescr)
         """, """
         [i0, f0]
-        i1 = getfield_gc(ConstClass(frame_info), descr=jfi_frame_size)
+        i1 = getfield_raw(ConstClass(frame_info), descr=jfi_frame_size)
         p1 = call_malloc_nursery_varsize_frame(i1)
         setfield_gc(p1, 0, descr=tiddescr)
-        i2 = getfield_gc(ConstClass(frame_info), descr=jfi_frame_depth)
+        i2 = getfield_raw(ConstClass(frame_info), descr=jfi_frame_depth)
         setfield_gc(p1, 0, descr=jf_extra_stack_depth)
         setfield_gc(p1, NULL, descr=jf_savedata)
         setfield_gc(p1, NULL, descr=jf_force_descr)

From noreply at buildbot.pypy.org  Tue Nov 11 18:37:11 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Tue, 11 Nov 2014 18:37:11 +0100 (CET)
Subject: [pypy-commit] pypy default: Remove extraneous "-" sign.
Message-ID: <20141111173711.B6F241C129D@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r74452:f5a631ede713
Date: 2014-11-11 18:36 +0100
http://bitbucket.org/pypy/pypy/changeset/f5a631ede713/

Log:	Remove extraneous "-" sign.

diff --git a/rpython/jit/backend/x86/regalloc.py b/rpython/jit/backend/x86/regalloc.py
--- a/rpython/jit/backend/x86/regalloc.py
+++ b/rpython/jit/backend/x86/regalloc.py
@@ -1398,7 +1398,7 @@
         startindex_loc = self.rm.make_sure_var_in_reg(args[1], args)
         if 0 <= constbytes <= 16 * 8 and (
                 valid_addressing_size(itemsize) or
--               isinstance(startindex_loc, ImmedLoc)):
+                isinstance(startindex_loc, ImmedLoc)):
             if IS_X86_64:
                 null_loc = X86_64_XMM_SCRATCH_REG
             else:

From noreply at buildbot.pypy.org  Tue Nov 11 18:52:00 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Tue, 11 Nov 2014 18:52:00 +0100 (CET)
Subject: [pypy-commit] pypy stmgc-c7: hg merge 9a6efa2d4493
Message-ID: <20141111175200.E11A11C02CB@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: stmgc-c7
Changeset: r74453:1565213578be
Date: 2014-11-11 18:44 +0100
http://bitbucket.org/pypy/pypy/changeset/1565213578be/

Log:	hg merge 9a6efa2d4493

	Merge default up to 7e7044080cc5; then additionally merge the
	improve-gc-tracing-hooks branch.

diff too long, truncating to 2000 out of 74808 lines

diff --git a/LICENSE b/LICENSE
--- a/LICENSE
+++ b/LICENSE
@@ -3,8 +3,8 @@
 
 Except when otherwise stated (look for LICENSE files in directories or
 information at the beginning of each file) all software and documentation in
-the 'rpython', 'pypy', 'ctype_configure', 'dotviewer', 'demo', and 'lib_pypy'
-directories is licensed as follows: 
+the 'rpython', 'pypy', 'ctype_configure', 'dotviewer', 'demo', 'lib_pypy',
+'py', and '_pytest' directories is licensed as follows: 
 
     The MIT License
 
@@ -35,280 +35,290 @@
 the beginning of each file) the files in the 'pypy' directory are each
 copyrighted by one or more of the following people and organizations:    
 
-    Armin Rigo
-    Maciej Fijalkowski
-    Carl Friedrich Bolz
-    Antonio Cuni
-    Amaury Forgeot d'Arc
-    Samuele Pedroni
-    Alex Gaynor
-    Michael Hudson
-    David Schneider
-    Matti Picus
-    Brian Kearns
-    Philip Jenvey
-    Holger Krekel
-    Christian Tismer
-    Hakan Ardo
-    Benjamin Peterson
-    Manuel Jacob
-    Anders Chrigstrom
-    Eric van Riet Paap
-    Wim Lavrijsen
-    Ronan Lamy
-    Richard Emslie
-    Alexander Schremmer
-    Dan Villiom Podlaski Christiansen
-    Lukas Diekmann
-    Sven Hager
-    Anders Lehmann
-    Aurelien Campeas
-    Niklaus Haldimann
-    Camillo Bruni
-    Laura Creighton
-    Toon Verwaest
-    Remi Meier
-    Leonardo Santagada
-    Seo Sanghyeon
-    Romain Guillebert
-    Justin Peel
-    Ronny Pfannschmidt
-    David Edelsohn
-    Anders Hammarquist
-    Jakub Gustak
-    Guido Wesdorp
-    Lawrence Oluyede
-    Bartosz Skowron
-    Daniel Roberts
-    Niko Matsakis
-    Adrien Di Mascio
-    Alexander Hesse
-    Ludovic Aubry
-    Jacob Hallen
-    Jason Creighton
-    Alex Martelli
-    Michal Bendowski
-    Jan de Mooij
-    stian
-    Michael Foord
-    Stephan Diehl
-    Stefan Schwarzer
-    Valentino Volonghi
-    Tomek Meka
-    Patrick Maupin
-    Bob Ippolito
-    Bruno Gola
-    Jean-Paul Calderone
-    Timo Paulssen
-    Squeaky
-    Alexandre Fayolle
-    Simon Burton
-    Marius Gedminas
-    John Witulski
-    Konstantin Lopuhin
-    Greg Price
-    Dario Bertini
-    Mark Pearse
-    Simon Cross
-    Andreas Stührk
-    Jean-Philippe St. Pierre
-    Guido van Rossum
-    Pavel Vinogradov
-    Paweł Piotr Przeradowski
-    Paul deGrandis
-    Ilya Osadchiy
-    Tobias Oberstein
-    Adrian Kuhn
-    Boris Feigin
-    Stefano Rivera
-    tav
-    Taavi Burns
-    Georg Brandl
-    Bert Freudenberg
-    Stian Andreassen
-    Laurence Tratt
-    Wanja Saatkamp
-    Ivan Sichmann Freitas
-    Gerald Klix
-    Mike Blume
-    Oscar Nierstrasz
-    Stefan H. Muller
-    Jeremy Thurgood
-    Gregor Wegberg
-    Rami Chowdhury
-    Tobias Pape
-    Edd Barrett
-    David Malcolm
-    Eugene Oden
-    Henry Mason
-    Preston Timmons
-    Jeff Terrace
-    David Ripton
-    Dusty Phillips
-    Lukas Renggli
-    Guenter Jantzen
-    Ned Batchelder
-    Amit Regmi
-    Ben Young
-    Nicolas Chauvat
-    Andrew Durdin
-    Andrew Chambers
-    Michael Schneider
-    Nicholas Riley
-    Jason Chu
-    Igor Trindade Oliveira
-    Rocco Moretti
-    Gintautas Miliauskas
-    Michael Twomey
-    Lucian Branescu Mihaila
-    Tim Felgentreff
-    Tyler Wade
-    Gabriel Lavoie
-    Olivier Dormond
-    Jared Grubb
-    Karl Bartel
-    Brian Dorsey
-    Victor Stinner
-    Andrews Medina
-    Stuart Williams
-    Jasper Schulz
-    Christian Hudon
-    Toby Watson
-    Antoine Pitrou
-    Aaron Iles
-    Michael Cheng
-    Justas Sadzevicius
-    Mikael Schönenberg
-    Gasper Zejn
-    Neil Shepperd
-    Elmo Mäntynen
-    Jonathan David Riehl
-    Stanislaw Halik
-    Anders Qvist
-    Chirag Jadwani
-    Beatrice During
-    Alex Perry
-    Vincent Legoll
-    Alan McIntyre
-    Alexander Sedov
-    Corbin Simpson
-    Christopher Pope
-    wenzhuman
-    Christian Tismer 
-    Marc Abramowitz
-    Dan Stromberg
-    Stefano Parmesan
-    Alexis Daboville
-    Jens-Uwe Mager
-    Carl Meyer
-    Karl Ramm
-    Pieter Zieschang
-    Gabriel
-    Lukas Vacek
-    Andrew Dalke
-    Sylvain Thenault
-    Nathan Taylor
-    Vladimir Kryachko
-    Jacek Generowicz
-    Alejandro J. Cura
-    Jacob Oscarson
-    Travis Francis Athougies
-    Ryan Gonzalez
-    Kristjan Valur Jonsson
-    Sebastian Pawluś
-    Neil Blakey-Milner
-    anatoly techtonik
-    Lutz Paelike
-    Lucio Torre
-    Lars Wassermann
-    Henrik Vendelbo
-    Dan Buch
-    Miguel de Val Borro
-    Artur Lisiecki
-    Sergey Kishchenko
-    Ignas Mikalajunas
-    Christoph Gerum
-    Martin Blais
-    Lene Wagner
-    Tomo Cocoa
-    roberto at goyle
-    Yury V. Zaytsev
-    Anna Katrina Dominguez
-    William Leslie
-    Bobby Impollonia
-    timo at eistee.fritz.box
-    Andrew Thompson
-    Ben Darnell
-    Roberto De Ioris
-    Juan Francisco Cantero Hurtado
-    Godefroid Chappelle
-    Joshua Gilbert
-    Dan Colish
-    Christopher Armstrong
-    Michael Hudson-Doyle
-    Anders Sigfridsson
-    Yasir Suhail
-    rafalgalczynski at gmail.com
-    Floris Bruynooghe
-    Laurens Van Houtven
-    Akira Li
-    Gustavo Niemeyer
-    Stephan Busemann
-    Rafał Gałczyński
-    Yusei Tahara
-    Christian Muirhead
-    James Lan
-    shoma hosaka
-    Daniel Neuh?user
-    Matthew Miller
-    Buck Golemon
-    Konrad Delong
-    Dinu Gherman
-    Chris Lambacher
-    coolbutuseless at gmail.com
-    Rodrigo Araújo
-    w31rd0
-    Jim Baker
-    James Robert
-    Armin Ronacher
-    Brett Cannon
-    yrttyr
-    aliceinwire
-    OlivierBlanvillain
-    Zooko Wilcox-O Hearn
-    Tomer Chachamu
-    Christopher Groskopf
-    Asmo Soinio
-    Stefan Marr
-    jiaaro
-    opassembler.py
-    Antony Lee
-    Jim Hunziker
-    Markus Unterwaditzer
-    Even Wiik Thomassen
-    jbs
-    soareschen
-    Kurt Griffiths
-    Mike Bayer
-    Flavio Percoco
-    Kristoffer Kleine
-    yasirs
-    Michael Chermside
-    Anna Ravencroft
-    Julien Phalip
-    Dan Loewenherz
+  Armin Rigo
+  Maciej Fijalkowski
+  Carl Friedrich Bolz
+  Antonio Cuni
+  Amaury Forgeot d'Arc
+  Samuele Pedroni
+  Alex Gaynor
+  Michael Hudson
+  David Schneider
+  Matti Picus
+  Brian Kearns
+  Philip Jenvey
+  Holger Krekel
+  Christian Tismer
+  Hakan Ardo
+  Benjamin Peterson
+  Manuel Jacob
+  Anders Chrigstrom
+  Eric van Riet Paap
+  Ronan Lamy
+  Wim Lavrijsen
+  Richard Emslie
+  Alexander Schremmer
+  Dan Villiom Podlaski Christiansen
+  Lukas Diekmann
+  Sven Hager
+  Anders Lehmann
+  Aurelien Campeas
+  Niklaus Haldimann
+  Remi Meier
+  Camillo Bruni
+  Laura Creighton
+  Toon Verwaest
+  Leonardo Santagada
+  Seo Sanghyeon
+  Romain Guillebert
+  Justin Peel
+  Ronny Pfannschmidt
+  David Edelsohn
+  Anders Hammarquist
+  Jakub Gustak
+  Guido Wesdorp
+  Lawrence Oluyede
+  Bartosz Skowron
+  Gregor Wegberg
+  Daniel Roberts
+  Niko Matsakis
+  Adrien Di Mascio
+  Alexander Hesse
+  Ludovic Aubry
+  Jacob Hallen
+  Jason Creighton
+  Alex Martelli
+  Michal Bendowski
+  Jan de Mooij
+  stian
+  Michael Foord
+  Stephan Diehl
+  Tyler Wade
+  Stefan Schwarzer
+  Valentino Volonghi
+  Tomek Meka
+  Patrick Maupin
+  Bob Ippolito
+  Bruno Gola
+  Jean-Paul Calderone
+  Timo Paulssen
+  Squeaky
+  Alexandre Fayolle
+  Simon Burton
+  Marius Gedminas
+  Martin Matusiak
+  Konstantin Lopuhin
+  John Witulski
+  Wenzhu Man
+  Greg Price
+  Dario Bertini
+  Mark Pearse
+  Simon Cross
+  Ivan Sichmann Freitas
+  Andreas Stührk
+  Jean-Philippe St. Pierre
+  Guido van Rossum
+  Pavel Vinogradov
+  Stefano Rivera
+  Paweł Piotr Przeradowski
+  Paul deGrandis
+  Ilya Osadchiy
+  Tobias Oberstein
+  Adrian Kuhn
+  Boris Feigin
+  tav
+  Taavi Burns
+  Georg Brandl
+  Laurence Tratt
+  Bert Freudenberg
+  Stian Andreassen
+  Wanja Saatkamp
+  Gerald Klix
+  Mike Blume
+  Oscar Nierstrasz
+  Stefan H. Muller
+  Edd Barrett
+  Jeremy Thurgood
+  Rami Chowdhury
+  Tobias Pape
+  David Malcolm
+  Eugene Oden
+  Henry Mason
+  Vasily Kuznetsov
+  Preston Timmons
+  Jeff Terrace
+  David Ripton
+  Dusty Phillips
+  Lukas Renggli
+  Guenter Jantzen
+  Ned Batchelder
+  Amit Regmi
+  Ben Young
+  Nicolas Chauvat
+  Andrew Durdin
+  Andrew Chambers
+  Michael Schneider
+  Nicholas Riley
+  Jason Chu
+  Igor Trindade Oliveira
+  Tim Felgentreff
+  Rocco Moretti
+  Gintautas Miliauskas
+  Michael Twomey
+  Lucian Branescu Mihaila
+  Gabriel Lavoie
+  Olivier Dormond
+  Jared Grubb
+  Karl Bartel
+  Brian Dorsey
+  Victor Stinner
+  Andrews Medina
+  Stuart Williams
+  Jasper Schulz
+  Christian Hudon
+  Toby Watson
+  Antoine Pitrou
+  Aaron Iles
+  Michael Cheng
+  Justas Sadzevicius
+  Gasper Zejn
+  anatoly techtonik
+  Neil Shepperd
+  Mikael Schönenberg
+  Elmo M?ntynen
+  Jonathan David Riehl
+  Stanislaw Halik
+  Anders Qvist
+  Corbin Simpson
+  Chirag Jadwani
+  Beatrice During
+  Alex Perry
+  Vincent Legoll
+  Alan McIntyre
+  Alexander Sedov
+  Christopher Pope
+  Christian Tismer 
+  Marc Abramowitz
+  Dan Stromberg
+  Stefano Parmesan
+  Alexis Daboville
+  Jens-Uwe Mager
+  Carl Meyer
+  Karl Ramm
+  Pieter Zieschang
+  Sebastian Pawluś
+  Gabriel
+  Lukas Vacek
+  Andrew Dalke
+  Sylvain Thenault
+  Nathan Taylor
+  Vladimir Kryachko
+  Arjun Naik
+  Attila Gobi
+  Jacek Generowicz
+  Alejandro J. Cura
+  Jacob Oscarson
+  Travis Francis Athougies
+  Ryan Gonzalez
+  Ian Foote
+  Kristjan Valur Jonsson
+  Neil Blakey-Milner
+  Lutz Paelike
+  Lucio Torre
+  Lars Wassermann
+  Valentina Mukhamedzhanova
+  Henrik Vendelbo
+  Dan Buch
+  Miguel de Val Borro
+  Artur Lisiecki
+  Sergey Kishchenko
+  Yichao Yu
+  Ignas Mikalajunas
+  Christoph Gerum
+  Martin Blais
+  Lene Wagner
+  Tomo Cocoa
+  roberto at goyle
+  Yury V. Zaytsev
+  Anna Katrina Dominguez
+  William Leslie
+  Bobby Impollonia
+  timo at eistee.fritz.box
+  Andrew Thompson
+  Yusei Tahara
+  Ben Darnell
+  Roberto De Ioris
+  Juan Francisco Cantero Hurtado
+  Godefroid Chappelle
+  Joshua Gilbert
+  Dan Colish
+  Christopher Armstrong
+  Michael Hudson-Doyle
+  Anders Sigfridsson
+  Yasir Suhail
+  Jason Michalski
+  rafalgalczynski at gmail.com
+  Floris Bruynooghe
+  Laurens Van Houtven
+  Akira Li
+  Gustavo Niemeyer
+  Stephan Busemann
+  Rafał Gałczyński
+  Christian Muirhead
+  James Lan
+  shoma hosaka
+  Daniel Neuh?user
+  Matthew Miller
+  Buck Golemon
+  Konrad Delong
+  Dinu Gherman
+  Chris Lambacher
+  coolbutuseless at gmail.com
+  Rodrigo Araújo
+  Jim Baker
+  James Robert
+  Armin Ronacher
+  Brett Cannon
+  yrttyr
+  aliceinwire
+  OlivierBlanvillain
+  Zooko Wilcox-O Hearn
+  Tomer Chachamu
+  Christopher Groskopf
+  Asmo Soinio
+  Stefan Marr
+  jiaaro
+  Mads Kiilerich
+  opassembler.py
+  Antony Lee
+  Jim Hunziker
+  Markus Unterwaditzer
+  Even Wiik Thomassen
+  jbs
+  soareschen
+  Kurt Griffiths
+  Mike Bayer
+  Matthew Miller
+  Flavio Percoco
+  Kristoffer Kleine
+  yasirs
+  Michael Chermside
+  Anna Ravencroft
+  Dan Crosta
+  Julien Phalip
+  Dan Loewenherz
 
-    Heinrich-Heine University, Germany 
-    Open End AB (formerly AB Strakt), Sweden
-    merlinux GmbH, Germany 
-    tismerysoft GmbH, Germany 
-    Logilab Paris, France 
-    DFKI GmbH, Germany 
-    Impara, Germany
-    Change Maker, Sweden 
-    University of California Berkeley, USA
-    Google Inc.
-    King's College London
+  Heinrich-Heine University, Germany 
+  Open End AB (formerly AB Strakt), Sweden
+  merlinux GmbH, Germany 
+  tismerysoft GmbH, Germany 
+  Logilab Paris, France 
+  DFKI GmbH, Germany 
+  Impara, Germany
+  Change Maker, Sweden 
+  University of California Berkeley, USA
+  Google Inc.
+  King's College London
 
 The PyPy Logo as used by http://speed.pypy.org and others was created
 by Samuel Reis and is distributed on terms of Creative Commons Share Alike
@@ -354,6 +364,46 @@
 See the License for the specific language governing permissions and
 limitations under the License.
 
-Detailled license information is contained in the NOTICE file in the
+Detailed license information is contained in the NOTICE file in the
 directory.
 
+
+Licenses and Acknowledgements for Incorporated Software
+=======================================================
+
+This section is an incomplete, but growing list of licenses and
+acknowledgements for third-party software incorporated in the PyPy
+distribution.
+
+License for 'Tcl/Tk'
+--------------------
+
+This copy of PyPy contains library code that may, when used, result in
+the Tcl/Tk library to be loaded.  PyPy also includes code that may be
+regarded as being a copy of some parts of the Tcl/Tk header files.
+You may see a copy of the License for Tcl/Tk in the file
+`lib_pypy/_tkinter/license.terms` included here.
+
+License for 'bzip2'
+-------------------
+
+This copy of PyPy may be linked (dynamically or statically) with the
+bzip2 library.  You may see a copy of the License for bzip2/libbzip2 at
+
+    http://www.bzip.org/1.0.5/bzip2-manual-1.0.5.html
+
+License for 'openssl'
+---------------------
+
+This copy of PyPy may be linked (dynamically or statically) with the
+openssl library.  You may see a copy of the License for OpenSSL at
+
+    https://www.openssl.org/source/license.html
+
+License for 'gdbm'
+------------------
+
+The gdbm module includes code from gdbm.h, which is distributed under
+the terms of the GPL license version 2 or any later version.  Thus the
+gdbm module, provided in the file lib_pypy/gdbm.py, is redistributed
+under the terms of the GPL license as well.
diff --git a/_pytest/README-BEFORE-UPDATING b/_pytest/README-BEFORE-UPDATING
new file mode 100644
--- /dev/null
+++ b/_pytest/README-BEFORE-UPDATING
@@ -0,0 +1,17 @@
+This is PyPy's code of the pytest lib.  We don't expect to upgrade it
+very often, but once we do:
+
+    WARNING!
+
+    WE HAVE MADE A FEW TWEAKS HERE!
+
+Please be sure that you don't just copy the newer version from
+upstream without checking the few changes that we did.  This
+can be done like this:
+
+    cd 
+    hg log . -v | less
+
+then search for all " _pytest/" in that list to know which are the
+relevant checkins.  (Look for the checkins that only edit one
+or two files in this directory.)
diff --git a/_pytest/resultlog.py b/_pytest/resultlog.py
--- a/_pytest/resultlog.py
+++ b/_pytest/resultlog.py
@@ -53,16 +53,24 @@
         self.config = config
         self.logfile = logfile # preferably line buffered
 
-    def write_log_entry(self, testpath, lettercode, longrepr):
-        py.builtin.print_("%s %s" % (lettercode, testpath), file=self.logfile)
+    def write_log_entry(self, testpath, lettercode, longrepr, sections=None):
+        _safeprint("%s %s" % (lettercode, testpath), file=self.logfile)
         for line in longrepr.splitlines():
-            py.builtin.print_(" %s" % line, file=self.logfile)
+            _safeprint(" %s" % line, file=self.logfile)
+        if sections is not None and (
+                lettercode in ('E', 'F')):    # to limit the size of logs
+            for title, content in sections:
+                _safeprint(" ---------- %s ----------" % (title,),
+                           file=self.logfile)
+                for line in content.splitlines():
+                    _safeprint(" %s" % line, file=self.logfile)
 
     def log_outcome(self, report, lettercode, longrepr):
         testpath = getattr(report, 'nodeid', None)
         if testpath is None:
             testpath = report.fspath
-        self.write_log_entry(testpath, lettercode, longrepr)
+        self.write_log_entry(testpath, lettercode, longrepr,
+                             getattr(report, 'sections', None))
 
     def pytest_runtest_logreport(self, report):
         if report.when != "call" and report.passed:
@@ -98,3 +106,8 @@
         if path is None:
             path = "cwd:%s" % py.path.local()
         self.write_log_entry(path, '!', str(excrepr))
+
+def _safeprint(s, file):
+    if isinstance(s, unicode):
+        s = s.encode('utf-8')
+    py.builtin.print_(s, file=file)
diff --git a/lib-python/2.7/CGIHTTPServer.py b/lib-python/2.7/CGIHTTPServer.py
--- a/lib-python/2.7/CGIHTTPServer.py
+++ b/lib-python/2.7/CGIHTTPServer.py
@@ -84,7 +84,7 @@
         path begins with one of the strings in self.cgi_directories
         (and the next character is a '/' or the end of the string).
         """
-        collapsed_path = _url_collapse_path(self.path)
+        collapsed_path = _url_collapse_path(urllib.unquote(self.path))
         dir_sep = collapsed_path.find('/', 1)
         head, tail = collapsed_path[:dir_sep], collapsed_path[dir_sep+1:]
         if head in self.cgi_directories:
diff --git a/lib-python/2.7/Cookie.py b/lib-python/2.7/Cookie.py
--- a/lib-python/2.7/Cookie.py
+++ b/lib-python/2.7/Cookie.py
@@ -1,6 +1,3 @@
-#!/usr/bin/env python
-#
-
 ####
 # Copyright 2000 by Timothy O'Malley 
 #
diff --git a/lib-python/2.7/HTMLParser.py b/lib-python/2.7/HTMLParser.py
--- a/lib-python/2.7/HTMLParser.py
+++ b/lib-python/2.7/HTMLParser.py
@@ -22,9 +22,12 @@
 starttagopen = re.compile('<[a-zA-Z]')
 piclose = re.compile('>')
 commentclose = re.compile(r'--\s*>')
-tagfind = re.compile('([a-zA-Z][-.a-zA-Z0-9:_]*)(?:\s|/(?!>))*')
+
 # see http://www.w3.org/TR/html5/tokenization.html#tag-open-state
 # and http://www.w3.org/TR/html5/tokenization.html#tag-name-state
+# note: if you change tagfind/attrfind remember to update locatestarttagend too
+tagfind = re.compile('([a-zA-Z][^\t\n\r\f />\x00]*)(?:\s|/(?!>))*')
+# this regex is currently unused, but left for backward compatibility
 tagfind_tolerant = re.compile('[a-zA-Z][^\t\n\r\f />\x00]*')
 
 attrfind = re.compile(
@@ -32,7 +35,7 @@
     r'(\'[^\']*\'|"[^"]*"|(?![\'"])[^>\s]*))?(?:\s|/(?!>))*')
 
 locatestarttagend = re.compile(r"""
-  <[a-zA-Z][-.a-zA-Z0-9:_]*          # tag name
+  <[a-zA-Z][^\t\n\r\f />\x00]*       # tag name
   (?:[\s/]*                          # optional whitespace before attribute name
     (?:(?<=['"\s/])[^\s/>][^\s/=>]*  # attribute name
       (?:\s*=+\s*                    # value indicator
@@ -192,9 +195,9 @@
                     i = self.updatepos(i, k)
                     continue
                 else:
-                    if ";" in rawdata[i:]: #bail by consuming &#
-                        self.handle_data(rawdata[0:2])
-                        i = self.updatepos(i, 2)
+                    if ";" in rawdata[i:]:  # bail by consuming '&#'
+                        self.handle_data(rawdata[i:i+2])
+                        i = self.updatepos(i, i+2)
                     break
             elif startswith('&', i):
                 match = entityref.match(rawdata, i)
@@ -373,14 +376,14 @@
                 self.handle_data(rawdata[i:gtpos])
                 return gtpos
             # find the name: w3.org/TR/html5/tokenization.html#tag-name-state
-            namematch = tagfind_tolerant.match(rawdata, i+2)
+            namematch = tagfind.match(rawdata, i+2)
             if not namematch:
                 # w3.org/TR/html5/tokenization.html#end-tag-open-state
                 if rawdata[i:i+3] == '':
                     return i+3
                 else:
                     return self.parse_bogus_comment(i)
-            tagname = namematch.group().lower()
+            tagname = namematch.group(1).lower()
             # consume and ignore other stuff between the name and the >
             # Note: this is not 100% correct, since we might have things like
             # , but looking for > after tha name should cover
diff --git a/lib-python/2.7/SimpleHTTPServer.py b/lib-python/2.7/SimpleHTTPServer.py
--- a/lib-python/2.7/SimpleHTTPServer.py
+++ b/lib-python/2.7/SimpleHTTPServer.py
@@ -43,8 +43,10 @@
         """Serve a GET request."""
         f = self.send_head()
         if f:
-            self.copyfile(f, self.wfile)
-            f.close()
+            try:
+                self.copyfile(f, self.wfile)
+            finally:
+                f.close()
 
     def do_HEAD(self):
         """Serve a HEAD request."""
@@ -88,13 +90,17 @@
         except IOError:
             self.send_error(404, "File not found")
             return None
-        self.send_response(200)
-        self.send_header("Content-type", ctype)
-        fs = os.fstat(f.fileno())
-        self.send_header("Content-Length", str(fs[6]))
-        self.send_header("Last-Modified", self.date_time_string(fs.st_mtime))
-        self.end_headers()
-        return f
+        try:
+            self.send_response(200)
+            self.send_header("Content-type", ctype)
+            fs = os.fstat(f.fileno())
+            self.send_header("Content-Length", str(fs[6]))
+            self.send_header("Last-Modified", self.date_time_string(fs.st_mtime))
+            self.end_headers()
+            return f
+        except:
+            f.close()
+            raise
 
     def list_directory(self, path):
         """Helper to produce a directory listing (absent index.html).
diff --git a/lib-python/2.7/SimpleXMLRPCServer.py b/lib-python/2.7/SimpleXMLRPCServer.py
--- a/lib-python/2.7/SimpleXMLRPCServer.py
+++ b/lib-python/2.7/SimpleXMLRPCServer.py
@@ -704,4 +704,5 @@
     server = SimpleXMLRPCServer(("localhost", 8000))
     server.register_function(pow)
     server.register_function(lambda x,y: x+y, 'add')
+    server.register_multicall_functions()
     server.serve_forever()
diff --git a/lib-python/2.7/SocketServer.py b/lib-python/2.7/SocketServer.py
--- a/lib-python/2.7/SocketServer.py
+++ b/lib-python/2.7/SocketServer.py
@@ -513,35 +513,37 @@
 
     def collect_children(self):
         """Internal routine to wait for children that have exited."""
-        if self.active_children is None: return
+        if self.active_children is None:
+            return
+
+        # If we're above the max number of children, wait and reap them until
+        # we go back below threshold. Note that we use waitpid(-1) below to be
+        # able to collect children in size() syscalls instead
+        # of size(): the downside is that this might reap children
+        # which we didn't spawn, which is why we only resort to this when we're
+        # above max_children.
         while len(self.active_children) >= self.max_children:
-            # XXX: This will wait for any child process, not just ones
-            # spawned by this library. This could confuse other
-            # libraries that expect to be able to wait for their own
-            # children.
             try:
-                pid, status = os.waitpid(0, 0)
-            except os.error:
-                pid = None
-            if pid not in self.active_children: continue
-            self.active_children.remove(pid)
+                pid, _ = os.waitpid(-1, 0)
+                self.active_children.discard(pid)
+            except OSError as e:
+                if e.errno == errno.ECHILD:
+                    # we don't have any children, we're done
+                    self.active_children.clear()
+                elif e.errno != errno.EINTR:
+                    break
 
-        # XXX: This loop runs more system calls than it ought
-        # to. There should be a way to put the active_children into a
-        # process group and then use os.waitpid(-pgid) to wait for any
-        # of that set, but I couldn't find a way to allocate pgids
-        # that couldn't collide.
-        for child in self.active_children:
+        # Now reap all defunct children.
+        for pid in self.active_children.copy():
             try:
-                pid, status = os.waitpid(child, os.WNOHANG)
-            except os.error:
-                pid = None
-            if not pid: continue
-            try:
-                self.active_children.remove(pid)
-            except ValueError, e:
-                raise ValueError('%s. x=%d and list=%r' % (e.message, pid,
-                                                           self.active_children))
+                pid, _ = os.waitpid(pid, os.WNOHANG)
+                # if the child hasn't exited yet, pid will be 0 and ignored by
+                # discard() below
+                self.active_children.discard(pid)
+            except OSError as e:
+                if e.errno == errno.ECHILD:
+                    # someone else reaped it
+                    self.active_children.discard(pid)
 
     def handle_timeout(self):
         """Wait for zombies after self.timeout seconds of inactivity.
@@ -557,8 +559,8 @@
         if pid:
             # Parent process
             if self.active_children is None:
-                self.active_children = []
-            self.active_children.append(pid)
+                self.active_children = set()
+            self.active_children.add(pid)
             self.close_request(request) #close handle in parent process
             return
         else:
diff --git a/lib-python/2.7/_MozillaCookieJar.py b/lib-python/2.7/_MozillaCookieJar.py
--- a/lib-python/2.7/_MozillaCookieJar.py
+++ b/lib-python/2.7/_MozillaCookieJar.py
@@ -39,7 +39,7 @@
     magic_re = "#( Netscape)? HTTP Cookie File"
     header = """\
 # Netscape HTTP Cookie File
-# http://www.netscape.com/newsref/std/cookie_spec.html
+# http://curl.haxx.se/rfc/cookie_spec.html
 # This is a generated file!  Do not edit.
 
 """
diff --git a/lib-python/2.7/_abcoll.py b/lib-python/2.7/_abcoll.py
--- a/lib-python/2.7/_abcoll.py
+++ b/lib-python/2.7/_abcoll.py
@@ -165,12 +165,17 @@
     def __gt__(self, other):
         if not isinstance(other, Set):
             return NotImplemented
-        return other < self
+        return len(self) > len(other) and self.__ge__(other)
 
     def __ge__(self, other):
         if not isinstance(other, Set):
             return NotImplemented
-        return other <= self
+        if len(self) < len(other):
+            return False
+        for elem in other:
+            if elem not in self:
+                return False
+        return True
 
     def __eq__(self, other):
         if not isinstance(other, Set):
@@ -194,6 +199,8 @@
             return NotImplemented
         return self._from_iterable(value for value in other if value in self)
 
+    __rand__ = __and__
+
     def isdisjoint(self, other):
         'Return True if two sets have a null intersection.'
         for value in other:
@@ -207,6 +214,8 @@
         chain = (e for s in (self, other) for e in s)
         return self._from_iterable(chain)
 
+    __ror__ = __or__
+
     def __sub__(self, other):
         if not isinstance(other, Set):
             if not isinstance(other, Iterable):
@@ -215,6 +224,14 @@
         return self._from_iterable(value for value in self
                                    if value not in other)
 
+    def __rsub__(self, other):
+        if not isinstance(other, Set):
+            if not isinstance(other, Iterable):
+                return NotImplemented
+            other = self._from_iterable(other)
+        return self._from_iterable(value for value in other
+                                   if value not in self)
+
     def __xor__(self, other):
         if not isinstance(other, Set):
             if not isinstance(other, Iterable):
@@ -222,6 +239,8 @@
             other = self._from_iterable(other)
         return (self - other) | (other - self)
 
+    __rxor__ = __xor__
+
     # Sets are not hashable by default, but subclasses can change this
     __hash__ = None
 
diff --git a/lib-python/2.7/_osx_support.py b/lib-python/2.7/_osx_support.py
--- a/lib-python/2.7/_osx_support.py
+++ b/lib-python/2.7/_osx_support.py
@@ -182,7 +182,7 @@
         # Compiler is GCC, check if it is LLVM-GCC
         data = _read_output("'%s' --version"
                              % (cc.replace("'", "'\"'\"'"),))
-        if 'llvm-gcc' in data:
+        if data and 'llvm-gcc' in data:
             # Found LLVM-GCC, fall back to clang
             cc = _find_build_tool('clang')
 
@@ -450,8 +450,16 @@
         # case and disallow installs.
         cflags = _config_vars.get(_INITPRE+'CFLAGS',
                                     _config_vars.get('CFLAGS', ''))
-        if ((macrelease + '.') >= '10.4.' and
-            '-arch' in cflags.strip()):
+        if macrelease:
+            try:
+                macrelease = tuple(int(i) for i in macrelease.split('.')[0:2])
+            except ValueError:
+                macrelease = (10, 0)
+        else:
+            # assume no universal support
+            macrelease = (10, 0)
+
+        if (macrelease >= (10, 4)) and '-arch' in cflags.strip():
             # The universal build will build fat binaries, but not on
             # systems before 10.4
 
diff --git a/lib-python/2.7/_pyio.py b/lib-python/2.7/_pyio.py
--- a/lib-python/2.7/_pyio.py
+++ b/lib-python/2.7/_pyio.py
@@ -192,38 +192,45 @@
                  (appending and "a" or "") +
                  (updating and "+" or ""),
                  closefd)
-    line_buffering = False
-    if buffering == 1 or buffering < 0 and raw.isatty():
-        buffering = -1
-        line_buffering = True
-    if buffering < 0:
-        buffering = DEFAULT_BUFFER_SIZE
-        try:
-            bs = os.fstat(raw.fileno()).st_blksize
-        except (os.error, AttributeError):
-            pass
+    result = raw
+    try:
+        line_buffering = False
+        if buffering == 1 or buffering < 0 and raw.isatty():
+            buffering = -1
+            line_buffering = True
+        if buffering < 0:
+            buffering = DEFAULT_BUFFER_SIZE
+            try:
+                bs = os.fstat(raw.fileno()).st_blksize
+            except (os.error, AttributeError):
+                pass
+            else:
+                if bs > 1:
+                    buffering = bs
+        if buffering < 0:
+            raise ValueError("invalid buffering size")
+        if buffering == 0:
+            if binary:
+                return result
+            raise ValueError("can't have unbuffered text I/O")
+        if updating:
+            buffer = BufferedRandom(raw, buffering)
+        elif writing or appending:
+            buffer = BufferedWriter(raw, buffering)
+        elif reading:
+            buffer = BufferedReader(raw, buffering)
         else:
-            if bs > 1:
-                buffering = bs
-    if buffering < 0:
-        raise ValueError("invalid buffering size")
-    if buffering == 0:
+            raise ValueError("unknown mode: %r" % mode)
+        result = buffer
         if binary:
-            return raw
-        raise ValueError("can't have unbuffered text I/O")
-    if updating:
-        buffer = BufferedRandom(raw, buffering)
-    elif writing or appending:
-        buffer = BufferedWriter(raw, buffering)
-    elif reading:
-        buffer = BufferedReader(raw, buffering)
-    else:
-        raise ValueError("unknown mode: %r" % mode)
-    if binary:
-        return buffer
-    text = TextIOWrapper(buffer, encoding, errors, newline, line_buffering)
-    text.mode = mode
-    return text
+            return result
+        text = TextIOWrapper(buffer, encoding, errors, newline, line_buffering)
+        result = text
+        text.mode = mode
+        return result
+    except:
+        result.close()
+        raise
 
 
 class DocDescriptor:
@@ -1997,7 +2004,13 @@
 
     def getvalue(self):
         self.flush()
-        return self.buffer.getvalue().decode(self._encoding, self._errors)
+        decoder = self._decoder or self._get_decoder()
+        old_state = decoder.getstate()
+        decoder.reset()
+        try:
+            return decoder.decode(self.buffer.getvalue(), final=True)
+        finally:
+            decoder.setstate(old_state)
 
     def __repr__(self):
         # TextIOWrapper tells the encoding in its repr. In StringIO,
diff --git a/lib-python/2.7/_weakrefset.py b/lib-python/2.7/_weakrefset.py
--- a/lib-python/2.7/_weakrefset.py
+++ b/lib-python/2.7/_weakrefset.py
@@ -60,6 +60,8 @@
             for itemref in self.data:
                 item = itemref()
                 if item is not None:
+                    # Caveat: the iterator will keep a strong reference to
+                    # `item` until it is resumed or closed.
                     yield item
 
     def __len__(self):
diff --git a/lib-python/2.7/aifc.py b/lib-python/2.7/aifc.py
--- a/lib-python/2.7/aifc.py
+++ b/lib-python/2.7/aifc.py
@@ -778,7 +778,7 @@
 
     def _ensure_header_written(self, datasize):
         if not self._nframeswritten:
-            if self._comptype in ('ULAW', 'ALAW'):
+            if self._comptype in ('ULAW', 'ulaw', 'ALAW', 'alaw'):
                 if not self._sampwidth:
                     self._sampwidth = 2
                 if self._sampwidth != 2:
@@ -844,7 +844,7 @@
         if self._datalength & 1:
             self._datalength = self._datalength + 1
         if self._aifc:
-            if self._comptype in ('ULAW', 'ALAW'):
+            if self._comptype in ('ULAW', 'ulaw', 'ALAW', 'alaw'):
                 self._datalength = self._datalength // 2
                 if self._datalength & 1:
                     self._datalength = self._datalength + 1
@@ -852,7 +852,10 @@
                 self._datalength = (self._datalength + 3) // 4
                 if self._datalength & 1:
                     self._datalength = self._datalength + 1
-        self._form_length_pos = self._file.tell()
+        try:
+            self._form_length_pos = self._file.tell()
+        except (AttributeError, IOError):
+            self._form_length_pos = None
         commlength = self._write_form_length(self._datalength)
         if self._aifc:
             self._file.write('AIFC')
@@ -864,7 +867,8 @@
         self._file.write('COMM')
         _write_ulong(self._file, commlength)
         _write_short(self._file, self._nchannels)
-        self._nframes_pos = self._file.tell()
+        if self._form_length_pos is not None:
+            self._nframes_pos = self._file.tell()
         _write_ulong(self._file, self._nframes)
         if self._comptype in ('ULAW', 'ulaw', 'ALAW', 'alaw', 'G722'):
             _write_short(self._file, 8)
@@ -875,7 +879,8 @@
             self._file.write(self._comptype)
             _write_string(self._file, self._compname)
         self._file.write('SSND')
-        self._ssnd_length_pos = self._file.tell()
+        if self._form_length_pos is not None:
+            self._ssnd_length_pos = self._file.tell()
         _write_ulong(self._file, self._datalength + 8)
         _write_ulong(self._file, 0)
         _write_ulong(self._file, 0)
diff --git a/lib-python/2.7/argparse.py b/lib-python/2.7/argparse.py
--- a/lib-python/2.7/argparse.py
+++ b/lib-python/2.7/argparse.py
@@ -168,6 +168,8 @@
         self._prog = prog
         self._indent_increment = indent_increment
         self._max_help_position = max_help_position
+        self._max_help_position = min(max_help_position,
+                                      max(width - 20, indent_increment * 2))
         self._width = width
 
         self._current_indent = 0
@@ -339,7 +341,7 @@
                     else:
                         line_len = len(indent) - 1
                     for part in parts:
-                        if line_len + 1 + len(part) > text_width:
+                        if line_len + 1 + len(part) > text_width and line:
                             lines.append(indent + ' '.join(line))
                             line = []
                             line_len = len(indent) - 1
@@ -478,7 +480,7 @@
     def _format_text(self, text):
         if '%(prog)' in text:
             text = text % dict(prog=self._prog)
-        text_width = self._width - self._current_indent
+        text_width = max(self._width - self._current_indent, 11)
         indent = ' ' * self._current_indent
         return self._fill_text(text, text_width, indent) + '\n\n'
 
@@ -486,7 +488,7 @@
         # determine the required width and the entry label
         help_position = min(self._action_max_length + 2,
                             self._max_help_position)
-        help_width = self._width - help_position
+        help_width = max(self._width - help_position, 11)
         action_width = help_position - self._current_indent - 2
         action_header = self._format_action_invocation(action)
 
@@ -1155,9 +1157,13 @@
     __hash__ = None
 
     def __eq__(self, other):
+        if not isinstance(other, Namespace):
+            return NotImplemented
         return vars(self) == vars(other)
 
     def __ne__(self, other):
+        if not isinstance(other, Namespace):
+            return NotImplemented
         return not (self == other)
 
     def __contains__(self, key):
diff --git a/lib-python/2.7/bsddb/dbshelve.py b/lib-python/2.7/bsddb/dbshelve.py
--- a/lib-python/2.7/bsddb/dbshelve.py
+++ b/lib-python/2.7/bsddb/dbshelve.py
@@ -1,4 +1,3 @@
-#!/usr/bin/env python
 #------------------------------------------------------------------------
 #           Copyright (c) 1997-2001 by Total Control Software
 #                         All Rights Reserved
diff --git a/lib-python/2.7/bsddb/test/test_dbtables.py b/lib-python/2.7/bsddb/test/test_dbtables.py
--- a/lib-python/2.7/bsddb/test/test_dbtables.py
+++ b/lib-python/2.7/bsddb/test/test_dbtables.py
@@ -1,5 +1,3 @@
-#!/usr/bin/env python
-#
 #-----------------------------------------------------------------------
 # A test suite for the table interface built on bsddb.db
 #-----------------------------------------------------------------------
diff --git a/lib-python/2.7/codecs.py b/lib-python/2.7/codecs.py
--- a/lib-python/2.7/codecs.py
+++ b/lib-python/2.7/codecs.py
@@ -456,15 +456,12 @@
 
         # read until we get the required number of characters (if available)
         while True:
-            # can the request can be satisfied from the character buffer?
-            if chars < 0:
-                if size < 0:
-                    if self.charbuffer:
-                        break
-                elif len(self.charbuffer) >= size:
+            # can the request be satisfied from the character buffer?
+            if chars >= 0:
+                if len(self.charbuffer) >= chars:
                     break
-            else:
-                if len(self.charbuffer) >= chars:
+            elif size >= 0:
+                if len(self.charbuffer) >= size:
                     break
             # we need more data
             if size < 0:
diff --git a/lib-python/2.7/collections.py b/lib-python/2.7/collections.py
--- a/lib-python/2.7/collections.py
+++ b/lib-python/2.7/collections.py
@@ -319,6 +319,7 @@
     if isinstance(field_names, basestring):
         field_names = field_names.replace(',', ' ').split()
     field_names = map(str, field_names)
+    typename = str(typename)
     if rename:
         seen = set()
         for index, name in enumerate(field_names):
@@ -331,6 +332,8 @@
                 field_names[index] = '_%d' % index
             seen.add(name)
     for name in [typename] + field_names:
+        if type(name) != str:
+            raise TypeError('Type names and field names must be strings')
         if not all(c.isalnum() or c=='_' for c in name):
             raise ValueError('Type names and field names can only contain '
                              'alphanumeric characters and underscores: %r' % name)
diff --git a/lib-python/2.7/csv.py b/lib-python/2.7/csv.py
--- a/lib-python/2.7/csv.py
+++ b/lib-python/2.7/csv.py
@@ -93,6 +93,10 @@
         self.line_num = self.reader.line_num
         return self._fieldnames
 
+    # Issue 20004: Because DictReader is a classic class, this setter is
+    # ignored.  At this point in 2.7's lifecycle, it is too late to change the
+    # base class for fear of breaking working code.  If you want to change
+    # fieldnames without overwriting the getter, set _fieldnames directly.
     @fieldnames.setter
     def fieldnames(self, value):
         self._fieldnames = value
@@ -140,8 +144,8 @@
         if self.extrasaction == "raise":
             wrong_fields = [k for k in rowdict if k not in self.fieldnames]
             if wrong_fields:
-                raise ValueError("dict contains fields not in fieldnames: " +
-                                 ", ".join(wrong_fields))
+                raise ValueError("dict contains fields not in fieldnames: "
+                                 + ", ".join([repr(x) for x in wrong_fields]))
         return [rowdict.get(key, self.restval) for key in self.fieldnames]
 
     def writerow(self, rowdict):
diff --git a/lib-python/2.7/ctypes/test/__init__.py b/lib-python/2.7/ctypes/test/__init__.py
--- a/lib-python/2.7/ctypes/test/__init__.py
+++ b/lib-python/2.7/ctypes/test/__init__.py
@@ -2,7 +2,15 @@
 
 use_resources = []
 
-class ResourceDenied(Exception):
+import ctypes
+ctypes_symbols = dir(ctypes)
+
+def need_symbol(name):
+    return unittest.skipUnless(name in ctypes_symbols,
+                               '{!r} is required'.format(name))
+
+
+class ResourceDenied(unittest.SkipTest):
     """Test skipped because it requested a disallowed resource.
 
     This is raised when a test calls requires() for a resource that
diff --git a/lib-python/2.7/ctypes/test/test_arrays.py b/lib-python/2.7/ctypes/test/test_arrays.py
--- a/lib-python/2.7/ctypes/test/test_arrays.py
+++ b/lib-python/2.7/ctypes/test/test_arrays.py
@@ -2,6 +2,8 @@
 from ctypes import *
 from test.test_support import impl_detail
 
+from ctypes.test import need_symbol
+
 formats = "bBhHiIlLqQfd"
 
 # c_longdouble commented out for PyPy, look at the commend in test_longdouble
@@ -98,8 +100,8 @@
         self.assertEqual(values, [1, 2, 3, 4, 5])
 
     def test_classcache(self):
-        self.assertTrue(not ARRAY(c_int, 3) is ARRAY(c_int, 4))
-        self.assertTrue(ARRAY(c_int, 3) is ARRAY(c_int, 3))
+        self.assertIsNot(ARRAY(c_int, 3), ARRAY(c_int, 4))
+        self.assertIs(ARRAY(c_int, 3), ARRAY(c_int, 3))
 
     def test_from_address(self):
         # Failed with 0.9.8, reported by JUrner
@@ -112,20 +114,16 @@
         self.assertEqual(sz[1:4:2], "o")
         self.assertEqual(sz.value, "foo")
 
-    try:
-        create_unicode_buffer
-    except NameError:
-        pass
-    else:
-        def test_from_addressW(self):
-            p = create_unicode_buffer("foo")
-            sz = (c_wchar * 3).from_address(addressof(p))
-            self.assertEqual(sz[:], "foo")
-            self.assertEqual(sz[::], "foo")
-            self.assertEqual(sz[::-1], "oof")
-            self.assertEqual(sz[::3], "f")
-            self.assertEqual(sz[1:4:2], "o")
-            self.assertEqual(sz.value, "foo")
+    @need_symbol('create_unicode_buffer')
+    def test_from_addressW(self):
+        p = create_unicode_buffer("foo")
+        sz = (c_wchar * 3).from_address(addressof(p))
+        self.assertEqual(sz[:], "foo")
+        self.assertEqual(sz[::], "foo")
+        self.assertEqual(sz[::-1], "oof")
+        self.assertEqual(sz[::3], "f")
+        self.assertEqual(sz[1:4:2], "o")
+        self.assertEqual(sz.value, "foo")
 
     def test_cache(self):
         # Array types are cached internally in the _ctypes extension,
@@ -139,7 +137,7 @@
         # Create a new array type based on it:
         t1 = my_int * 1
         t2 = my_int * 1
-        self.assertTrue(t1 is t2)
+        self.assertIs(t1, t2)
 
 if __name__ == '__main__':
     unittest.main()
diff --git a/lib-python/2.7/ctypes/test/test_as_parameter.py b/lib-python/2.7/ctypes/test/test_as_parameter.py
--- a/lib-python/2.7/ctypes/test/test_as_parameter.py
+++ b/lib-python/2.7/ctypes/test/test_as_parameter.py
@@ -1,5 +1,6 @@
 import unittest
 from ctypes import *
+from ctypes.test import need_symbol
 import _ctypes_test
 
 dll = CDLL(_ctypes_test.__file__)
@@ -17,11 +18,8 @@
     def wrap(self, param):
         return param
 
+    @need_symbol('c_wchar')
     def test_wchar_parm(self):
-        try:
-            c_wchar
-        except NameError:
-            return
         f = dll._testfunc_i_bhilfd
         f.argtypes = [c_byte, c_wchar, c_int, c_long, c_float, c_double]
         result = f(self.wrap(1), self.wrap(u"x"), self.wrap(3), self.wrap(4), self.wrap(5.0), self.wrap(6.0))
@@ -134,7 +132,7 @@
         f.argtypes = [c_longlong, MyCallback]
 
         def callback(value):
-            self.assertTrue(isinstance(value, (int, long)))
+            self.assertIsInstance(value, (int, long))
             return value & 0x7FFFFFFF
 
         cb = MyCallback(callback)
diff --git a/lib-python/2.7/ctypes/test/test_bitfields.py b/lib-python/2.7/ctypes/test/test_bitfields.py
--- a/lib-python/2.7/ctypes/test/test_bitfields.py
+++ b/lib-python/2.7/ctypes/test/test_bitfields.py
@@ -1,4 +1,5 @@
 from ctypes import *
+from ctypes.test import need_symbol
 import unittest
 import os
 
@@ -131,15 +132,6 @@
         self.assertEqual(result[0], TypeError)
         self.assertIn('bit fields not allowed for type', result[1])
 
-        try:
-            c_wchar
-        except NameError:
-            pass
-        else:
-            result = self.fail_fields(("a", c_wchar, 1))
-            self.assertEqual(result[0], TypeError)
-            self.assertIn('bit fields not allowed for type', result[1])
-
         class Dummy(Structure):
             _fields_ = []
 
@@ -147,6 +139,12 @@
         self.assertEqual(result[0], TypeError)
         self.assertIn('bit fields not allowed for type', result[1])
 
+    @need_symbol('c_wchar')
+    def test_c_wchar(self):
+        result = self.fail_fields(("a", c_wchar, 1))
+        self.assertEqual(result,
+                (TypeError, 'bit fields not allowed for type c_wchar'))
+
     def test_single_bitfield_size(self):
         for c_typ in int_types:
             result = self.fail_fields(("a", c_typ, -1))
@@ -213,7 +211,7 @@
         class X(Structure):
             _fields_ = [("a", c_byte, 4),
                         ("b", c_int, 32)]
-        self.assertEqual(sizeof(X), sizeof(c_int)*2)
+        self.assertEqual(sizeof(X), alignment(c_int)+sizeof(c_int))
 
     def test_mixed_3(self):
         class X(Structure):
@@ -246,7 +244,7 @@
             _anonymous_ = ["_"]
             _fields_ = [("_", X)]
 
-    @unittest.skipUnless(hasattr(ctypes, "c_uint32"), "c_int32 is required")
+    @need_symbol('c_uint32')
     def test_uint32(self):
         class X(Structure):
             _fields_ = [("a", c_uint32, 32)]
@@ -256,7 +254,7 @@
         x.a = 0xFDCBA987
         self.assertEqual(x.a, 0xFDCBA987)
 
-    @unittest.skipUnless(hasattr(ctypes, "c_uint64"), "c_int64 is required")
+    @need_symbol('c_uint64')
     def test_uint64(self):
         class X(Structure):
             _fields_ = [("a", c_uint64, 64)]
diff --git a/lib-python/2.7/ctypes/test/test_buffers.py b/lib-python/2.7/ctypes/test/test_buffers.py
--- a/lib-python/2.7/ctypes/test/test_buffers.py
+++ b/lib-python/2.7/ctypes/test/test_buffers.py
@@ -1,4 +1,5 @@
 from ctypes import *
+from ctypes.test import need_symbol
 import unittest
 
 class StringBufferTestCase(unittest.TestCase):
@@ -7,12 +8,12 @@
         b = create_string_buffer(32)
         self.assertEqual(len(b), 32)
         self.assertEqual(sizeof(b), 32 * sizeof(c_char))
-        self.assertTrue(type(b[0]) is str)
+        self.assertIs(type(b[0]), str)
 
         b = create_string_buffer("abc")
         self.assertEqual(len(b), 4) # trailing nul char
         self.assertEqual(sizeof(b), 4 * sizeof(c_char))
-        self.assertTrue(type(b[0]) is str)
+        self.assertIs(type(b[0]), str)
         self.assertEqual(b[0], "a")
         self.assertEqual(b[:], "abc\0")
         self.assertEqual(b[::], "abc\0")
@@ -36,39 +37,36 @@
         self.assertEqual(b[::2], "ac")
         self.assertEqual(b[::5], "a")
 
-    try:
-        c_wchar
-    except NameError:
-        pass
-    else:
-        def test_unicode_buffer(self):
-            b = create_unicode_buffer(32)
-            self.assertEqual(len(b), 32)
-            self.assertEqual(sizeof(b), 32 * sizeof(c_wchar))
-            self.assertTrue(type(b[0]) is unicode)
+    @need_symbol('c_wchar')
+    def test_unicode_buffer(self):
+        b = create_unicode_buffer(32)
+        self.assertEqual(len(b), 32)
+        self.assertEqual(sizeof(b), 32 * sizeof(c_wchar))
+        self.assertIs(type(b[0]), unicode)
 
-            b = create_unicode_buffer(u"abc")
-            self.assertEqual(len(b), 4) # trailing nul char
-            self.assertEqual(sizeof(b), 4 * sizeof(c_wchar))
-            self.assertTrue(type(b[0]) is unicode)
-            self.assertEqual(b[0], u"a")
-            self.assertEqual(b[:], "abc\0")
-            self.assertEqual(b[::], "abc\0")
-            self.assertEqual(b[::-1], "\0cba")
-            self.assertEqual(b[::2], "ac")
-            self.assertEqual(b[::5], "a")
+        b = create_unicode_buffer(u"abc")
+        self.assertEqual(len(b), 4) # trailing nul char
+        self.assertEqual(sizeof(b), 4 * sizeof(c_wchar))
+        self.assertIs(type(b[0]), unicode)
+        self.assertEqual(b[0], u"a")
+        self.assertEqual(b[:], "abc\0")
+        self.assertEqual(b[::], "abc\0")
+        self.assertEqual(b[::-1], "\0cba")
+        self.assertEqual(b[::2], "ac")
+        self.assertEqual(b[::5], "a")
 
-        def test_unicode_conversion(self):
-            b = create_unicode_buffer("abc")
-            self.assertEqual(len(b), 4) # trailing nul char
-            self.assertEqual(sizeof(b), 4 * sizeof(c_wchar))
-            self.assertTrue(type(b[0]) is unicode)
-            self.assertEqual(b[0], u"a")
-            self.assertEqual(b[:], "abc\0")
-            self.assertEqual(b[::], "abc\0")
-            self.assertEqual(b[::-1], "\0cba")
-            self.assertEqual(b[::2], "ac")
-            self.assertEqual(b[::5], "a")
+    @need_symbol('c_wchar')
+    def test_unicode_conversion(self):
+        b = create_unicode_buffer("abc")
+        self.assertEqual(len(b), 4) # trailing nul char
+        self.assertEqual(sizeof(b), 4 * sizeof(c_wchar))
+        self.assertIs(type(b[0]), unicode)
+        self.assertEqual(b[0], u"a")
+        self.assertEqual(b[:], "abc\0")
+        self.assertEqual(b[::], "abc\0")
+        self.assertEqual(b[::-1], "\0cba")
+        self.assertEqual(b[::2], "ac")
+        self.assertEqual(b[::5], "a")
 
 if __name__ == "__main__":
     unittest.main()
diff --git a/lib-python/2.7/ctypes/test/test_byteswap.py b/lib-python/2.7/ctypes/test/test_byteswap.py
--- a/lib-python/2.7/ctypes/test/test_byteswap.py
+++ b/lib-python/2.7/ctypes/test/test_byteswap.py
@@ -15,7 +15,8 @@
 # For Structures and Unions, these types are created on demand.
 
 class Test(unittest.TestCase):
-    def X_test(self):
+    @unittest.skip('test disabled')
+    def test_X(self):
         print >> sys.stderr,  sys.byteorder
         for i in range(32):
             bits = BITS()
@@ -25,11 +26,11 @@
     @xfail
     def test_endian_short(self):
         if sys.byteorder == "little":
-            self.assertTrue(c_short.__ctype_le__ is c_short)
-            self.assertTrue(c_short.__ctype_be__.__ctype_le__ is c_short)
+            self.assertIs(c_short.__ctype_le__, c_short)
+            self.assertIs(c_short.__ctype_be__.__ctype_le__, c_short)
         else:
-            self.assertTrue(c_short.__ctype_be__ is c_short)
-            self.assertTrue(c_short.__ctype_le__.__ctype_be__ is c_short)
+            self.assertIs(c_short.__ctype_be__, c_short)
+            self.assertIs(c_short.__ctype_le__.__ctype_be__, c_short)
         s = c_short.__ctype_be__(0x1234)
         self.assertEqual(bin(struct.pack(">h", 0x1234)), "1234")
         self.assertEqual(bin(s), "1234")
@@ -53,11 +54,11 @@
     @xfail
     def test_endian_int(self):
         if sys.byteorder == "little":
-            self.assertTrue(c_int.__ctype_le__ is c_int)
-            self.assertTrue(c_int.__ctype_be__.__ctype_le__ is c_int)
+            self.assertIs(c_int.__ctype_le__, c_int)
+            self.assertIs(c_int.__ctype_be__.__ctype_le__, c_int)
         else:
-            self.assertTrue(c_int.__ctype_be__ is c_int)
-            self.assertTrue(c_int.__ctype_le__.__ctype_be__ is c_int)
+            self.assertIs(c_int.__ctype_be__, c_int)
+            self.assertIs(c_int.__ctype_le__.__ctype_be__, c_int)
 
         s = c_int.__ctype_be__(0x12345678)
         self.assertEqual(bin(struct.pack(">i", 0x12345678)), "12345678")
@@ -82,11 +83,11 @@
     @xfail
     def test_endian_longlong(self):
         if sys.byteorder == "little":
-            self.assertTrue(c_longlong.__ctype_le__ is c_longlong)
-            self.assertTrue(c_longlong.__ctype_be__.__ctype_le__ is c_longlong)
+            self.assertIs(c_longlong.__ctype_le__, c_longlong)
+            self.assertIs(c_longlong.__ctype_be__.__ctype_le__, c_longlong)
         else:
-            self.assertTrue(c_longlong.__ctype_be__ is c_longlong)
-            self.assertTrue(c_longlong.__ctype_le__.__ctype_be__ is c_longlong)
+            self.assertIs(c_longlong.__ctype_be__, c_longlong)
+            self.assertIs(c_longlong.__ctype_le__.__ctype_be__, c_longlong)
 
         s = c_longlong.__ctype_be__(0x1234567890ABCDEF)
         self.assertEqual(bin(struct.pack(">q", 0x1234567890ABCDEF)), "1234567890ABCDEF")
@@ -111,11 +112,11 @@
     @xfail
     def test_endian_float(self):
         if sys.byteorder == "little":
-            self.assertTrue(c_float.__ctype_le__ is c_float)
-            self.assertTrue(c_float.__ctype_be__.__ctype_le__ is c_float)
+            self.assertIs(c_float.__ctype_le__, c_float)
+            self.assertIs(c_float.__ctype_be__.__ctype_le__, c_float)
         else:
-            self.assertTrue(c_float.__ctype_be__ is c_float)
-            self.assertTrue(c_float.__ctype_le__.__ctype_be__ is c_float)
+            self.assertIs(c_float.__ctype_be__, c_float)
+            self.assertIs(c_float.__ctype_le__.__ctype_be__, c_float)
         s = c_float(math.pi)
         self.assertEqual(bin(struct.pack("f", math.pi)), bin(s))
         # Hm, what's the precision of a float compared to a double?
@@ -130,11 +131,11 @@
     @xfail
     def test_endian_double(self):
         if sys.byteorder == "little":
-            self.assertTrue(c_double.__ctype_le__ is c_double)
-            self.assertTrue(c_double.__ctype_be__.__ctype_le__ is c_double)
+            self.assertIs(c_double.__ctype_le__, c_double)
+            self.assertIs(c_double.__ctype_be__.__ctype_le__, c_double)
         else:
-            self.assertTrue(c_double.__ctype_be__ is c_double)
-            self.assertTrue(c_double.__ctype_le__.__ctype_be__ is c_double)
+            self.assertIs(c_double.__ctype_be__, c_double)
+            self.assertIs(c_double.__ctype_le__.__ctype_be__, c_double)
         s = c_double(math.pi)
         self.assertEqual(s.value, math.pi)
         self.assertEqual(bin(struct.pack("d", math.pi)), bin(s))
@@ -146,14 +147,14 @@
         self.assertEqual(bin(struct.pack(">d", math.pi)), bin(s))
 
     def test_endian_other(self):
-        self.assertTrue(c_byte.__ctype_le__ is c_byte)
-        self.assertTrue(c_byte.__ctype_be__ is c_byte)
+        self.assertIs(c_byte.__ctype_le__, c_byte)
+        self.assertIs(c_byte.__ctype_be__, c_byte)
 
-        self.assertTrue(c_ubyte.__ctype_le__ is c_ubyte)
-        self.assertTrue(c_ubyte.__ctype_be__ is c_ubyte)
+        self.assertIs(c_ubyte.__ctype_le__, c_ubyte)
+        self.assertIs(c_ubyte.__ctype_be__, c_ubyte)
 
-        self.assertTrue(c_char.__ctype_le__ is c_char)
-        self.assertTrue(c_char.__ctype_be__ is c_char)
+        self.assertIs(c_char.__ctype_le__, c_char)
+        self.assertIs(c_char.__ctype_be__, c_char)
 
     @xfail
     def test_struct_fields_1(self):
diff --git a/lib-python/2.7/ctypes/test/test_callbacks.py b/lib-python/2.7/ctypes/test/test_callbacks.py
--- a/lib-python/2.7/ctypes/test/test_callbacks.py
+++ b/lib-python/2.7/ctypes/test/test_callbacks.py
@@ -1,5 +1,6 @@
 import unittest
 from ctypes import *
+from ctypes.test import need_symbol
 from ctypes.test import xfail
 import _ctypes_test
 
@@ -95,9 +96,10 @@
     # disabled: would now (correctly) raise a RuntimeWarning about
     # a memory leak.  A callback function cannot return a non-integral
     # C type without causing a memory leak.
-##    def test_char_p(self):
-##        self.check_type(c_char_p, "abc")
-##        self.check_type(c_char_p, "def")
+    @unittest.skip('test disabled')
+    def test_char_p(self):
+        self.check_type(c_char_p, "abc")
+        self.check_type(c_char_p, "def")
 
     @xfail
     def test_pyobject(self):
@@ -150,13 +152,12 @@
         CFUNCTYPE(None)(lambda x=Nasty(): None)
 
 
-try:
-    WINFUNCTYPE
-except NameError:
-    pass
-else:
-    class StdcallCallbacks(Callbacks):
+ at need_symbol('WINFUNCTYPE')
+class StdcallCallbacks(Callbacks):
+    try:
         functype = WINFUNCTYPE
+    except NameError:
+        pass
 
 ################################################################
 
@@ -186,7 +187,7 @@
         from ctypes.util import find_library
         libc_path = find_library("c")
         if not libc_path:
-            return # cannot test
+            self.skipTest('could not find libc')
         libc = CDLL(libc_path)
 
         @CFUNCTYPE(c_int, POINTER(c_int), POINTER(c_int))
@@ -198,23 +199,19 @@
         libc.qsort(array, len(array), sizeof(c_int), cmp_func)
         self.assertEqual(array[:], [1, 5, 7, 33, 99])
 
-    try:
-        WINFUNCTYPE
-    except NameError:
-        pass
-    else:
-        def test_issue_8959_b(self):
-            from ctypes.wintypes import BOOL, HWND, LPARAM
+    @need_symbol('WINFUNCTYPE')
+    def test_issue_8959_b(self):
+        from ctypes.wintypes import BOOL, HWND, LPARAM
+        global windowCount
+        windowCount = 0
+
+        @WINFUNCTYPE(BOOL, HWND, LPARAM)
+        def EnumWindowsCallbackFunc(hwnd, lParam):
             global windowCount
-            windowCount = 0
+            windowCount += 1
+            return True #Allow windows to keep enumerating
 
-            @WINFUNCTYPE(BOOL, HWND, LPARAM)
-            def EnumWindowsCallbackFunc(hwnd, lParam):
-                global windowCount
-                windowCount += 1
-                return True #Allow windows to keep enumerating
-
-            windll.user32.EnumWindows(EnumWindowsCallbackFunc, 0)
+        windll.user32.EnumWindows(EnumWindowsCallbackFunc, 0)
 
     def test_callback_register_int(self):
         # Issue #8275: buggy handling of callback args under Win64
diff --git a/lib-python/2.7/ctypes/test/test_cast.py b/lib-python/2.7/ctypes/test/test_cast.py
--- a/lib-python/2.7/ctypes/test/test_cast.py
+++ b/lib-python/2.7/ctypes/test/test_cast.py
@@ -1,4 +1,5 @@
 from ctypes import *
+from ctypes.test import need_symbol
 import unittest
 import sys
 
@@ -38,14 +39,14 @@
 
         p = cast(array, POINTER(c_char_p))
         # array and p share a common _objects attribute
-        self.assertTrue(p._objects is array._objects)
+        self.assertIs(p._objects, array._objects)
         self.assertEqual(array._objects, {'0': "foo bar", id(array): array})
         p[0] = "spam spam"
         self.assertEqual(p._objects, {'0': "spam spam", id(array): array})
-        self.assertTrue(array._objects is p._objects)
+        self.assertIs(array._objects, p._objects)
         p[1] = "foo bar"
         self.assertEqual(p._objects, {'1': 'foo bar', '0': "spam spam", id(array): array})
-        self.assertTrue(array._objects is p._objects)
+        self.assertIs(array._objects, p._objects)
 
     def test_other(self):
         p = cast((c_int * 4)(1, 2, 3, 4), POINTER(c_int))
@@ -75,15 +76,11 @@
         self.assertEqual(cast(cast(s, c_void_p), c_char_p).value,
                              "hiho")
 
-    try:
-        c_wchar_p
-    except NameError:
-        pass
-    else:
-        def test_wchar_p(self):
-            s = c_wchar_p("hiho")
-            self.assertEqual(cast(cast(s, c_void_p), c_wchar_p).value,
-                                 "hiho")
+    @need_symbol('c_wchar_p')
+    def test_wchar_p(self):
+        s = c_wchar_p("hiho")
+        self.assertEqual(cast(cast(s, c_void_p), c_wchar_p).value,
+                             "hiho")
 
 if __name__ == "__main__":
     unittest.main()
diff --git a/lib-python/2.7/ctypes/test/test_cfuncs.py b/lib-python/2.7/ctypes/test/test_cfuncs.py
--- a/lib-python/2.7/ctypes/test/test_cfuncs.py
+++ b/lib-python/2.7/ctypes/test/test_cfuncs.py
@@ -3,6 +3,7 @@
 
 import unittest
 from ctypes import *
+from ctypes.test import need_symbol
 
 import _ctypes_test
 from test.test_support import impl_detail
@@ -196,7 +197,7 @@
 try:
     WinDLL
 except NameError:
-    pass
+    def stdcall_dll(*_): pass
 else:
     class stdcall_dll(WinDLL):
         def __getattr__(self, name):
@@ -206,9 +207,9 @@
             setattr(self, name, func)
             return func
 
-    class stdcallCFunctions(CFunctions):
-        _dll = stdcall_dll(_ctypes_test.__file__)
-        pass
+ at need_symbol('WinDLL')
+class stdcallCFunctions(CFunctions):
+    _dll = stdcall_dll(_ctypes_test.__file__)
 
 if __name__ == '__main__':
     unittest.main()
diff --git a/lib-python/2.7/ctypes/test/test_checkretval.py b/lib-python/2.7/ctypes/test/test_checkretval.py
--- a/lib-python/2.7/ctypes/test/test_checkretval.py
+++ b/lib-python/2.7/ctypes/test/test_checkretval.py
@@ -1,6 +1,7 @@
 import unittest
 
 from ctypes import *
+from ctypes.test import need_symbol
 
 class CHECKED(c_int):
     def _check_retval_(value):
@@ -25,15 +26,11 @@
         del dll._testfunc_p_p.restype
         self.assertEqual(42, dll._testfunc_p_p(42))
 
-    try:
-        oledll
-    except NameError:
-        pass
-    else:
-        def test_oledll(self):
-            self.assertRaises(WindowsError,
-                                  oledll.oleaut32.CreateTypeLib2,
-                                  0, None, None)
+    @need_symbol('oledll')
+    def test_oledll(self):
+        self.assertRaises(WindowsError,
+                              oledll.oleaut32.CreateTypeLib2,
+                              0, None, None)
 
 if __name__ == "__main__":
     unittest.main()
diff --git a/lib-python/2.7/ctypes/test/test_errcheck.py b/lib-python/2.7/ctypes/test/test_errcheck.py
deleted file mode 100644
--- a/lib-python/2.7/ctypes/test/test_errcheck.py
+++ /dev/null
@@ -1,19 +0,0 @@
-import sys
-from ctypes import *
-
-##class HMODULE(Structure):
-##    _fields_ = [("value", c_void_p)]
-
-##    def __repr__(self):
-##        return "" % self.value
-
-##windll.kernel32.GetModuleHandleA.restype = HMODULE
-
-##print windll.kernel32.GetModuleHandleA("python23.dll")
-##print hex(sys.dllhandle)
-
-##def nonzero(handle):
-##    return (GetLastError(), handle)
-
-##windll.kernel32.GetModuleHandleA.errcheck = nonzero
-##print windll.kernel32.GetModuleHandleA("spam")
diff --git a/lib-python/2.7/ctypes/test/test_find.py b/lib-python/2.7/ctypes/test/test_find.py
--- a/lib-python/2.7/ctypes/test/test_find.py
+++ b/lib-python/2.7/ctypes/test/test_find.py
@@ -1,4 +1,5 @@
 import unittest
+import os
 import sys
 from ctypes import *
 from ctypes.util import find_library
@@ -40,43 +41,43 @@
             except OSError:
                 pass
 
-    if lib_gl:
-        def test_gl(self):
-            if self.gl:
-                self.gl.glClearIndex
+    @unittest.skipUnless(lib_gl, 'lib_gl not available')
+    def test_gl(self):
+        if self.gl:
+            self.gl.glClearIndex
 
-    if lib_glu:
-        def test_glu(self):
-            if self.glu:
-                self.glu.gluBeginCurve
+    @unittest.skipUnless(lib_glu, 'lib_glu not available')
+    def test_glu(self):
+        if self.glu:
+            self.glu.gluBeginCurve
 
-    if lib_gle:
-        def test_gle(self):
-            if self.gle:
-                self.gle.gleGetJoinStyle
+    @unittest.skipUnless(lib_gle, 'lib_gle not available')
+    def test_gle(self):
+        if self.gle:
+            self.gle.gleGetJoinStyle
 
-##if os.name == "posix" and sys.platform != "darwin":
-
-##    # On platforms where the default shared library suffix is '.so',
-##    # at least some libraries can be loaded as attributes of the cdll
-##    # object, since ctypes now tries loading the lib again
-##    # with '.so' appended of the first try fails.
-##    #
-##    # Won't work for libc, unfortunately.  OTOH, it isn't
-##    # needed for libc since this is already mapped into the current
-##    # process (?)
-##    #
-##    # On MAC OSX, it won't work either, because dlopen() needs a full path,
-##    # and the default suffix is either none or '.dylib'.
-
-##    class LoadLibs(unittest.TestCase):
-##        def test_libm(self):
-##            import math
-##            libm = cdll.libm
-##            sqrt = libm.sqrt
-##            sqrt.argtypes = (c_double,)
-##            sqrt.restype = c_double
-##            self.assertEqual(sqrt(2), math.sqrt(2))
+# On platforms where the default shared library suffix is '.so',
+# at least some libraries can be loaded as attributes of the cdll
+# object, since ctypes now tries loading the lib again
+# with '.so' appended of the first try fails.
+#
+# Won't work for libc, unfortunately.  OTOH, it isn't
+# needed for libc since this is already mapped into the current
+# process (?)
+#
+# On MAC OSX, it won't work either, because dlopen() needs a full path,
+# and the default suffix is either none or '.dylib'.
+ at unittest.skip('test disabled')
+ at unittest.skipUnless(os.name=="posix" and sys.platform != "darwin",
+                     'test not suitable for this platform')
+class LoadLibs(unittest.TestCase):
+    def test_libm(self):
+        import math
+        libm = cdll.libm
+        sqrt = libm.sqrt
+        sqrt.argtypes = (c_double,)
+        sqrt.restype = c_double
+        self.assertEqual(sqrt(2), math.sqrt(2))
 
 if __name__ == "__main__":
     unittest.main()
diff --git a/lib-python/2.7/ctypes/test/test_frombuffer.py b/lib-python/2.7/ctypes/test/test_frombuffer.py
--- a/lib-python/2.7/ctypes/test/test_frombuffer.py
+++ b/lib-python/2.7/ctypes/test/test_frombuffer.py

From noreply at buildbot.pypy.org  Tue Nov 11 18:52:02 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Tue, 11 Nov 2014 18:52:02 +0100 (CET)
Subject: [pypy-commit] pypy stmgc-c7: fix
Message-ID: <20141111175202.187D81C02CB@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: stmgc-c7
Changeset: r74454:18ed12091d09
Date: 2014-11-11 18:47 +0100
http://bitbucket.org/pypy/pypy/changeset/18ed12091d09/

Log:	fix

diff --git a/rpython/memory/gctransform/stmframework.py b/rpython/memory/gctransform/stmframework.py
--- a/rpython/memory/gctransform/stmframework.py
+++ b/rpython/memory/gctransform/stmframework.py
@@ -5,7 +5,7 @@
 from rpython.memory.gctransform.framework import (TYPE_ID,
      BaseFrameworkGCTransformer, BaseRootWalker, sizeofaddr)
 from rpython.memory.gctypelayout import WEAKREF, WEAKREFPTR
-from rpython.memory.gc.stm import StmGC
+from rpython.memory.gc.stmgc import StmGC
 from rpython.rtyper import rmodel, llannotation
 from rpython.translator.backendopt.support import var_needsgc
 from rpython.rlib.objectmodel import specialize

From noreply at buildbot.pypy.org  Tue Nov 11 18:52:03 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Tue, 11 Nov 2014 18:52:03 +0100 (CET)
Subject: [pypy-commit] pypy stmgc-c7: fix some imports
Message-ID: <20141111175203.4351F1C02CB@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: stmgc-c7
Changeset: r74455:26ebcac51ad5
Date: 2014-11-11 18:47 +0100
http://bitbucket.org/pypy/pypy/changeset/26ebcac51ad5/

Log:	fix some imports

diff --git a/rpython/rtyper/lltypesystem/rstr.py b/rpython/rtyper/lltypesystem/rstr.py
--- a/rpython/rtyper/lltypesystem/rstr.py
+++ b/rpython/rtyper/lltypesystem/rstr.py
@@ -394,6 +394,7 @@
             x = LLHelpers._ll_compute_strhash(s)
         return x
 
+    @staticmethod
     @jit.dont_look_inside
     def _ll_compute_strhash(s):
         x = _hash_string(s.chars)
diff --git a/rpython/translator/stm/inevitable.py b/rpython/translator/stm/inevitable.py
--- a/rpython/translator/stm/inevitable.py
+++ b/rpython/translator/stm/inevitable.py
@@ -1,4 +1,5 @@
-from rpython.rtyper.lltypesystem import lltype, lloperation, rclass
+from rpython.rtyper.lltypesystem import lltype, lloperation
+from rpython.rtyper import rclass
 from rpython.translator.stm.support import is_immutable
 from rpython.flowspace.model import SpaceOperation, Constant
 from rpython.translator.unsimplify import varoftype
diff --git a/rpython/translator/stm/test/targetdemo2.py b/rpython/translator/stm/test/targetdemo2.py
--- a/rpython/translator/stm/test/targetdemo2.py
+++ b/rpython/translator/stm/test/targetdemo2.py
@@ -4,7 +4,8 @@
 from rpython.rlib.objectmodel import we_are_translated
 from rpython.rlib.objectmodel import compute_identity_hash
 from rpython.rlib.debug import ll_assert
-from rpython.rtyper.lltypesystem import lltype, rffi, rclass
+from rpython.rtyper.lltypesystem import lltype, rffi
+from rpython.rtyper import rclass
 
 
 class Node:
diff --git a/rpython/translator/stm/test/test_ztranslated.py b/rpython/translator/stm/test/test_ztranslated.py
--- a/rpython/translator/stm/test/test_ztranslated.py
+++ b/rpython/translator/stm/test/test_ztranslated.py
@@ -2,8 +2,8 @@
 from rpython.rlib import rstm, rgc, objectmodel
 from rpython.rlib.debug import debug_print
 from rpython.rtyper.lltypesystem import lltype, llmemory, rffi
-from rpython.rtyper.lltypesystem.rclass import OBJECTPTR
 from rpython.rtyper.lltypesystem.lloperation import llop
+from rpython.rtyper.rclass import OBJECTPTR
 from rpython.rtyper.annlowlevel import cast_instance_to_gcref
 from rpython.rtyper.annlowlevel import cast_gcref_to_instance
 from rpython.translator.stm.test.support import CompiledSTMTests

From noreply at buildbot.pypy.org  Tue Nov 11 18:52:04 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Tue, 11 Nov 2014 18:52:04 +0100 (CET)
Subject: [pypy-commit] pypy improve-gc-tracing-hooks: Close the branch,
	ready for merging
Message-ID: <20141111175204.66E0D1C02CB@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: improve-gc-tracing-hooks
Changeset: r74456:367c01713036
Date: 2014-11-11 18:50 +0100
http://bitbucket.org/pypy/pypy/changeset/367c01713036/

Log:	Close the branch, ready for merging


From noreply at buildbot.pypy.org  Tue Nov 11 18:52:05 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Tue, 11 Nov 2014 18:52:05 +0100 (CET)
Subject: [pypy-commit] pypy default: hg merge improve-gc-tracing-hooks
Message-ID: <20141111175205.A46A01C02CB@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r74457:9069f9c784ba
Date: 2014-11-11 18:51 +0100
http://bitbucket.org/pypy/pypy/changeset/9069f9c784ba/

Log:	hg merge improve-gc-tracing-hooks

	A small branch (fijal, arigo) that cleans up the custom tracers
	called by the GC.

diff --git a/rpython/jit/backend/llsupport/jitframe.py b/rpython/jit/backend/llsupport/jitframe.py
--- a/rpython/jit/backend/llsupport/jitframe.py
+++ b/rpython/jit/backend/llsupport/jitframe.py
@@ -3,6 +3,7 @@
 from rpython.rlib.objectmodel import specialize
 from rpython.rlib.debug import ll_assert
 from rpython.rlib.objectmodel import enforceargs
+from rpython.rlib import rgc
 
 SIZEOFSIGNED = rffi.sizeof(lltype.Signed)
 IS_32BIT = (SIZEOFSIGNED == 4)
@@ -45,6 +46,7 @@
 # detailed explanation how it is on your architecture
 
 def jitframe_allocate(frame_info):
+    rgc.register_custom_trace_hook(JITFRAME, lambda_jitframe_trace)
     frame = lltype.malloc(JITFRAME, frame_info.jfi_frame_depth)
     frame.jf_frame_info = frame_info
     frame.jf_extra_stack_depth = 0
@@ -80,8 +82,6 @@
     ('jf_guard_exc', llmemory.GCREF),
     # in case the frame got reallocated, we have to forward it somewhere
     ('jf_forward', lltype.Ptr(JITFRAME)),
-    # absolutely useless field used to make up for tracing hooks inflexibilities
-    ('jf_gc_trace_state', lltype.Signed),
     # the actual frame
     ('jf_frame', lltype.Array(lltype.Signed)),
     # note that we keep length field, because it's crucial to have the data
@@ -105,75 +105,38 @@
 UNSIGN_SIZE = llmemory.sizeof(lltype.Unsigned)
 STACK_DEPTH_OFS = getofs('jf_extra_stack_depth')
 
-def jitframe_trace(obj_addr, prev):
-    if prev == llmemory.NULL:
-        (obj_addr + getofs('jf_gc_trace_state')).signed[0] = -1
-        return obj_addr + getofs('jf_descr')
-    fld = (obj_addr + getofs('jf_gc_trace_state')).signed[0]
-    if fld < 0:
-        if fld == -1:
-            (obj_addr + getofs('jf_gc_trace_state')).signed[0] = -2
-            return obj_addr + getofs('jf_force_descr')
-        elif fld == -2:
-            (obj_addr + getofs('jf_gc_trace_state')).signed[0] = -3
-            return obj_addr + getofs('jf_savedata')
-        elif fld == -3:
-            (obj_addr + getofs('jf_gc_trace_state')).signed[0] = -4
-            return obj_addr + getofs('jf_guard_exc')
-        elif fld == -4:
-            (obj_addr + getofs('jf_gc_trace_state')).signed[0] = -5
-            return obj_addr + getofs('jf_forward')
-        else:
-            if not (obj_addr + getofs('jf_gcmap')).address[0]:
-                return llmemory.NULL    # done
-            else:
-                fld = 0    # fall-through
-    # bit pattern
-    # decode the pattern
+def jitframe_trace(gc, obj_addr, callback, arg):
+    gc._trace_callback(callback, arg, obj_addr + getofs('jf_descr'))
+    gc._trace_callback(callback, arg, obj_addr + getofs('jf_force_descr'))
+    gc._trace_callback(callback, arg, obj_addr + getofs('jf_savedata'))
+    gc._trace_callback(callback, arg, obj_addr + getofs('jf_guard_exc'))
+    gc._trace_callback(callback, arg, obj_addr + getofs('jf_forward'))
+
     if IS_32BIT:
-        # 32 possible bits
-        state = fld & 0x1f
-        no = fld >> 5
         MAX = 32
     else:
-        # 64 possible bits
-        state = fld & 0x3f
-        no = fld >> 6
         MAX = 64
     gcmap = (obj_addr + getofs('jf_gcmap')).address[0]
+    if not gcmap:
+        return      # done
     gcmap_lgt = (gcmap + GCMAPLENGTHOFS).signed[0]
+    no = 0
     while no < gcmap_lgt:
         cur = (gcmap + GCMAPBASEOFS + UNSIGN_SIZE * no).unsigned[0]
-        while not (cur & (1 << state)):
-            state += 1
-            if state == MAX:
-                no += 1
-                state = 0
-                break      # next iteration of the outermost loop
-        else:
-            # found it
-            index = no * SIZEOFSIGNED * 8 + state
-            # save new state
-            state += 1
-            if state == MAX:
-                no += 1
-                state = 0
-            if IS_32BIT:
-                new_state = state | (no << 5)
-            else:
-                new_state = state | (no << 6)
-            (obj_addr + getofs('jf_gc_trace_state')).signed[0] = new_state
-            # sanity check
-            frame_lgt = (obj_addr + getofs('jf_frame') + LENGTHOFS).signed[0]
-            ll_assert(index < frame_lgt, "bogus frame field get")
-            return (obj_addr + getofs('jf_frame') + BASEITEMOFS + SIGN_SIZE *
-                    (index))
-    return llmemory.NULL
-
-CUSTOMTRACEFUNC = lltype.FuncType([llmemory.Address, llmemory.Address],
-                                  llmemory.Address)
-jitframe_trace_ptr = llhelper(lltype.Ptr(CUSTOMTRACEFUNC), jitframe_trace)
-
-lltype.attachRuntimeTypeInfo(JITFRAME, customtraceptr=jitframe_trace_ptr)
+        bitindex = 0
+        while bitindex < MAX:
+            if cur & (1 << bitindex):
+                # the 'bitindex' is set in 'cur'
+                index = no * SIZEOFSIGNED * 8 + bitindex
+                # sanity check
+                frame_lgt = (obj_addr + getofs('jf_frame') + LENGTHOFS) \
+                    .signed[0]
+                ll_assert(index < frame_lgt, "bogus frame field get")
+                gc._trace_callback(callback, arg,
+                                   obj_addr + getofs('jf_frame') +
+                                   BASEITEMOFS + SIGN_SIZE * index)
+            bitindex += 1
+        no += 1
+lambda_jitframe_trace = lambda: jitframe_trace
 
 JITFRAMEPTR = lltype.Ptr(JITFRAME)
diff --git a/rpython/jit/backend/llsupport/test/test_gc.py b/rpython/jit/backend/llsupport/test/test_gc.py
--- a/rpython/jit/backend/llsupport/test/test_gc.py
+++ b/rpython/jit/backend/llsupport/test/test_gc.py
@@ -254,11 +254,15 @@
     frame.jf_gcmap[2] = r_uint(2 | 16 | 32 | 128)
     frame.jf_gcmap[3] = r_uint(0)
     frame_adr = llmemory.cast_ptr_to_adr(frame)
+    #
     all_addrs = []
-    next = jitframe.jitframe_trace(frame_adr, llmemory.NULL)
-    while next:
-        all_addrs.append(next)
-        next = jitframe.jitframe_trace(frame_adr, next)
+    class FakeGC:
+        def _trace_callback(self, callback, arg, addr):
+            assert callback == "hello"
+            assert arg == "world"
+            all_addrs.append(addr)
+    jitframe.jitframe_trace(FakeGC(), frame_adr, "hello", "world")
+    #
     counter = 0
     for name in jitframe.JITFRAME._names:
         TP = getattr(jitframe.JITFRAME, name)
@@ -297,12 +301,12 @@
     frame.jf_gcmap[0] = r_uint(18446744073441116160)
     frame.jf_gcmap[1] = r_uint(18446740775107559407)
     frame.jf_gcmap[2] = r_uint(3)
-    all_addrs = []
     frame_adr = llmemory.cast_ptr_to_adr(frame)
-    next = jitframe.jitframe_trace(frame_adr, llmemory.NULL)
-    while next:
-        all_addrs.append(next)
-        next = jitframe.jitframe_trace(frame_adr, next)
+    class FakeGC:
+        def _trace_callback(self, callback, arg, addr):
+            assert callback == "hello"
+            assert arg == "world"
+    jitframe.jitframe_trace(FakeGC(), frame_adr, "hello", "world")
     # assert did not hang
 
     lltype.free(frame_info, flavor='raw')
diff --git a/rpython/memory/gc/base.py b/rpython/memory/gc/base.py
--- a/rpython/memory/gc/base.py
+++ b/rpython/memory/gc/base.py
@@ -71,7 +71,6 @@
                             member_index,
                             is_rpython_class,
                             has_custom_trace,
-                            get_custom_trace,
                             fast_path_tracing,
                             has_gcptr,
                             cannot_pin):
@@ -90,7 +89,6 @@
         self.member_index = member_index
         self.is_rpython_class = is_rpython_class
         self.has_custom_trace = has_custom_trace
-        self.get_custom_trace = get_custom_trace
         self.fast_path_tracing = fast_path_tracing
         self.has_gcptr = has_gcptr
         self.cannot_pin = cannot_pin
@@ -235,16 +233,14 @@
                 item += itemlength
                 length -= 1
         if self.has_custom_trace(typeid):
-            generator = self.get_custom_trace(typeid)
-            item = llmemory.NULL
-            while True:
-                item = generator(obj, item)
-                if not item:
-                    break
-                if self.points_to_valid_gc_object(item):
-                    callback(item, arg)
+            self.custom_trace_dispatcher(obj, typeid, callback, arg)
     _trace_slow_path._annspecialcase_ = 'specialize:arg(2)'
 
+    def _trace_callback(self, callback, arg, addr):
+        if self.is_valid_gc_object(addr.address[0]):
+            callback(addr, arg)
+    _trace_callback._annspecialcase_ = 'specialize:arg(1)'
+
     def trace_partial(self, obj, start, stop, callback, arg):
         """Like trace(), but only walk the array part, for indices in
         range(start, stop).  Must only be called if has_gcptr_in_varsize().
diff --git a/rpython/memory/gctransform/framework.py b/rpython/memory/gctransform/framework.py
--- a/rpython/memory/gctransform/framework.py
+++ b/rpython/memory/gctransform/framework.py
@@ -1,9 +1,11 @@
 from rpython.annotator import model as annmodel
 from rpython.rtyper.llannotation import SomeAddress, SomePtr
 from rpython.rlib import rgc
+from rpython.rlib.objectmodel import specialize
+from rpython.rlib.unroll import unrolling_iterable
 from rpython.rtyper import rmodel, annlowlevel
 from rpython.rtyper.lltypesystem import lltype, llmemory, rffi, llgroup
-from rpython.rtyper.lltypesystem.lloperation import LL_OPERATIONS
+from rpython.rtyper.lltypesystem.lloperation import LL_OPERATIONS, llop
 from rpython.memory import gctypelayout
 from rpython.memory.gctransform.log import log
 from rpython.memory.gctransform.support import get_rtti, ll_call_destructor
@@ -239,6 +241,7 @@
             root_walker.need_stacklet_support(self, getfn)
 
         self.layoutbuilder.encode_type_shapes_now()
+        self.create_custom_trace_funcs(gcdata.gc, translator.rtyper)
 
         annhelper.finish()   # at this point, annotate all mix-level helpers
         annhelper.backend_optimize()
@@ -502,6 +505,29 @@
                                                    [SomeAddress()],
                                                    annmodel.s_None)
 
+    def create_custom_trace_funcs(self, gc, rtyper):
+        custom_trace_funcs = tuple(rtyper.custom_trace_funcs)
+        rtyper.custom_trace_funcs = custom_trace_funcs
+        # too late to register new custom trace functions afterwards
+
+        custom_trace_funcs_unrolled = unrolling_iterable(
+            [(self.get_type_id(TP), func) for TP, func in custom_trace_funcs])
+
+        @specialize.arg(2)
+        def custom_trace_dispatcher(obj, typeid, callback, arg):
+            for type_id_exp, func in custom_trace_funcs_unrolled:
+                if (llop.combine_ushort(lltype.Signed, typeid, 0) ==
+                    llop.combine_ushort(lltype.Signed, type_id_exp, 0)):
+                    func(gc, obj, callback, arg)
+                    return
+            else:
+                assert False
+
+        gc.custom_trace_dispatcher = custom_trace_dispatcher
+
+        for TP, func in custom_trace_funcs:
+            self.gcdata._has_got_custom_trace(self.get_type_id(TP))
+            specialize.arg(2)(func)
 
     def consider_constant(self, TYPE, value):
         self.layoutbuilder.consider_constant(TYPE, value, self.gcdata.gc)
diff --git a/rpython/memory/gctransform/shadowstack.py b/rpython/memory/gctransform/shadowstack.py
--- a/rpython/memory/gctransform/shadowstack.py
+++ b/rpython/memory/gctransform/shadowstack.py
@@ -73,16 +73,13 @@
             return top
         self.decr_stack = decr_stack
 
-        root_iterator = get_root_iterator(gctransformer)
         def walk_stack_root(callback, start, end):
-            root_iterator.setcontext(NonConstant(llmemory.NULL))
             gc = self.gc
             addr = end
-            while True:
-                addr = root_iterator.nextleft(gc, start, addr)
-                if addr == llmemory.NULL:
-                    return
-                callback(gc, addr)
+            while addr != start:
+                addr -= sizeofaddr
+                if gc.points_to_valid_gc_object(addr):
+                    callback(gc, addr)
         self.rootstackhook = walk_stack_root
 
         self.shadow_stack_pool = ShadowStackPool(gcdata)
@@ -349,25 +346,6 @@
                 raise MemoryError
 
 
-def get_root_iterator(gctransformer):
-    if hasattr(gctransformer, '_root_iterator'):
-        return gctransformer._root_iterator     # if already built
-    class RootIterator(object):
-        def _freeze_(self):
-            return True
-        def setcontext(self, context):
-            pass
-        def nextleft(self, gc, start, addr):
-            while addr != start:
-                addr -= sizeofaddr
-                if gc.points_to_valid_gc_object(addr):
-                    return addr
-            return llmemory.NULL
-    result = RootIterator()
-    gctransformer._root_iterator = result
-    return result
-
-
 def get_shadowstackref(root_walker, gctransformer):
     if hasattr(gctransformer, '_SHADOWSTACKREF'):
         return gctransformer._SHADOWSTACKREF
@@ -381,19 +359,19 @@
                                      rtti=True)
     SHADOWSTACKREFPTR.TO.become(SHADOWSTACKREF)
 
+    def customtrace(gc, obj, callback, arg):
+        obj = llmemory.cast_adr_to_ptr(obj, SHADOWSTACKREFPTR)
+        addr = obj.top
+        start = obj.base
+        while addr != start:
+            addr -= sizeofaddr
+            gc._trace_callback(callback, arg, addr)
+
     gc = gctransformer.gcdata.gc
-    root_iterator = get_root_iterator(gctransformer)
-
-    def customtrace(obj, prev):
-        obj = llmemory.cast_adr_to_ptr(obj, SHADOWSTACKREFPTR)
-        if not prev:
-            root_iterator.setcontext(obj.context)
-            prev = obj.top
-        return root_iterator.nextleft(gc, obj.base, prev)
-
-    CUSTOMTRACEFUNC = lltype.FuncType([llmemory.Address, llmemory.Address],
-                                      llmemory.Address)
-    customtraceptr = llhelper(lltype.Ptr(CUSTOMTRACEFUNC), customtrace)
+    assert not hasattr(gc, 'custom_trace_dispatcher')
+    # ^^^ create_custom_trace_funcs() must not run before this
+    gctransformer.translator.rtyper.custom_trace_funcs.append(
+        (SHADOWSTACKREF, customtrace))
 
     def shadowstack_destructor(shadowstackref):
         if root_walker.stacklet_support:
@@ -414,8 +392,7 @@
     destrptr = gctransformer.annotate_helper(shadowstack_destructor,
                                              [SHADOWSTACKREFPTR], lltype.Void)
 
-    lltype.attachRuntimeTypeInfo(SHADOWSTACKREF, customtraceptr=customtraceptr,
-                                 destrptr=destrptr)
+    lltype.attachRuntimeTypeInfo(SHADOWSTACKREF, destrptr=destrptr)
 
     gctransformer._SHADOWSTACKREF = SHADOWSTACKREF
     return SHADOWSTACKREF
diff --git a/rpython/memory/gctypelayout.py b/rpython/memory/gctypelayout.py
--- a/rpython/memory/gctypelayout.py
+++ b/rpython/memory/gctypelayout.py
@@ -21,18 +21,12 @@
     # It is called with the object as first argument, and the previous
     # returned address (or NULL the first time) as the second argument.
     FINALIZER_FUNC = lltype.FuncType([llmemory.Address], lltype.Void)
-    CUSTOMTRACER_FUNC = lltype.FuncType([llmemory.Address, llmemory.Address],
-                                        llmemory.Address)
     FINALIZER = lltype.Ptr(FINALIZER_FUNC)
-    CUSTOMTRACER = lltype.Ptr(CUSTOMTRACER_FUNC)
-    EXTRA = lltype.Struct("type_info_extra",
-                          ('finalizer', FINALIZER),
-                          ('customtracer', CUSTOMTRACER))
 
     # structure describing the layout of a typeid
     TYPE_INFO = lltype.Struct("type_info",
         ("infobits",       lltype.Signed),    # combination of the T_xxx consts
-        ("extra",          lltype.Ptr(EXTRA)),
+        ("finalizer",      FINALIZER),
         ("fixedsize",      lltype.Signed),
         ("ofstoptrs",      lltype.Ptr(OFFSETS_TO_GC_PTR)),
         hints={'immutable': True},
@@ -92,18 +86,13 @@
         return (infobits & ANY) != 0
 
     def q_finalizer(self, typeid):
-        typeinfo = self.get(typeid)
-        if typeinfo.infobits & T_HAS_FINALIZER:
-            return typeinfo.extra.finalizer
-        else:
-            return lltype.nullptr(GCData.FINALIZER_FUNC)
+        return self.get(typeid).finalizer
 
     def q_light_finalizer(self, typeid):
         typeinfo = self.get(typeid)
         if typeinfo.infobits & T_HAS_LIGHTWEIGHT_FINALIZER:
-            return typeinfo.extra.finalizer
-        else:
-            return lltype.nullptr(GCData.FINALIZER_FUNC)
+            return typeinfo.finalizer
+        return lltype.nullptr(GCData.FINALIZER_FUNC)
 
     def q_offsets_to_gc_pointers(self, typeid):
         return self.get(typeid).ofstoptrs
@@ -141,12 +130,6 @@
         infobits = self.get(typeid).infobits
         return infobits & T_HAS_CUSTOM_TRACE != 0
 
-    def q_get_custom_trace(self, typeid):
-        ll_assert(self.q_has_custom_trace(typeid),
-                  "T_HAS_CUSTOM_TRACE missing")
-        typeinfo = self.get(typeid)
-        return typeinfo.extra.customtracer
-
     def q_fast_path_tracing(self, typeid):
         # return True if none of the flags T_HAS_GCPTR_IN_VARSIZE,
         # T_IS_GCARRAY_OF_GCPTR or T_HAS_CUSTOM_TRACE is set
@@ -173,11 +156,14 @@
             self.q_member_index,
             self.q_is_rpython_class,
             self.q_has_custom_trace,
-            self.q_get_custom_trace,
             self.q_fast_path_tracing,
             self.q_has_gcptr,
             self.q_cannot_pin)
 
+    def _has_got_custom_trace(self, typeid):
+        type_info = self.get(typeid)
+        type_info.infobits |= (T_HAS_CUSTOM_TRACE | T_HAS_GCPTR)
+
 
 # the lowest 16bits are used to store group member index
 T_MEMBER_INDEX              =   0xffff
@@ -186,9 +172,8 @@
 T_IS_GCARRAY_OF_GCPTR       = 0x040000
 T_IS_WEAKREF                = 0x080000
 T_IS_RPYTHON_INSTANCE       = 0x100000 # the type is a subclass of OBJECT
-T_HAS_FINALIZER             = 0x200000
-T_HAS_CUSTOM_TRACE          = 0x400000
-T_HAS_LIGHTWEIGHT_FINALIZER = 0x800000
+T_HAS_CUSTOM_TRACE          = 0x200000
+T_HAS_LIGHTWEIGHT_FINALIZER = 0x400000
 T_HAS_GCPTR                 = 0x1000000
 T_KEY_MASK                  = intmask(0xFE000000) # bug detection only
 T_KEY_VALUE                 = intmask(0x5A000000) # bug detection only
@@ -217,18 +202,11 @@
     #
     fptrs = builder.special_funcptr_for_type(TYPE)
     if fptrs:
-        extra = lltype.malloc(GCData.EXTRA, zero=True, immortal=True,
-                              flavor='raw')
         if "finalizer" in fptrs:
-            extra.finalizer = fptrs["finalizer"]
-            infobits |= T_HAS_FINALIZER
+            info.finalizer = fptrs["finalizer"]
         if "light_finalizer" in fptrs:
-            extra.finalizer = fptrs["light_finalizer"]
-            infobits |= T_HAS_FINALIZER | T_HAS_LIGHTWEIGHT_FINALIZER
-        if "custom_trace" in fptrs:
-            extra.customtracer = fptrs["custom_trace"]
-            infobits |= T_HAS_CUSTOM_TRACE | T_HAS_GCPTR
-        info.extra = extra
+            info.finalizer = fptrs["light_finalizer"]
+            infobits |= T_HAS_LIGHTWEIGHT_FINALIZER
     #
     if not TYPE._is_varsize():
         info.fixedsize = llarena.round_up_for_allocation(
@@ -420,7 +398,9 @@
         return None
 
     def initialize_gc_query_function(self, gc):
-        return GCData(self.type_info_group).set_query_functions(gc)
+        gcdata = GCData(self.type_info_group)
+        gcdata.set_query_functions(gc)
+        return gcdata
 
     def consider_constant(self, TYPE, value, gc):
         if value is not lltype.top_container(value):
diff --git a/rpython/memory/gcwrapper.py b/rpython/memory/gcwrapper.py
--- a/rpython/memory/gcwrapper.py
+++ b/rpython/memory/gcwrapper.py
@@ -29,7 +29,7 @@
                                                lltype2vtable,
                                                self.llinterp)
         self.get_type_id = layoutbuilder.get_type_id
-        layoutbuilder.initialize_gc_query_function(self.gc)
+        gcdata = layoutbuilder.initialize_gc_query_function(self.gc)
 
         constants = collect_constants(flowgraphs)
         for obj in constants:
@@ -38,8 +38,25 @@
 
         self.constantroots = layoutbuilder.addresses_of_static_ptrs
         self.constantrootsnongc = layoutbuilder.addresses_of_static_ptrs_in_nongc
+        self.prepare_custom_trace_funcs(gcdata)
         self._all_prebuilt_gc = layoutbuilder.all_prebuilt_gc
 
+    def prepare_custom_trace_funcs(self, gcdata):
+        custom_trace_funcs = self.llinterp.typer.custom_trace_funcs
+
+        def custom_trace(obj, typeid, callback, arg):
+            for TP, func in custom_trace_funcs:
+                if typeid == self.get_type_id(TP):
+                    func(self.gc, obj, callback, arg)
+                    return
+            else:
+                assert False
+        
+        for TP, func in custom_trace_funcs:
+            gcdata._has_got_custom_trace(self.get_type_id(TP))
+
+        self.gc.custom_trace_dispatcher = custom_trace
+
     # ____________________________________________________________
     #
     # Interface for the llinterp
diff --git a/rpython/memory/test/gc_test_base.py b/rpython/memory/test/gc_test_base.py
--- a/rpython/memory/test/gc_test_base.py
+++ b/rpython/memory/test/gc_test_base.py
@@ -6,7 +6,7 @@
 from rpython.rtyper.test.test_llinterp import get_interpreter
 from rpython.rtyper.lltypesystem import lltype
 from rpython.rtyper.lltypesystem.lloperation import llop
-from rpython.rlib.objectmodel import we_are_translated
+from rpython.rlib.objectmodel import we_are_translated, keepalive_until_here
 from rpython.rlib.objectmodel import compute_unique_id
 from rpython.rlib import rgc
 from rpython.rlib.rstring import StringBuilder
@@ -237,26 +237,20 @@
         assert 160 <= res <= 165
 
     def test_custom_trace(self):
-        from rpython.rtyper.annlowlevel import llhelper
         from rpython.rtyper.lltypesystem import llmemory
         from rpython.rtyper.lltypesystem.llarena import ArenaError
         #
         S = lltype.GcStruct('S', ('x', llmemory.Address),
-                                 ('y', llmemory.Address), rtti=True)
+                                 ('y', llmemory.Address))
         T = lltype.GcStruct('T', ('z', lltype.Signed))
         offset_of_x = llmemory.offsetof(S, 'x')
-        def customtrace(obj, prev):
-            if not prev:
-                return obj + offset_of_x
-            else:
-                return llmemory.NULL
-        CUSTOMTRACEFUNC = lltype.FuncType([llmemory.Address, llmemory.Address],
-                                          llmemory.Address)
-        customtraceptr = llhelper(lltype.Ptr(CUSTOMTRACEFUNC), customtrace)
-        lltype.attachRuntimeTypeInfo(S, customtraceptr=customtraceptr)
+        def customtrace(gc, obj, callback, arg):
+            gc._trace_callback(callback, arg, obj + offset_of_x)
+        lambda_customtrace = lambda: customtrace
         #
         for attrname in ['x', 'y']:
             def setup():
+                rgc.register_custom_trace_hook(S, lambda_customtrace)
                 s1 = lltype.malloc(S)
                 tx = lltype.malloc(T)
                 tx.z = 42
@@ -762,6 +756,23 @@
             assert rgc.get_gcflag_extra(a1) == False
             assert rgc.get_gcflag_extra(a2) == False
         self.interpret(fn, [])
+    
+    def test_register_custom_trace_hook(self):
+        S = lltype.GcStruct('S', ('x', lltype.Signed))
+        called = []
+
+        def trace_hook(gc, obj, callback, arg):
+            called.append("called")
+        lambda_trace_hook = lambda: trace_hook
+
+        def f():
+            rgc.register_custom_trace_hook(S, lambda_trace_hook)
+            s = lltype.malloc(S)
+            rgc.collect()
+            keepalive_until_here(s)
+
+        self.interpret(f, [])
+        assert called # not empty, can contain more than one item
 
     def test_pinning(self):
         def fn(n):
diff --git a/rpython/memory/test/test_transformed_gc.py b/rpython/memory/test/test_transformed_gc.py
--- a/rpython/memory/test/test_transformed_gc.py
+++ b/rpython/memory/test/test_transformed_gc.py
@@ -14,7 +14,7 @@
 from rpython.conftest import option
 from rpython.rlib.rstring import StringBuilder
 from rpython.rlib.rarithmetic import LONG_BIT
-import pdb
+
 
 WORD = LONG_BIT // 8
 
@@ -385,26 +385,20 @@
         assert 160 <= res <= 165
 
     def define_custom_trace(cls):
-        from rpython.rtyper.annlowlevel import llhelper
-        from rpython.rtyper.lltypesystem import llmemory
         #
-        S = lltype.GcStruct('S', ('x', llmemory.Address), rtti=True)
+        S = lltype.GcStruct('S', ('x', llmemory.Address))
         T = lltype.GcStruct('T', ('z', lltype.Signed))
         offset_of_x = llmemory.offsetof(S, 'x')
-        def customtrace(obj, prev):
-            if not prev:
-                return obj + offset_of_x
-            else:
-                return llmemory.NULL
-        CUSTOMTRACEFUNC = lltype.FuncType([llmemory.Address, llmemory.Address],
-                                          llmemory.Address)
-        customtraceptr = llhelper(lltype.Ptr(CUSTOMTRACEFUNC), customtrace)
-        lltype.attachRuntimeTypeInfo(S, customtraceptr=customtraceptr)
+        def customtrace(gc, obj, callback, arg):
+            gc._trace_callback(callback, arg, obj + offset_of_x)
+        lambda_customtrace = lambda: customtrace
+
         #
         def setup():
-            s1 = lltype.malloc(S)
+            rgc.register_custom_trace_hook(S, lambda_customtrace)
             tx = lltype.malloc(T)
             tx.z = 4243
+            s1 = lltype.malloc(S)
             s1.x = llmemory.cast_ptr_to_adr(tx)
             return s1
         def f():
diff --git a/rpython/rlib/_stacklet_asmgcc.py b/rpython/rlib/_stacklet_asmgcc.py
--- a/rpython/rlib/_stacklet_asmgcc.py
+++ b/rpython/rlib/_stacklet_asmgcc.py
@@ -1,4 +1,6 @@
 from rpython.rlib.debug import ll_assert
+from rpython.rlib import rgc
+from rpython.rlib.objectmodel import specialize
 from rpython.rtyper.lltypesystem import lltype, llmemory, rffi
 from rpython.rtyper.lltypesystem.lloperation import llop
 from rpython.rtyper.annlowlevel import llhelper, MixLevelHelperAnnotator
@@ -11,6 +13,10 @@
 _stackletrootwalker = None
 
 def get_stackletrootwalker():
+    # XXX this is too complicated now; we don't need a StackletRootWalker
+    # instance to store global state.  We could rewrite it all in one big
+    # function.  We don't care enough for now.
+
     # lazily called, to make the following imports lazy
     global _stackletrootwalker
     if _stackletrootwalker is not None:
@@ -25,8 +31,6 @@
     class StackletRootWalker(object):
         _alloc_flavor_ = "raw"
 
-        enumerating = False
-
         def setup(self, obj):
             # initialization: read the SUSPSTACK object
             p = llmemory.cast_adr_to_ptr(obj, lltype.Ptr(SUSPSTACK))
@@ -66,7 +70,8 @@
                 self.fill_initial_frame(self.curframe, anchor)
                 return True
 
-        def next(self, obj, prev):
+        @specialize.arg(3)
+        def customtrace(self, gc, obj, callback, arg):
             #
             # Pointers to the stack can be "translated" or not:
             #
@@ -79,29 +84,20 @@
             # Note that 'curframe' contains non-translated pointers, and
             # of course the stack itself is full of non-translated pointers.
             #
+            if not self.setup(obj):
+                return
+
             while True:
-                if not self.enumerating:
-                    if not prev:
-                        if not self.setup(obj):      # one-time initialization
-                            return llmemory.NULL
-                        prev = obj   # random value, but non-NULL
-                    callee = self.curframe
-                    retaddraddr = self.translateptr(callee.frame_address)
-                    retaddr = retaddraddr.address[0]
-                    ebp_in_caller = callee.regs_stored_at[INDEX_OF_EBP]
-                    ebp_in_caller = self.translateptr(ebp_in_caller)
-                    ebp_in_caller = ebp_in_caller.address[0]
-                    basewalker.locate_caller_based_on_retaddr(retaddr,
-                                                              ebp_in_caller)
-                    self.enumerating = True
-                else:
-                    callee = self.curframe
-                    ebp_in_caller = callee.regs_stored_at[INDEX_OF_EBP]
-                    ebp_in_caller = self.translateptr(ebp_in_caller)
-                    ebp_in_caller = ebp_in_caller.address[0]
-                #
-                # not really a loop, but kept this way for similarity
-                # with asmgcroot:
+                callee = self.curframe
+                retaddraddr = self.translateptr(callee.frame_address)
+                retaddr = retaddraddr.address[0]
+                ebp_in_caller = callee.regs_stored_at[INDEX_OF_EBP]
+                ebp_in_caller = self.translateptr(ebp_in_caller)
+                ebp_in_caller = ebp_in_caller.address[0]
+                basewalker.locate_caller_based_on_retaddr(retaddr,
+                                                          ebp_in_caller)
+
+                # see asmgcroot for similarity:
                 while True:
                     location = basewalker._shape_decompressor.next()
                     if location == 0:
@@ -109,9 +105,9 @@
                     addr = basewalker.getlocation(callee, ebp_in_caller,
                                                   location)
                     # yield the translated addr of the next GCREF in the stack
-                    return self.translateptr(addr)
-                #
-                self.enumerating = False
+                    addr = self.translateptr(addr)
+                    gc._trace_callback(callback, arg, addr)
+
                 caller = self.otherframe
                 reg = CALLEE_SAVED_REGS - 1
                 while reg >= 0:
@@ -129,7 +125,7 @@
                 if caller.frame_address == llmemory.NULL:
                     # completely done with this piece of stack
                     if not self.fetch_next_stack_piece():
-                        return llmemory.NULL
+                        return
                     continue
                 #
                 self.otherframe = callee
@@ -154,9 +150,10 @@
     lltype.attachRuntimeTypeInfo(SUSPSTACK, destrptr=destrptr)
 
 
-def customtrace(obj, prev):
+def customtrace(gc, obj, callback, arg):
     stackletrootwalker = get_stackletrootwalker()
-    return stackletrootwalker.next(obj, prev)
+    stackletrootwalker.customtrace(gc, obj, callback, arg)
+lambda_customtrace = lambda: customtrace
 
 def suspstack_destructor(suspstack):
     h = suspstack.handle
@@ -170,10 +167,6 @@
                             ('callback_pieces', llmemory.Address),
                             rtti=True)
 NULL_SUSPSTACK = lltype.nullptr(SUSPSTACK)
-CUSTOMTRACEFUNC = lltype.FuncType([llmemory.Address, llmemory.Address],
-                                  llmemory.Address)
-customtraceptr = llhelper(lltype.Ptr(CUSTOMTRACEFUNC), customtrace)
-lltype.attachRuntimeTypeInfo(SUSPSTACK, customtraceptr=customtraceptr)
 
 ASM_FRAMEDATA_HEAD_PTR = lltype.Ptr(lltype.ForwardReference())
 ASM_FRAMEDATA_HEAD_PTR.TO.become(lltype.Struct('ASM_FRAMEDATA_HEAD',
@@ -263,6 +256,7 @@
         self.runfn = callback
         self.arg = arg
         # make a fresh new clean SUSPSTACK
+        rgc.register_custom_trace_hook(SUSPSTACK, lambda_customtrace)
         newsuspstack = lltype.malloc(SUSPSTACK)
         newsuspstack.handle = _c.null_handle
         self.suspstack = newsuspstack
diff --git a/rpython/rlib/rgc.py b/rpython/rlib/rgc.py
--- a/rpython/rlib/rgc.py
+++ b/rpython/rlib/rgc.py
@@ -643,3 +643,22 @@
 
 def lltype_is_gc(TP):
     return getattr(getattr(TP, "TO", None), "_gckind", "?") == 'gc'
+
+def register_custom_trace_hook(TP, lambda_func):
+    """ This function does not do anything, but called from any annotated
+    place, will tell that "func" is used to trace GC roots inside any instance
+    of the type TP.  The func must be specified as "lambda: func" in this
+    call, for internal reasons.
+    """
+
+class RegisterGcTraceEntry(ExtRegistryEntry):
+    _about_ = register_custom_trace_hook
+
+    def compute_result_annotation(self, *args_s):
+        pass
+
+    def specialize_call(self, hop):
+        TP = hop.args_s[0].const
+        lambda_func = hop.args_s[1].const
+        hop.exception_cannot_occur()
+        hop.rtyper.custom_trace_funcs.append((TP, lambda_func()))
diff --git a/rpython/rlib/test/test_rgc.py b/rpython/rlib/test/test_rgc.py
--- a/rpython/rlib/test/test_rgc.py
+++ b/rpython/rlib/test/test_rgc.py
@@ -228,3 +228,17 @@
     x1 = X()
     n = rgc.get_rpy_memory_usage(rgc.cast_instance_to_gcref(x1))
     assert n >= 8 and n <= 64
+
+def test_register_custom_trace_hook():
+    TP = lltype.GcStruct('X')
+
+    def trace_func():
+        xxx # should not be annotated here
+    lambda_trace_func = lambda: trace_func
+    
+    def f():
+        rgc.register_custom_trace_hook(TP, lambda_trace_func)
+    
+    t, typer, graph = gengraph(f, [])
+
+    assert typer.custom_trace_funcs == [(TP, trace_func)]
diff --git a/rpython/rtyper/lltypesystem/lltype.py b/rpython/rtyper/lltypesystem/lltype.py
--- a/rpython/rtyper/lltypesystem/lltype.py
+++ b/rpython/rtyper/lltypesystem/lltype.py
@@ -383,8 +383,7 @@
                                                 about=self)._obj
         Struct._install_extras(self, **kwds)
 
-    def _attach_runtime_type_info_funcptr(self, funcptr, destrptr,
-                                          customtraceptr):
+    def _attach_runtime_type_info_funcptr(self, funcptr, destrptr):
         if self._runtime_type_info is None:
             raise TypeError("attachRuntimeTypeInfo: %r must have been built "
                             "with the rtti=True argument" % (self,))
@@ -408,18 +407,6 @@
                 raise TypeError("expected a destructor function "
                                 "implementation, got: %s" % destrptr)
             self._runtime_type_info.destructor_funcptr = destrptr
-        if customtraceptr is not None:
-            from rpython.rtyper.lltypesystem import llmemory
-            T = typeOf(customtraceptr)
-            if (not isinstance(T, Ptr) or
-                not isinstance(T.TO, FuncType) or
-                len(T.TO.ARGS) != 2 or
-                T.TO.RESULT != llmemory.Address or
-                T.TO.ARGS[0] != llmemory.Address or
-                T.TO.ARGS[1] != llmemory.Address):
-                raise TypeError("expected a custom trace function "
-                                "implementation, got: %s" % customtraceptr)
-            self._runtime_type_info.custom_trace_funcptr = customtraceptr
 
 class GcStruct(RttiStruct):
     _gckind = 'gc'
@@ -2288,12 +2275,10 @@
     return SomePtr(ll_ptrtype=PtrT.const)
 
 
-def attachRuntimeTypeInfo(GCSTRUCT, funcptr=None, destrptr=None,
-                          customtraceptr=None):
+def attachRuntimeTypeInfo(GCSTRUCT, funcptr=None, destrptr=None):
     if not isinstance(GCSTRUCT, RttiStruct):
         raise TypeError("expected a RttiStruct: %s" % GCSTRUCT)
-    GCSTRUCT._attach_runtime_type_info_funcptr(funcptr, destrptr,
-                                               customtraceptr)
+    GCSTRUCT._attach_runtime_type_info_funcptr(funcptr, destrptr)
     return _ptr(Ptr(RuntimeTypeInfo), GCSTRUCT._runtime_type_info)
 
 def getRuntimeTypeInfo(GCSTRUCT):
diff --git a/rpython/rtyper/lltypesystem/opimpl.py b/rpython/rtyper/lltypesystem/opimpl.py
--- a/rpython/rtyper/lltypesystem/opimpl.py
+++ b/rpython/rtyper/lltypesystem/opimpl.py
@@ -82,13 +82,11 @@
         else:
             def op_function(x, y):
                 if not isinstance(x, argtype):
-                    if not (isinstance(x, AddressAsInt) and argtype is int):
-                        raise TypeError("%r arg 1 must be %s, got %r instead"% (
-                            fullopname, typname, type(x).__name__))
+                    raise TypeError("%r arg 1 must be %s, got %r instead"% (
+                        fullopname, typname, type(x).__name__))
                 if not isinstance(y, argtype):
-                    if not (isinstance(y, AddressAsInt) and argtype is int):
-                        raise TypeError("%r arg 2 must be %s, got %r instead"% (
-                            fullopname, typname, type(y).__name__))
+                    raise TypeError("%r arg 2 must be %s, got %r instead"% (
+                        fullopname, typname, type(y).__name__))
                 return adjust_result(func(x, y))
 
     return func_with_new_name(op_function, 'op_' + fullopname)
@@ -104,6 +102,19 @@
             lltype.typeOf(adr),))
 
 
+def op_int_eq(x, y):
+    if not isinstance(x, (int, long)):
+        from rpython.rtyper.lltypesystem import llgroup
+        assert isinstance(x, llgroup.CombinedSymbolic), (
+            "'int_eq' arg 1 must be int-like, got %r instead" % (
+                type(x).__name__,))
+    if not isinstance(y, (int, long)):
+        from rpython.rtyper.lltypesystem import llgroup
+        assert isinstance(y, llgroup.CombinedSymbolic), (
+            "'int_eq' arg 2 must be int-like, got %r instead" % (
+                type(y).__name__,))
+    return x == y
+
 def op_ptr_eq(ptr1, ptr2):
     checkptr(ptr1)
     checkptr(ptr2)
diff --git a/rpython/rtyper/rtyper.py b/rpython/rtyper/rtyper.py
--- a/rpython/rtyper/rtyper.py
+++ b/rpython/rtyper/rtyper.py
@@ -60,6 +60,7 @@
         # make the primitive_to_repr constant mapping
         self.primitive_to_repr = {}
         self.exceptiondata = ExceptionData(self)
+        self.custom_trace_funcs = []
 
         try:
             self.seed = int(os.getenv('RTYPERSEED'))
@@ -645,7 +646,7 @@
             raise TyperError("runtime type info function %r returns %r, "
                              "excepted Ptr(RuntimeTypeInfo)" % (func, s))
         funcptr = self.getcallable(graph)
-        attachRuntimeTypeInfo(GCSTRUCT, funcptr, destrptr, None)
+        attachRuntimeTypeInfo(GCSTRUCT, funcptr, destrptr)
 
 # register operations from annotation model
 RPythonTyper._registeroperations(unaryop.UNARY_OPERATIONS, binaryop.BINARY_OPERATIONS)
diff --git a/rpython/translator/c/test/test_newgc.py b/rpython/translator/c/test/test_newgc.py
--- a/rpython/translator/c/test/test_newgc.py
+++ b/rpython/translator/c/test/test_newgc.py
@@ -443,19 +443,14 @@
     def define_custom_trace(cls):
         from rpython.rtyper.annlowlevel import llhelper
         #
-        S = lltype.GcStruct('S', ('x', llmemory.Address), rtti=True)
+        S = lltype.GcStruct('S', ('x', llmemory.Address))
         offset_of_x = llmemory.offsetof(S, 'x')
-        def customtrace(obj, prev):
-            if not prev:
-                return obj + offset_of_x
-            else:
-                return llmemory.NULL
-        CUSTOMTRACEFUNC = lltype.FuncType([llmemory.Address, llmemory.Address],
-                                          llmemory.Address)
-        customtraceptr = llhelper(lltype.Ptr(CUSTOMTRACEFUNC), customtrace)
-        lltype.attachRuntimeTypeInfo(S, customtraceptr=customtraceptr)
+        def customtrace(gc, obj, callback, arg):
+            gc._trace_callback(callback, arg, obj + offset_of_x)
+        lambda_customtrace = lambda: customtrace
         #
         def setup():
+            rgc.register_custom_trace_hook(S, lambda_customtrace)
             s = lltype.nullptr(S)
             for i in range(10000):
                 t = lltype.malloc(S)

From noreply at buildbot.pypy.org  Tue Nov 11 19:52:51 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Tue, 11 Nov 2014 19:52:51 +0100 (CET)
Subject: [pypy-commit] pypy default: Oups,
	an indirect conflict between two branches
Message-ID: <20141111185251.18D8E1D28DC@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r74458:4abe9a6d1c12
Date: 2014-11-11 19:52 +0100
http://bitbucket.org/pypy/pypy/changeset/4abe9a6d1c12/

Log:	Oups, an indirect conflict between two branches

diff --git a/rpython/memory/gctypelayout.py b/rpython/memory/gctypelayout.py
--- a/rpython/memory/gctypelayout.py
+++ b/rpython/memory/gctypelayout.py
@@ -78,12 +78,9 @@
         return (infobits & T_IS_GCARRAY_OF_GCPTR) != 0
 
     def q_cannot_pin(self, typeid):
-        infobits = self.get(typeid).infobits
-        ANY = (T_HAS_GCPTR |
-               T_IS_WEAKREF |
-               T_HAS_FINALIZER |
-               T_HAS_LIGHTWEIGHT_FINALIZER)
-        return (infobits & ANY) != 0
+        typeinfo = self.get(typeid)
+        ANY = (T_HAS_GCPTR | T_IS_WEAKREF)
+        return (typeinfo.infobits & ANY) != 0 or bool(typeinfo.finalizer)
 
     def q_finalizer(self, typeid):
         return self.get(typeid).finalizer

From noreply at buildbot.pypy.org  Tue Nov 11 20:54:22 2014
From: noreply at buildbot.pypy.org (rlamy)
Date: Tue, 11 Nov 2014 20:54:22 +0100 (CET)
Subject: [pypy-commit] pypy expressions: move V_Type to flowspace/
Message-ID: <20141111195422.826261C07BB@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy 
Branch: expressions
Changeset: r74459:df43eb19a931
Date: 2014-11-11 19:53 +0000
http://bitbucket.org/pypy/pypy/changeset/df43eb19a931/

Log:	move V_Type to flowspace/

diff --git a/rpython/annotator/expression.py b/rpython/flowspace/expression.py
rename from rpython/annotator/expression.py
rename to rpython/flowspace/expression.py
diff --git a/rpython/flowspace/generator.py b/rpython/flowspace/generator.py
--- a/rpython/flowspace/generator.py
+++ b/rpython/flowspace/generator.py
@@ -106,7 +106,7 @@
 def tweak_generator_body_graph(Entry, graph):
     # First, always run simplify_graph in order to reduce the number of
     # variables passed around
-    from rpython.annotator.expression import V_Type
+    from rpython.flowspace.expression import V_Type
     simplify_graph(graph)
     insert_empty_startblock(None, graph)
     _insert_reads(graph.startblock, Entry.varnames)
diff --git a/rpython/flowspace/operation.py b/rpython/flowspace/operation.py
--- a/rpython/flowspace/operation.py
+++ b/rpython/flowspace/operation.py
@@ -453,7 +453,7 @@
         if result is not None:
             return result
         ctx.merge_point()
-        from rpython.annotator.expression import V_Type
+        from rpython.flowspace.expression import V_Type
         v_instance, = self.args
         return V_Type(v_instance)
 
diff --git a/rpython/translator/simplify.py b/rpython/translator/simplify.py
--- a/rpython/translator/simplify.py
+++ b/rpython/translator/simplify.py
@@ -10,7 +10,7 @@
 from rpython.flowspace.model import (Variable, Constant,
                                      c_last_exception, checkgraph, mkentrymap)
 from rpython.flowspace.operation import OverflowingOperation, op
-from rpython.annotator.expression import V_Type
+from rpython.flowspace.expression import V_Type
 from rpython.rlib import rarithmetic
 from rpython.translator import unsimplify
 from rpython.translator.backendopt import ssa
diff --git a/rpython/translator/transform.py b/rpython/translator/transform.py
--- a/rpython/translator/transform.py
+++ b/rpython/translator/transform.py
@@ -8,7 +8,7 @@
 from rpython.flowspace.model import SpaceOperation
 from rpython.flowspace.model import Variable, Constant, Link
 from rpython.flowspace.model import c_last_exception, checkgraph
-from rpython.annotator.expression import V_Type
+from rpython.flowspace.expression import V_Type
 from rpython.annotator import model as annmodel
 from rpython.rtyper.lltypesystem import lltype
 

From noreply at buildbot.pypy.org  Tue Nov 11 21:16:42 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Tue, 11 Nov 2014 21:16:42 +0100 (CET)
Subject: [pypy-commit] pypy default: These uses of setarg() are a bit
 pointless, because emit_operation()
Message-ID: <20141111201642.C16981C03DC@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r74460:1727061f26b3
Date: 2014-11-11 20:06 +0100
http://bitbucket.org/pypy/pypy/changeset/1727061f26b3/

Log:	These uses of setarg() are a bit pointless, because emit_operation()
	will anyway force the argument.

diff --git a/rpython/jit/metainterp/optimizeopt/virtualize.py b/rpython/jit/metainterp/optimizeopt/virtualize.py
--- a/rpython/jit/metainterp/optimizeopt/virtualize.py
+++ b/rpython/jit/metainterp/optimizeopt/virtualize.py
@@ -806,12 +806,10 @@
                 try:
                     itemvalue = value.getitem_raw(offset, itemsize, descr)
                 except InvalidRawOperation:
-                    box = value.force_box(self)
-                    op.setarg(0, box)
-                    self.emit_operation(op)
+                    pass
                 else:
                     self.make_equal_to(op.result, itemvalue)
-                return
+                    return
         value.ensure_nonnull()
         self.emit_operation(op)
 
@@ -824,11 +822,9 @@
                 itemvalue = self.getvalue(op.getarg(2))
                 try:
                     value.setitem_raw(offset, itemsize, descr, itemvalue)
+                    return
                 except InvalidRawOperation:
-                    box = value.force_box(self)
-                    op.setarg(0, box)
-                    self.emit_operation(op)
-                return
+                    pass
         value.ensure_nonnull()
         self.emit_operation(op)
 
@@ -848,12 +844,10 @@
                 try:
                     itemvalue = value.getitem_raw(offset, itemsize, descr)
                 except InvalidRawOperation:
-                    box = value.force_box(self)
-                    op.setarg(0, box)
-                    self.emit_operation(op)
+                    pass
                 else:
                     self.make_equal_to(op.result, itemvalue)
-                return
+                    return
         value.ensure_nonnull()
         self.emit_operation(op)
 
@@ -866,11 +860,9 @@
                 itemvalue = self.getvalue(op.getarg(2))
                 try:
                     value.setitem_raw(offset, itemsize, descr, itemvalue)
+                    return
                 except InvalidRawOperation:
-                    box = value.force_box(self)
-                    op.setarg(0, box)
-                    self.emit_operation(op)
-                return
+                    pass
         value.ensure_nonnull()
         self.emit_operation(op)
 

From noreply at buildbot.pypy.org  Tue Nov 11 21:16:44 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Tue, 11 Nov 2014 21:16:44 +0100 (CET)
Subject: [pypy-commit] pypy default: Issue #1924: test and fix.
Message-ID: <20141111201644.1A9C71C03DC@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r74461:04c100cbae5b
Date: 2014-11-11 21:16 +0100
http://bitbucket.org/pypy/pypy/changeset/04c100cbae5b/

Log:	Issue #1924: test and fix.

diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
@@ -2024,6 +2024,27 @@
         """
         self.optimize_loop(ops, expected)
 
+    def test_virtual_raw_buffer_forced_but_slice_not_forced(self):
+        ops = """
+        [f1]
+        i0 = call('malloc', 16, descr=raw_malloc_descr)
+        guard_no_exception() []
+        i1 = int_add(i0, 8)
+        escape(i0)
+        setarrayitem_raw(i1, 1, f1, descr=rawarraydescr_float)
+        jump(f1)
+        """
+        expected = """
+        [f1]
+        i0 = call('malloc', 16, descr=raw_malloc_descr)
+        #guard_no_exception() []  # XXX should appear
+        escape(i0)
+        i1 = int_add(i0, 8)
+        setarrayitem_raw(i1, 1, f1, descr=rawarraydescr_float)
+        jump(f1)
+        """
+        self.optimize_loop(ops, expected)
+
     def test_duplicate_getfield_1(self):
         ops = """
         [p1, p2]
diff --git a/rpython/jit/metainterp/optimizeopt/virtualize.py b/rpython/jit/metainterp/optimizeopt/virtualize.py
--- a/rpython/jit/metainterp/optimizeopt/virtualize.py
+++ b/rpython/jit/metainterp/optimizeopt/virtualize.py
@@ -443,9 +443,17 @@
         self.buffer.values[i] = newval
 
     def getitem_raw(self, offset, length, descr):
+        if not self.is_virtual():
+            raise InvalidRawOperation
+            # see 'test_virtual_raw_buffer_forced_but_slice_not_forced'
+            # for the test above: it's not enough to check is_virtual()
+            # on the original object, because it might be a VRawSliceValue
+            # instead.  If it is a virtual one, then we'll reach here anway.
         return self.buffer.read_value(offset, length, descr)
 
     def setitem_raw(self, offset, length, descr, value):
+        if not self.is_virtual():
+            raise InvalidRawOperation
         self.buffer.write_value(offset, length, descr, value)
 
     def _really_force(self, optforce):

From noreply at buildbot.pypy.org  Tue Nov 11 21:34:01 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Tue, 11 Nov 2014 21:34:01 +0100 (CET)
Subject: [pypy-commit] pypy default: That was intended with a setarrayitem
 index of 0, although it doesn't
Message-ID: <20141111203401.64F991C03DC@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r74462:22e929dc69e7
Date: 2014-11-11 21:33 +0100
http://bitbucket.org/pypy/pypy/changeset/22e929dc69e7/

Log:	That was intended with a setarrayitem index of 0, although it
	doesn't change much.

diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
@@ -2031,7 +2031,7 @@
         guard_no_exception() []
         i1 = int_add(i0, 8)
         escape(i0)
-        setarrayitem_raw(i1, 1, f1, descr=rawarraydescr_float)
+        setarrayitem_raw(i1, 0, f1, descr=rawarraydescr_float)
         jump(f1)
         """
         expected = """
@@ -2040,7 +2040,7 @@
         #guard_no_exception() []  # XXX should appear
         escape(i0)
         i1 = int_add(i0, 8)
-        setarrayitem_raw(i1, 1, f1, descr=rawarraydescr_float)
+        setarrayitem_raw(i1, 0, f1, descr=rawarraydescr_float)
         jump(f1)
         """
         self.optimize_loop(ops, expected)

From noreply at buildbot.pypy.org  Tue Nov 11 22:16:51 2014
From: noreply at buildbot.pypy.org (mattip)
Date: Tue, 11 Nov 2014 22:16:51 +0100 (CET)
Subject: [pypy-commit] pypy default: fix: non-exported globals
Message-ID: <20141111211651.275B21C03DC@cobra.cs.uni-duesseldorf.de>

Author: Matti Picus 
Branch: 
Changeset: r74463:598730a422d2
Date: 2014-11-11 20:05 -0600
http://bitbucket.org/pypy/pypy/changeset/598730a422d2/

Log:	fix: non-exported globals

diff --git a/pypy/module/test_lib_pypy/ctypes_tests/_ctypes_test.c b/pypy/module/test_lib_pypy/ctypes_tests/_ctypes_test.c
--- a/pypy/module/test_lib_pypy/ctypes_tests/_ctypes_test.c
+++ b/pypy/module/test_lib_pypy/ctypes_tests/_ctypes_test.c
@@ -316,8 +316,8 @@
 {
 }
 
-EXPORT(LONG_LONG) last_tf_arg_s;
-EXPORT(unsigned LONG_LONG) last_tf_arg_u;
+LONG_LONG last_tf_arg_s;
+unsigned LONG_LONG last_tf_arg_u;
 
 struct BITS {
 	int A: 1, B:2, C:3, D:4, E: 5, F: 6, G: 7, H: 8, I: 9;

From noreply at buildbot.pypy.org  Tue Nov 11 22:16:52 2014
From: noreply at buildbot.pypy.org (mattip)
Date: Tue, 11 Nov 2014 22:16:52 +0100 (CET)
Subject: [pypy-commit] pypy default: stdcall unavoidably mangles names in a
	dll
Message-ID: <20141111211652.6AF4B1C03DC@cobra.cs.uni-duesseldorf.de>

Author: Matti Picus 
Branch: 
Changeset: r74464:bda954894395
Date: 2014-11-11 23:03 -0600
http://bitbucket.org/pypy/pypy/changeset/bda954894395/

Log:	stdcall unavoidably mangles names in a dll

diff --git a/rpython/rlib/test/test_libffi.py b/rpython/rlib/test/test_libffi.py
--- a/rpython/rlib/test/test_libffi.py
+++ b/rpython/rlib/test/test_libffi.py
@@ -576,7 +576,9 @@
             }
             """
             libfoo = self.get_libfoo()
-            func = (libfoo, 'std_diff_xy', [types.sint, types.signed], types.sint)
+            # __stdcall without a DEF file decorates the name with the number of bytes
+            # that the callee will remove from the call stack
+            func = (libfoo, '_std_diff_xy at 8', [types.sint, types.signed], types.sint)
             try:
                 self.call(func, [50, 8], lltype.Signed)
             except ValueError, e:
@@ -613,7 +615,9 @@
             """
             from rpython.rlib.libffi import WinDLL
             dll = WinDLL(self.libfoo_name)
-            f_by_name = dll.getpointer('BBB_second_ordinal_function' ,[],
+            # __stdcall without a DEF file decorates the name with the number of bytes
+            # that the callee will remove from the call stack
+            f_by_name = dll.getpointer('_BBB_second_ordinal_function at 0' ,[],
                                           types.uint)
             f_by_ordinal = dll.getpointer_by_ordinal(2 ,[], types.uint)
             print dir(f_by_name)

From noreply at buildbot.pypy.org  Tue Nov 11 22:16:53 2014
From: noreply at buildbot.pypy.org (mattip)
Date: Tue, 11 Nov 2014 22:16:53 +0100 (CET)
Subject: [pypy-commit] pypy default: as per precommondefs.h,
 all declarations need an attribute
Message-ID: <20141111211653.A7B5F1C03DC@cobra.cs.uni-duesseldorf.de>

Author: Matti Picus 
Branch: 
Changeset: r74465:3532a779652c
Date: 2014-11-11 23:11 -0600
http://bitbucket.org/pypy/pypy/changeset/3532a779652c/

Log:	as per precommondefs.h, all declarations need an attribute

diff --git a/pypy/module/test_lib_pypy/ctypes_tests/_ctypes_test.c b/pypy/module/test_lib_pypy/ctypes_tests/_ctypes_test.c
--- a/pypy/module/test_lib_pypy/ctypes_tests/_ctypes_test.c
+++ b/pypy/module/test_lib_pypy/ctypes_tests/_ctypes_test.c
@@ -316,8 +316,8 @@
 {
 }
 
-LONG_LONG last_tf_arg_s;
-unsigned LONG_LONG last_tf_arg_u;
+static LONG_LONG last_tf_arg_s;
+static unsigned LONG_LONG last_tf_arg_u;
 
 struct BITS {
 	int A: 1, B:2, C:3, D:4, E: 5, F: 6, G: 7, H: 8, I: 9;

From noreply at buildbot.pypy.org  Tue Nov 11 22:45:03 2014
From: noreply at buildbot.pypy.org (amauryfa)
Date: Tue, 11 Nov 2014 22:45:03 +0100 (CET)
Subject: [pypy-commit] pypy default: Allow "python -m rpython" to run
	RPython translation.
Message-ID: <20141111214503.8009B1C0548@cobra.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc 
Branch: 
Changeset: r74466:0e89a463e021
Date: 2014-11-11 22:44 +0100
http://bitbucket.org/pypy/pypy/changeset/0e89a463e021/

Log:	Allow "python -m rpython" to run RPython translation.

	It's slightly shorter than rpython/bin/rpython, specially when the
	interpreter is specified.

diff --git a/rpython/__main__.py b/rpython/__main__.py
new file mode 100644
--- /dev/null
+++ b/rpython/__main__.py
@@ -0,0 +1,16 @@
+"""RPython translation usage:
+
+rpython  target 
+
+run with --help for more information
+"""
+
+import sys
+
+# no implicit targets
+if len(sys.argv) == 1:
+    print __doc__
+    sys.exit(1)
+
+from rpython.translator.goal.translate import main
+main()

From noreply at buildbot.pypy.org  Wed Nov 12 00:21:36 2014
From: noreply at buildbot.pypy.org (amauryfa)
Date: Wed, 12 Nov 2014 00:21:36 +0100 (CET)
Subject: [pypy-commit] pypy kill-rctime: Kill "rctime" module,
	use time directly.
Message-ID: <20141111232136.40E001C07BB@cobra.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc 
Branch: kill-rctime
Changeset: r74467:9a47327d5451
Date: 2014-11-11 23:55 +0100
http://bitbucket.org/pypy/pypy/changeset/9a47327d5451/

Log:	Kill "rctime" module, use time directly.


From noreply at buildbot.pypy.org  Wed Nov 12 00:21:37 2014
From: noreply at buildbot.pypy.org (amauryfa)
Date: Wed, 12 Nov 2014 00:21:37 +0100 (CET)
Subject: [pypy-commit] pypy kill-rctime: Rename 'rctime' to plain 'time'.
Message-ID: <20141111232137.C322D1C07BB@cobra.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc 
Branch: kill-rctime
Changeset: r74468:d53aec985fd4
Date: 2014-11-12 00:20 +0100
http://bitbucket.org/pypy/pypy/changeset/d53aec985fd4/

Log:	Rename 'rctime' to plain 'time'. Try to remove all references to
	rctime.

diff too long, truncating to 2000 out of 2637 lines

diff --git a/lib-python/conftest.py b/lib-python/conftest.py
--- a/lib-python/conftest.py
+++ b/lib-python/conftest.py
@@ -59,7 +59,7 @@
     def __init__(self, basename, core=False, compiler=None, usemodules='',
                  skip=None):
         self.basename = basename
-        self._usemodules = usemodules.split() + ['signal', 'rctime', 'itertools', '_socket']
+        self._usemodules = usemodules.split() + ['signal', 'time', 'itertools', '_socket']
         self._compiler = compiler
         self.core = core
         self.skip = skip
diff --git a/pypy/config/pypyoption.py b/pypy/config/pypyoption.py
--- a/pypy/config/pypyoption.py
+++ b/pypy/config/pypyoption.py
@@ -29,7 +29,7 @@
 # --allworkingmodules
 working_modules = default_modules.copy()
 working_modules.update([
-    "_socket", "unicodedata", "mmap", "fcntl", "_locale", "pwd", "rctime" ,
+    "_socket", "unicodedata", "mmap", "fcntl", "_locale", "pwd", "time" ,
     "select", "zipimport", "_lsprof", "crypt", "signal", "_rawffi", "termios",
     "zlib", "bz2", "struct", "_hashlib", "_md5", "_sha", "_minimal_curses",
     "cStringIO", "thread", "itertools", "pyexpat", "_ssl", "cpyext", "array",
@@ -40,7 +40,7 @@
 
 translation_modules = default_modules.copy()
 translation_modules.update([
-    "fcntl", "rctime", "select", "signal", "_rawffi", "zlib", "struct", "_md5",
+    "fcntl", "time", "select", "signal", "_rawffi", "zlib", "struct", "_md5",
     "cStringIO", "array", "binascii",
     # the following are needed for pyrepl (and hence for the
     # interactive prompt/pdb)
@@ -65,18 +65,18 @@
 
 if sys.platform == "sunos5":
     working_modules.remove('mmap')   # depend on ctypes, can't get at c-level 'errono'
-    working_modules.remove('rctime') # depend on ctypes, missing tm_zone/tm_gmtoff
+    working_modules.remove('time')   # depend on ctypes, missing tm_zone/tm_gmtoff
     working_modules.remove('signal') # depend on ctypes, can't get at c-level 'errono'
     working_modules.remove('fcntl')  # LOCK_NB not defined
     working_modules.remove("_minimal_curses")
     working_modules.remove("termios")
-    working_modules.remove("_multiprocessing")   # depends on rctime
+    working_modules.remove("_multiprocessing")   # depends on time
     if "cppyy" in working_modules:
         working_modules.remove("cppyy")  # depends on ctypes
 
 
 module_dependencies = {
-    '_multiprocessing': [('objspace.usemodules.rctime', True),
+    '_multiprocessing': [('objspace.usemodules.time', True),
                          ('objspace.usemodules.thread', True)],
     'cpyext': [('objspace.usemodules.array', True)],
     'cppyy': [('objspace.usemodules.cpyext', True)],
diff --git a/pypy/doc/config/objspace.usemodules.rctime.txt b/pypy/doc/config/objspace.usemodules.rctime.txt
deleted file mode 100644
--- a/pypy/doc/config/objspace.usemodules.rctime.txt
+++ /dev/null
@@ -1,7 +0,0 @@
-Use the 'rctime' module. 
-
-'rctime' is our `rffi`_ based implementation of the builtin 'time' module.
-It supersedes the less complete :config:`objspace.usemodules.time`,
-at least for C-like targets (the C and LLVM backends).
-
-.. _`rffi`: ../rffi.html
diff --git a/pypy/doc/config/objspace.usemodules.time.txt b/pypy/doc/config/objspace.usemodules.time.txt
--- a/pypy/doc/config/objspace.usemodules.time.txt
+++ b/pypy/doc/config/objspace.usemodules.time.txt
@@ -1,5 +1,1 @@
 Use the 'time' module. 
-
-Obsolete; use :config:`objspace.usemodules.rctime` for our up-to-date version
-of the application-level 'time' module, at least for C-like targets (the C
-and LLVM backends).
diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -522,11 +522,6 @@
                 if name not in modules:
                     modules.append(name)
 
-        # a bit of custom logic: rctime take precedence over time
-        # XXX this could probably be done as a "requires" in the config
-        if 'rctime' in modules and 'time' in modules:
-            modules.remove('time')
-
         self._builtinmodule_list = modules
         return self._builtinmodule_list
 
diff --git a/pypy/module/_file/test/test_file_extra.py b/pypy/module/_file/test/test_file_extra.py
--- a/pypy/module/_file/test/test_file_extra.py
+++ b/pypy/module/_file/test/test_file_extra.py
@@ -221,7 +221,7 @@
     expected_filename = str(udir.join('sample'))
     expected_mode = 'rb'
     extra_args = ()
-    spaceconfig = {'usemodules': ['binascii', 'rctime', 'struct']}
+    spaceconfig = {'usemodules': ['binascii', 'time', 'struct']}
 
     def setup_method(self, method):
         space = self.space
@@ -281,7 +281,7 @@
     expected_filename = ''
     expected_mode = 'rb'
     extra_args = ()
-    spaceconfig = {'usemodules': ['binascii', 'rctime', 'struct']}
+    spaceconfig = {'usemodules': ['binascii', 'time', 'struct']}
 
     def setup_method(self, method):
         space = self.space
@@ -359,7 +359,7 @@
 #  A few extra tests
 
 class AppTestAFewExtra:
-    spaceconfig = {'usemodules': ['_socket', 'array', 'binascii', 'rctime',
+    spaceconfig = {'usemodules': ['_socket', 'array', 'binascii', 'time',
                                   'struct']}
 
     def setup_method(self, method):
diff --git a/pypy/module/_lsprof/test/test_cprofile.py b/pypy/module/_lsprof/test/test_cprofile.py
--- a/pypy/module/_lsprof/test/test_cprofile.py
+++ b/pypy/module/_lsprof/test/test_cprofile.py
@@ -1,6 +1,6 @@
 class AppTestCProfile(object):
     spaceconfig = {
-        "usemodules": ['_lsprof', 'rctime'],
+        "usemodules": ['_lsprof', 'time'],
     }
 
     def setup_class(cls):
diff --git a/pypy/module/_md5/test/test_md5.py b/pypy/module/_md5/test/test_md5.py
--- a/pypy/module/_md5/test/test_md5.py
+++ b/pypy/module/_md5/test/test_md5.py
@@ -5,7 +5,7 @@
 
 class AppTestMD5(object):
     spaceconfig = {
-        'usemodules': ['_md5', 'binascii', 'rctime', 'struct'],
+        'usemodules': ['_md5', 'binascii', 'time', 'struct'],
     }
 
     def setup_class(cls):
diff --git a/pypy/module/_multiprocessing/interp_semaphore.py b/pypy/module/_multiprocessing/interp_semaphore.py
--- a/pypy/module/_multiprocessing/interp_semaphore.py
+++ b/pypy/module/_multiprocessing/interp_semaphore.py
@@ -254,7 +254,7 @@
         start = _GetTickCount()
 
         while True:
-            from pypy.module.rctime.interp_time import State
+            from pypy.module.time.interp_time import State
             interrupt_event = space.fromcache(State).get_interrupt_event()
             handles = [self.handle, interrupt_event]
 
diff --git a/pypy/module/_random/test/test_random.py b/pypy/module/_random/test/test_random.py
--- a/pypy/module/_random/test/test_random.py
+++ b/pypy/module/_random/test/test_random.py
@@ -1,6 +1,6 @@
 class AppTestRandom:
     spaceconfig = {
-        "usemodules": ['_random', 'rctime'],
+        "usemodules": ['_random', 'time'],
     }
 
     def test_dict(self):
diff --git a/pypy/module/_sha/test/test_sha.py b/pypy/module/_sha/test/test_sha.py
--- a/pypy/module/_sha/test/test_sha.py
+++ b/pypy/module/_sha/test/test_sha.py
@@ -5,7 +5,7 @@
 
 class AppTestSHA(object):
     spaceconfig = {
-        'usemodules': ['_sha', 'binascii', 'rctime', 'struct'],
+        'usemodules': ['_sha', 'binascii', 'time', 'struct'],
     }
 
     def setup_class(cls):
diff --git a/pypy/module/bz2/test/test_bz2_file.py b/pypy/module/bz2/test/test_bz2_file.py
--- a/pypy/module/bz2/test/test_bz2_file.py
+++ b/pypy/module/bz2/test/test_bz2_file.py
@@ -53,7 +53,7 @@
 
 class AppTestBZ2File(CheckAllocation):
     spaceconfig = {
-        'usemodules': ['bz2', 'binascii', 'rctime', 'struct']
+        'usemodules': ['bz2', 'binascii', 'time', 'struct']
     }
 
     def setup_class(cls):
diff --git a/pypy/module/cpyext/test/conftest.py b/pypy/module/cpyext/test/conftest.py
--- a/pypy/module/cpyext/test/conftest.py
+++ b/pypy/module/cpyext/test/conftest.py
@@ -7,7 +7,7 @@
     # it's necessary to run "import time" at least once before any
     # other cpyext test, otherwise the same statement will fail in
     # test_datetime.py.
-    space = gettestobjspace(usemodules=['rctime'])
+    space = gettestobjspace(usemodules=['time'])
     space.getbuiltinmodule("time")
 
 def pytest_ignore_collect(path, config):
diff --git a/pypy/module/cpyext/test/test_cpyext.py b/pypy/module/cpyext/test/test_cpyext.py
--- a/pypy/module/cpyext/test/test_cpyext.py
+++ b/pypy/module/cpyext/test/test_cpyext.py
@@ -102,7 +102,7 @@
 class LeakCheckingTest(object):
     """Base class for all cpyext tests."""
     spaceconfig = dict(usemodules=['cpyext', 'thread', '_rawffi', 'array',
-                                   'itertools', 'rctime', 'binascii', 'micronumpy'])
+                                   'itertools', 'time', 'binascii', 'micronumpy'])
     spaceconfig['std.withmethodcache'] = True
 
     enable_leak_checking = True
diff --git a/pypy/module/fcntl/test/test_fcntl.py b/pypy/module/fcntl/test/test_fcntl.py
--- a/pypy/module/fcntl/test/test_fcntl.py
+++ b/pypy/module/fcntl/test/test_fcntl.py
@@ -12,7 +12,7 @@
 
 class AppTestFcntl:
     spaceconfig = dict(usemodules=('fcntl', 'array', 'struct', 'termios',
-                                   'select', 'rctime'))
+                                   'select', 'time'))
 
     def setup_class(cls):
         tmpprefix = str(udir.ensure('test_fcntl', dir=1).join('tmp_'))
diff --git a/pypy/module/imp/test/test_app.py b/pypy/module/imp/test/test_app.py
--- a/pypy/module/imp/test/test_app.py
+++ b/pypy/module/imp/test/test_app.py
@@ -4,7 +4,7 @@
 
 class AppTestImpModule:
     spaceconfig = {
-        'usemodules': ['binascii', 'imp', 'itertools', 'rctime', 'struct'],
+        'usemodules': ['binascii', 'imp', 'itertools', 'time', 'struct'],
     }
 
     def setup_class(cls):
diff --git a/pypy/module/imp/test/test_import.py b/pypy/module/imp/test/test_import.py
--- a/pypy/module/imp/test/test_import.py
+++ b/pypy/module/imp/test/test_import.py
@@ -149,7 +149,7 @@
 
 class AppTestImport:
     spaceconfig = {
-        "usemodules": ['_md5', 'rctime'],
+        "usemodules": ['_md5', 'time'],
     }
 
     def setup_class(cls):
@@ -1044,7 +1044,7 @@
 
 class AppTestImportHooks(object):
     spaceconfig = {
-        "usemodules": ['struct', 'itertools', 'rctime'],
+        "usemodules": ['struct', 'itertools', 'time'],
     }
 
     def setup_class(cls):
@@ -1304,7 +1304,7 @@
 
 
 class AppTestMultithreadedImp(object):
-    spaceconfig = dict(usemodules=['thread', 'rctime'])
+    spaceconfig = dict(usemodules=['thread', 'time'])
 
     def setup_class(cls):
         #if not conftest.option.runappdirect:
diff --git a/pypy/module/math/test/test_math.py b/pypy/module/math/test/test_math.py
--- a/pypy/module/math/test/test_math.py
+++ b/pypy/module/math/test/test_math.py
@@ -7,7 +7,7 @@
 
 class AppTestMath:
     spaceconfig = {
-        "usemodules": ['math', 'struct', 'itertools', 'rctime', 'binascii'],
+        "usemodules": ['math', 'struct', 'itertools', 'time', 'binascii'],
     }
 
     def setup_class(cls):
diff --git a/pypy/module/posix/test/test_posix2.py b/pypy/module/posix/test/test_posix2.py
--- a/pypy/module/posix/test/test_posix2.py
+++ b/pypy/module/posix/test/test_posix2.py
@@ -14,7 +14,7 @@
 import signal
 
 def setup_module(mod):
-    usemodules = ['binascii', 'posix', 'struct', 'rctime']
+    usemodules = ['binascii', 'posix', 'struct', 'time']
     if os.name != 'nt':
         usemodules += ['fcntl']
     else:
diff --git a/pypy/module/posix/test/test_posix_libfile.py b/pypy/module/posix/test/test_posix_libfile.py
--- a/pypy/module/posix/test/test_posix_libfile.py
+++ b/pypy/module/posix/test/test_posix_libfile.py
@@ -10,7 +10,7 @@
 
 class AppTestPosix:
     spaceconfig = {
-        "usemodules": ['posix', 'rctime'],
+        "usemodules": ['posix', 'time'],
     }
 
     def setup_class(cls):
diff --git a/pypy/module/pyexpat/test/test_parser.py b/pypy/module/pyexpat/test/test_parser.py
--- a/pypy/module/pyexpat/test/test_parser.py
+++ b/pypy/module/pyexpat/test/test_parser.py
@@ -177,7 +177,7 @@
 
 class AppTestPyexpat2:
     spaceconfig = dict(usemodules=['pyexpat', 'itertools', '_socket',
-                                   'rctime', 'struct', 'binascii'])
+                                   'time', 'struct', 'binascii'])
 
     def test_django_bug(self):
         xml_str = ''
diff --git a/pypy/module/pypyjit/test/test_policy.py b/pypy/module/pypyjit/test/test_policy.py
--- a/pypy/module/pypyjit/test/test_policy.py
+++ b/pypy/module/pypyjit/test/test_policy.py
@@ -29,7 +29,7 @@
     assert pypypolicy.look_inside_function(get_ident)
 
 def test_time():
-    from pypy.module.rctime.interp_time import time
+    from pypy.module.time.interp_time import time
     assert pypypolicy.look_inside_function(time)
 
 def test_io():
diff --git a/pypy/module/rctime/__init__.py b/pypy/module/rctime/__init__.py
deleted file mode 100644
--- a/pypy/module/rctime/__init__.py
+++ /dev/null
@@ -1,42 +0,0 @@
-
-from pypy.interpreter.mixedmodule import MixedModule
-import os
-
-_WIN = os.name == "nt"
-
-class Module(MixedModule):
-    applevel_name = 'time'
-
-    interpleveldefs = {
-        'time': 'interp_time.time',
-        'clock': 'interp_time.clock',
-        'ctime': 'interp_time.ctime',
-        'asctime': 'interp_time.asctime',
-        'gmtime': 'interp_time.gmtime',
-        'localtime': 'interp_time.localtime',
-        'mktime': 'interp_time.mktime',
-        'strftime': 'interp_time.strftime',
-        'sleep' : 'interp_time.sleep',
-    }
-
-    if os.name == "posix":
-        interpleveldefs['tzset'] = 'interp_time.tzset'
-
-    appleveldefs = {
-        'struct_time': 'app_time.struct_time',
-        '__doc__': 'app_time.__doc__',
-        'strptime': 'app_time.strptime',
-    }
-
-    def startup(self, space):
-        if _WIN:
-            from pypy.module.rctime.interp_time import State
-            space.fromcache(State).startup(space)
-
-        # this machinery is needed to expose constants
-        # that have to be initialized one time only
-        from pypy.module.rctime import interp_time
-
-        interp_time._init_timezone(space)
-        interp_time._init_accept2dyear(space)
-
diff --git a/pypy/module/rctime/interp_time.py b/pypy/module/rctime/interp_time.py
deleted file mode 100644
--- a/pypy/module/rctime/interp_time.py
+++ /dev/null
@@ -1,670 +0,0 @@
-from rpython.rtyper.tool import rffi_platform as platform
-from rpython.rtyper.lltypesystem import rffi
-from pypy.interpreter.error import OperationError, oefmt
-from pypy.interpreter.gateway import unwrap_spec
-from rpython.rtyper.lltypesystem import lltype
-from rpython.rlib.rarithmetic import intmask
-from rpython.rlib import rposix
-from rpython.translator.tool.cbuild import ExternalCompilationInfo
-import os
-import sys
-import time as pytime
-
-_POSIX = os.name == "posix"
-_WIN = os.name == "nt"
-_CYGWIN = sys.platform == "cygwin"
-
-_time_zones = []
-if _CYGWIN:
-    _time_zones = ["GMT-12", "GMT-11", "GMT-10", "GMT-9", "GMT-8", "GMT-7",
-                   "GMT-6", "GMT-5", "GMT-4", "GMT-3", "GMT-2", "GMT-1",
-                   "GMT",  "GMT+1", "GMT+2", "GMT+3", "GMT+4", "GMT+5",
-                   "GMT+6",  "GMT+7", "GMT+8", "GMT+9", "GMT+10", "GMT+11",
-                   "GMT+12",  "GMT+13", "GMT+14"]
-
-if _WIN:
-    # Interruptible sleeps on Windows:
-    # We install a specific Console Ctrl Handler which sets an 'event'.
-    # time.sleep() will actually call WaitForSingleObject with the desired
-    # timeout.  On Ctrl-C, the signal handler is called, the event is set,
-    # and the wait function exits.
-    from rpython.rlib import rwin32
-    from pypy.interpreter.error import wrap_windowserror, wrap_oserror
-    from rpython.rlib import rthread as thread
-
-    eci = ExternalCompilationInfo(
-        includes = ['windows.h'],
-        post_include_bits = [
-            "RPY_EXTERN\n"
-            "BOOL pypy_timemodule_setCtrlHandler(HANDLE event);"],
-        separate_module_sources=['''
-            static HANDLE interrupt_event;
-
-            static BOOL WINAPI CtrlHandlerRoutine(
-              DWORD dwCtrlType)
-            {
-                SetEvent(interrupt_event);
-                /* allow other default handlers to be called.
-                 * Default Python handler will setup the
-                 * KeyboardInterrupt exception.
-                 */
-                return 0;
-            }
-
-            BOOL pypy_timemodule_setCtrlHandler(HANDLE event)
-            {
-                interrupt_event = event;
-                return SetConsoleCtrlHandler(CtrlHandlerRoutine, TRUE);
-            }
-
-        '''],
-        )
-    _setCtrlHandlerRoutine = rffi.llexternal(
-        'pypy_timemodule_setCtrlHandler',
-        [rwin32.HANDLE], rwin32.BOOL,
-        compilation_info=eci)
-
-    class GlobalState:
-        def __init__(self):
-            self.init()
-
-        def init(self):
-            self.interrupt_event = rwin32.NULL_HANDLE
-
-        def startup(self, space):
-            # Initialize the event handle used to signal Ctrl-C
-            try:
-                globalState.interrupt_event = rwin32.CreateEvent(
-                    rffi.NULL, True, False, rffi.NULL)
-            except WindowsError, e:
-                raise wrap_windowserror(space, e)
-            if not _setCtrlHandlerRoutine(globalState.interrupt_event):
-                raise wrap_windowserror(
-                    space, rwin32.lastWindowsError("SetConsoleCtrlHandler"))
-
-    globalState = GlobalState()
-
-    class State:
-        def __init__(self, space):
-            self.main_thread = 0
-
-        def _cleanup_(self):
-            self.main_thread = 0
-            globalState.init()
-
-        def startup(self, space):
-            self.main_thread = thread.get_ident()
-            globalState.startup(space)
-
-        def get_interrupt_event(self):
-            return globalState.interrupt_event
-
-
-_includes = ["time.h"]
-if _POSIX:
-    _includes.append('sys/time.h')
-
-class CConfig:
-    _compilation_info_ = ExternalCompilationInfo(
-        includes = _includes
-    )
-    CLOCKS_PER_SEC = platform.ConstantInteger("CLOCKS_PER_SEC")
-    clock_t = platform.SimpleType("clock_t", rffi.ULONG)
-    has_gettimeofday = platform.Has('gettimeofday')
-
-if _POSIX:
-    calling_conv = 'c'
-    CConfig.timeval = platform.Struct("struct timeval",
-                                      [("tv_sec", rffi.INT),
-                                       ("tv_usec", rffi.INT)])
-    if _CYGWIN:
-        CConfig.tm = platform.Struct("struct tm", [("tm_sec", rffi.INT),
-            ("tm_min", rffi.INT), ("tm_hour", rffi.INT), ("tm_mday", rffi.INT),
-            ("tm_mon", rffi.INT), ("tm_year", rffi.INT), ("tm_wday", rffi.INT),
-            ("tm_yday", rffi.INT), ("tm_isdst", rffi.INT)])
-    else:
-        CConfig.tm = platform.Struct("struct tm", [("tm_sec", rffi.INT),
-            ("tm_min", rffi.INT), ("tm_hour", rffi.INT), ("tm_mday", rffi.INT),
-            ("tm_mon", rffi.INT), ("tm_year", rffi.INT), ("tm_wday", rffi.INT),
-            ("tm_yday", rffi.INT), ("tm_isdst", rffi.INT), ("tm_gmtoff", rffi.LONG),
-            ("tm_zone", rffi.CCHARP)])
-elif _WIN:
-    calling_conv = 'win'
-    CConfig.tm = platform.Struct("struct tm", [("tm_sec", rffi.INT),
-        ("tm_min", rffi.INT), ("tm_hour", rffi.INT), ("tm_mday", rffi.INT),
-        ("tm_mon", rffi.INT), ("tm_year", rffi.INT), ("tm_wday", rffi.INT),
-        ("tm_yday", rffi.INT), ("tm_isdst", rffi.INT)])
-
-class cConfig:
-    pass
-
-for k, v in platform.configure(CConfig).items():
-    setattr(cConfig, k, v)
-cConfig.tm.__name__ = "_tm"
-
-def external(name, args, result, eci=CConfig._compilation_info_):
-    if _WIN and rffi.sizeof(rffi.TIME_T) == 8:
-        # Recent Microsoft compilers use 64bit time_t and
-        # the corresponding functions are named differently
-        if (rffi.TIME_T in args or rffi.TIME_TP in args
-            or result in (rffi.TIME_T, rffi.TIME_TP)):
-            name = '_' + name + '64'
-    return rffi.llexternal(name, args, result,
-                           compilation_info=eci,
-                           calling_conv=calling_conv,
-                           releasegil=False)
-
-if _POSIX:
-    cConfig.timeval.__name__ = "_timeval"
-    timeval = cConfig.timeval
-
-CLOCKS_PER_SEC = cConfig.CLOCKS_PER_SEC
-clock_t = cConfig.clock_t
-tm = cConfig.tm
-glob_buf = lltype.malloc(tm, flavor='raw', zero=True, immortal=True)
-
-if cConfig.has_gettimeofday:
-    c_gettimeofday = external('gettimeofday', [rffi.VOIDP, rffi.VOIDP], rffi.INT)
-TM_P = lltype.Ptr(tm)
-c_clock = external('clock', [rffi.TIME_TP], clock_t)
-c_time = external('time', [rffi.TIME_TP], rffi.TIME_T)
-c_ctime = external('ctime', [rffi.TIME_TP], rffi.CCHARP)
-c_gmtime = external('gmtime', [rffi.TIME_TP], TM_P)
-c_mktime = external('mktime', [TM_P], rffi.TIME_T)
-c_asctime = external('asctime', [TM_P], rffi.CCHARP)
-c_localtime = external('localtime', [rffi.TIME_TP], TM_P)
-if _POSIX:
-    c_tzset = external('tzset', [], lltype.Void)
-if _WIN:
-    win_eci = ExternalCompilationInfo(
-        includes = ["time.h"],
-        post_include_bits = ["RPY_EXTERN "
-                             "long pypy_get_timezone();\n"
-                             "RPY_EXTERN "
-                             "int pypy_get_daylight();\n"
-                             "RPY_EXTERN "
-                             "char** pypy_get_tzname();\n"
-                             "RPY_EXTERN "
-                             "void pypy__tzset();"],
-        separate_module_sources = ["""
-        long pypy_get_timezone() { return timezone; }
-        int pypy_get_daylight() { return daylight; }
-        char** pypy_get_tzname() { return tzname; }
-        void pypy__tzset() { _tzset(); }
-        """])
-    # Ensure sure that we use _tzset() and timezone from the same C Runtime.
-    c_tzset = external('pypy__tzset', [], lltype.Void, win_eci)
-    c_get_timezone = external('pypy_get_timezone', [], rffi.LONG, win_eci)
-    c_get_daylight = external('pypy_get_daylight', [], rffi.INT, win_eci)
-    c_get_tzname = external('pypy_get_tzname', [], rffi.CCHARPP, win_eci)
-
-c_strftime = external('strftime', [rffi.CCHARP, rffi.SIZE_T, rffi.CCHARP, TM_P],
-                      rffi.SIZE_T)
-
-def _init_accept2dyear(space):
-    if os.environ.get("PYTHONY2K"):
-        accept2dyear = 0
-    else:
-        accept2dyear = 1
-    _set_module_object(space, "accept2dyear", space.wrap(accept2dyear))
-
-def _init_timezone(space):
-    timezone = daylight = altzone = 0
-    tzname = ["", ""]
-
-    if _WIN:
-        c_tzset()
-        timezone = c_get_timezone()
-        altzone = timezone - 3600
-        daylight = c_get_daylight()
-        tzname_ptr = c_get_tzname()
-        tzname = rffi.charp2str(tzname_ptr[0]), rffi.charp2str(tzname_ptr[1])
-
-    if _POSIX:
-        if _CYGWIN:
-            YEAR = (365 * 24 + 6) * 3600
-
-            # about January 11th
-            t = (((c_time(lltype.nullptr(rffi.TIME_TP.TO))) / YEAR) * YEAR + 10 * 24 * 3600)
-            # we cannot have reference to stack variable, put it on the heap
-            t_ref = lltype.malloc(rffi.TIME_TP.TO, 1, flavor='raw')
-            t_ref[0] = rffi.cast(rffi.TIME_T, t)
-            p = c_localtime(t_ref)
-            q = c_gmtime(t_ref)
-            janzone = (p.c_tm_hour + 24 * p.c_tm_mday) - (q.c_tm_hour + 24 * q.c_tm_mday)
-            if janzone < -12:
-                janname = "   "
-            elif janzone > 14:
-                janname = "   "
-            else:
-                janname = _time_zones[janzone - 12]
-            janzone = janzone * 3600
-            # about July 11th
-            tt = t + YEAR / 2
-            t_ref[0] = rffi.cast(rffi.TIME_T, tt)
-            p = c_localtime(t_ref)
-            q = c_gmtime(t_ref)
-            julyzone = (p.c_tm_hour + 24 * p.c_tm_mday) - (q.c_tm_hour + 24 * q.c_tm_mday)
-            if julyzone < -12:
-                julyname = "   "
-            elif julyzone > 14:
-                julyname = "   "
-            else:
-                julyname = _time_zones[julyzone - 12]
-            julyzone = julyzone * 3600
-            lltype.free(t_ref, flavor='raw')
-
-            if janzone < julyzone:
-                # DST is reversed in the southern hemisphere
-                timezone = julyzone
-                altzone = janzone
-                daylight = int(janzone != julyzone)
-                tzname = [julyname, janname]
-            else:
-                timezone = janzone
-                altzone = julyzone
-                daylight = int(janzone != julyzone)
-                tzname = [janname, julyname]
-
-        else:
-            YEAR = (365 * 24 + 6) * 3600
-
-            t = (((c_time(lltype.nullptr(rffi.TIME_TP.TO))) / YEAR) * YEAR)
-            # we cannot have reference to stack variable, put it on the heap
-            t_ref = lltype.malloc(rffi.TIME_TP.TO, 1, flavor='raw')
-            t_ref[0] = rffi.cast(rffi.TIME_T, t)
-            p = c_localtime(t_ref)
-            janzone = -p.c_tm_gmtoff
-            tm_zone = rffi.charp2str(p.c_tm_zone)
-            janname = ["   ", tm_zone][bool(tm_zone)]
-            tt = t + YEAR / 2
-            t_ref[0] = rffi.cast(rffi.TIME_T, tt)
-            p = c_localtime(t_ref)
-            lltype.free(t_ref, flavor='raw')
-            tm_zone = rffi.charp2str(p.c_tm_zone)
-            julyzone = -p.c_tm_gmtoff
-            julyname = ["   ", tm_zone][bool(tm_zone)]
-
-            if janzone < julyzone:
-                # DST is reversed in the southern hemisphere
-                timezone = julyzone
-                altzone = janzone
-                daylight = int(janzone != julyzone)
-                tzname = [julyname, janname]
-            else:
-                timezone = janzone
-                altzone = julyzone
-                daylight = int(janzone != julyzone)
-                tzname = [janname, julyname]
-
-    _set_module_object(space, "timezone", space.wrap(timezone))
-    _set_module_object(space, 'daylight', space.wrap(daylight))
-    tzname_w = [space.wrap(tzname[0]), space.wrap(tzname[1])]
-    _set_module_object(space, 'tzname', space.newtuple(tzname_w))
-    _set_module_object(space, 'altzone', space.wrap(altzone))
-
-def _get_error_msg():
-    errno = rposix.get_errno()
-    return os.strerror(errno)
-
-if sys.platform != 'win32':
-    @unwrap_spec(secs=float)
-    def sleep(space, secs):
-        if secs < 0:
-            raise OperationError(space.w_IOError,
-                                 space.wrap("Invalid argument: negative time in sleep"))
-        pytime.sleep(secs)
-else:
-    from rpython.rlib import rwin32
-    from errno import EINTR
-    def _simple_sleep(space, secs, interruptible):
-        if secs == 0.0 or not interruptible:
-            pytime.sleep(secs)
-        else:
-            millisecs = int(secs * 1000)
-            interrupt_event = space.fromcache(State).get_interrupt_event()
-            rwin32.ResetEvent(interrupt_event)
-            rc = rwin32.WaitForSingleObject(interrupt_event, millisecs)
-            if rc == rwin32.WAIT_OBJECT_0:
-                # Yield to make sure real Python signal handler
-                # called.
-                pytime.sleep(0.001)
-                raise wrap_oserror(space,
-                                   OSError(EINTR, "sleep() interrupted"))
-    @unwrap_spec(secs=float)
-    def sleep(space, secs):
-        if secs < 0:
-            raise OperationError(space.w_IOError,
-                                 space.wrap("Invalid argument: negative time in sleep"))
-        # as decreed by Guido, only the main thread can be
-        # interrupted.
-        main_thread = space.fromcache(State).main_thread
-        interruptible = (main_thread == thread.get_ident())
-        MAX = sys.maxint / 1000.0 # > 24 days
-        while secs > MAX:
-            _simple_sleep(space, MAX, interruptible)
-            secs -= MAX
-        _simple_sleep(space, secs, interruptible)
-
-def _get_module_object(space, obj_name):
-    w_module = space.getbuiltinmodule('time')
-    w_obj = space.getattr(w_module, space.wrap(obj_name))
-    return w_obj
-
-def _set_module_object(space, obj_name, w_obj_value):
-    w_module = space.getbuiltinmodule('time')
-    space.setattr(w_module, space.wrap(obj_name), w_obj_value)
-
-def _get_inttime(space, w_seconds):
-    # w_seconds can be a wrapped None (it will be automatically wrapped
-    # in the callers, so we never get a real None here).
-    if space.is_none(w_seconds):
-        seconds = pytime.time()
-    else:
-        seconds = space.float_w(w_seconds)
-    #
-    t = rffi.cast(rffi.TIME_T, seconds)
-    #
-    # Logic from CPython: How much info did we lose?  We assume that
-    # time_t is an integral type.  If we lost a second or more, the
-    # input doesn't fit in a time_t; call it an error.
-    diff = seconds - rffi.cast(lltype.Float, t)
-    if diff <= -1.0 or diff >= 1.0:
-        raise OperationError(space.w_ValueError,
-                      space.wrap("timestamp out of range for platform time_t"))
-    return t
-
-def _tm_to_tuple(space, t):
-    time_tuple = [
-        space.wrap(rffi.getintfield(t, 'c_tm_year') + 1900),
-        space.wrap(rffi.getintfield(t, 'c_tm_mon') + 1), # want january == 1
-        space.wrap(rffi.getintfield(t, 'c_tm_mday')),
-        space.wrap(rffi.getintfield(t, 'c_tm_hour')),
-        space.wrap(rffi.getintfield(t, 'c_tm_min')),
-        space.wrap(rffi.getintfield(t, 'c_tm_sec')),
-        space.wrap((rffi.getintfield(t, 'c_tm_wday') + 6) % 7), # want monday == 0
-        space.wrap(rffi.getintfield(t, 'c_tm_yday') + 1), # want january, 1 == 1
-        space.wrap(rffi.getintfield(t, 'c_tm_isdst'))]
-
-    w_struct_time = _get_module_object(space, 'struct_time')
-    w_time_tuple = space.newtuple(time_tuple)
-    return space.call_function(w_struct_time, w_time_tuple)
-
-def _gettmarg(space, w_tup, allowNone=True):
-    if space.is_none(w_tup):
-        if not allowNone:
-            raise OperationError(space.w_TypeError,
-                                 space.wrap("tuple expected"))
-        # default to the current local time
-        tt = rffi.r_time_t(int(pytime.time()))
-        t_ref = lltype.malloc(rffi.TIME_TP.TO, 1, flavor='raw')
-        t_ref[0] = tt
-        pbuf = c_localtime(t_ref)
-        lltype.free(t_ref, flavor='raw')
-        if not pbuf:
-            raise OperationError(space.w_ValueError,
-                space.wrap(_get_error_msg()))
-        return pbuf
-
-    tup_w = space.fixedview(w_tup)
-    if len(tup_w) != 9:
-        raise oefmt(space.w_TypeError,
-                    "argument must be sequence of length 9, not %d",
-                    len(tup_w))
-
-    y = space.int_w(tup_w[0])
-    tm_mon = space.int_w(tup_w[1])
-    if tm_mon == 0:
-        tm_mon = 1
-    tm_mday = space.int_w(tup_w[2])
-    if tm_mday == 0:
-        tm_mday = 1
-    tm_yday = space.int_w(tup_w[7])
-    if tm_yday == 0:
-        tm_yday = 1
-    rffi.setintfield(glob_buf, 'c_tm_mon', tm_mon)
-    rffi.setintfield(glob_buf, 'c_tm_mday', tm_mday)
-    rffi.setintfield(glob_buf, 'c_tm_hour', space.int_w(tup_w[3]))
-    rffi.setintfield(glob_buf, 'c_tm_min', space.int_w(tup_w[4]))
-    rffi.setintfield(glob_buf, 'c_tm_sec', space.int_w(tup_w[5]))
-    rffi.setintfield(glob_buf, 'c_tm_wday', space.int_w(tup_w[6]))
-    rffi.setintfield(glob_buf, 'c_tm_yday', tm_yday)
-    rffi.setintfield(glob_buf, 'c_tm_isdst', space.int_w(tup_w[8]))
-    if _POSIX:
-        if _CYGWIN:
-            pass
-        else:
-            # actually never happens, but makes annotator happy
-            glob_buf.c_tm_zone = lltype.nullptr(rffi.CCHARP.TO)
-            rffi.setintfield(glob_buf, 'c_tm_gmtoff', 0)
-
-    if y < 1900:
-        w_accept2dyear = _get_module_object(space, "accept2dyear")
-        accept2dyear = space.int_w(w_accept2dyear)
-
-        if not accept2dyear:
-            raise OperationError(space.w_ValueError,
-                space.wrap("year >= 1900 required"))
-
-        if 69 <= y <= 99:
-            y += 1900
-        elif 0 <= y <= 68:
-            y += 2000
-        else:
-            raise OperationError(space.w_ValueError,
-                space.wrap("year out of range"))
-
-    # tm_wday does not need checking of its upper-bound since taking "%
-    #  7" in gettmarg() automatically restricts the range.
-    if rffi.getintfield(glob_buf, 'c_tm_wday') < -1:
-        raise OperationError(space.w_ValueError,
-                             space.wrap("day of week out of range"))
-
-    rffi.setintfield(glob_buf, 'c_tm_year', y - 1900)
-    rffi.setintfield(glob_buf, 'c_tm_mon',
-                     rffi.getintfield(glob_buf, 'c_tm_mon') - 1)
-    rffi.setintfield(glob_buf, 'c_tm_wday',
-                     (rffi.getintfield(glob_buf, 'c_tm_wday') + 1) % 7)
-    rffi.setintfield(glob_buf, 'c_tm_yday',
-                     rffi.getintfield(glob_buf, 'c_tm_yday') - 1)
-
-    return glob_buf
-
-def time(space):
-    """time() -> floating point number
-
-    Return the current time in seconds since the Epoch.
-    Fractions of a second may be present if the system clock provides them."""
-
-    secs = pytime.time()
-    return space.wrap(secs)
-
-if _WIN:
-    class PCCache:
-        pass
-    pccache = PCCache()
-    pccache.divisor = 0.0
-    pccache.ctrStart = 0
-
-def clock(space):
-    """clock() -> floating point number
-
-    Return the CPU time or real time since the start of the process or since
-    the first call to clock().  This has as much precision as the system
-    records."""
-
-    return space.wrap(pytime.clock())
-
-def ctime(space, w_seconds=None):
-    """ctime([seconds]) -> string
-
-    Convert a time in seconds since the Epoch to a string in local time.
-    This is equivalent to asctime(localtime(seconds)). When the time tuple is
-    not present, current time as returned by localtime() is used."""
-
-    seconds = _get_inttime(space, w_seconds)
-
-    t_ref = lltype.malloc(rffi.TIME_TP.TO, 1, flavor='raw')
-    t_ref[0] = seconds
-    p = c_ctime(t_ref)
-    lltype.free(t_ref, flavor='raw')
-    if not p:
-        raise OperationError(space.w_ValueError,
-            space.wrap("unconvertible time"))
-
-    return space.wrap(rffi.charp2str(p)[:-1]) # get rid of new line
-
-# by now w_tup is an optional argument (and not *args)
-# because of the ext. compiler bugs in handling such arguments (*args, **kwds)
-def asctime(space, w_tup=None):
-    """asctime([tuple]) -> string
-
-    Convert a time tuple to a string, e.g. 'Sat Jun 06 16:26:11 1998'.
-    When the time tuple is not present, current time as returned by localtime()
-    is used."""
-    buf_value = _gettmarg(space, w_tup)
-    p = c_asctime(buf_value)
-    if not p:
-        raise OperationError(space.w_ValueError,
-            space.wrap("unconvertible time"))
-
-    return space.wrap(rffi.charp2str(p)[:-1]) # get rid of new line
-
-def gmtime(space, w_seconds=None):
-    """gmtime([seconds]) -> (tm_year, tm_mon, tm_day, tm_hour, tm_min,
-                          tm_sec, tm_wday, tm_yday, tm_isdst)
-
-    Convert seconds since the Epoch to a time tuple expressing UTC (a.k.a.
-    GMT).  When 'seconds' is not passed in, convert the current time instead.
-    """
-
-    # rpython does not support that a variable has two incompatible builtins
-    # as value so we have to duplicate the code. NOT GOOD! see localtime() too
-    seconds = _get_inttime(space, w_seconds)
-    t_ref = lltype.malloc(rffi.TIME_TP.TO, 1, flavor='raw')
-    t_ref[0] = seconds
-    p = c_gmtime(t_ref)
-    lltype.free(t_ref, flavor='raw')
-
-    if not p:
-        raise OperationError(space.w_ValueError, space.wrap(_get_error_msg()))
-    return _tm_to_tuple(space, p)
-
-def localtime(space, w_seconds=None):
-    """localtime([seconds]) -> (tm_year, tm_mon, tm_day, tm_hour, tm_min,
-                             tm_sec, tm_wday, tm_yday, tm_isdst)
-
-    Convert seconds since the Epoch to a time tuple expressing local time.
-    When 'seconds' is not passed in, convert the current time instead."""
-
-    seconds = _get_inttime(space, w_seconds)
-    t_ref = lltype.malloc(rffi.TIME_TP.TO, 1, flavor='raw')
-    t_ref[0] = seconds
-    p = c_localtime(t_ref)
-    lltype.free(t_ref, flavor='raw')
-
-    if not p:
-        raise OperationError(space.w_ValueError, space.wrap(_get_error_msg()))
-    return _tm_to_tuple(space, p)
-
-def mktime(space, w_tup):
-    """mktime(tuple) -> floating point number
-
-    Convert a time tuple in local time to seconds since the Epoch."""
-
-    buf = _gettmarg(space, w_tup, allowNone=False)
-    rffi.setintfield(buf, "c_tm_wday", -1)
-    tt = c_mktime(buf)
-    # A return value of -1 does not necessarily mean an error, but tm_wday
-    # cannot remain set to -1 if mktime succeeds.
-    if tt == -1 and rffi.getintfield(buf, "c_tm_wday") == -1:
-        raise OperationError(space.w_OverflowError,
-            space.wrap("mktime argument out of range"))
-
-    return space.wrap(float(tt))
-
-if _POSIX:
-    def tzset(space):
-        """tzset()
-
-        Initialize, or reinitialize, the local timezone to the value stored in
-        os.environ['TZ']. The TZ environment variable should be specified in
-        standard Unix timezone format as documented in the tzset man page
-        (eg. 'US/Eastern', 'Europe/Amsterdam'). Unknown timezones will silently
-        fall back to UTC. If the TZ environment variable is not set, the local
-        timezone is set to the systems best guess of wallclock time.
-        Changing the TZ environment variable without calling tzset *may* change
-        the local timezone used by methods such as localtime, but this behaviour
-        should not be relied on"""
-
-        c_tzset()
-
-        # reset timezone, altzone, daylight and tzname
-        _init_timezone(space)
-
- at unwrap_spec(format=str)
-def strftime(space, format, w_tup=None):
-    """strftime(format[, tuple]) -> string
-
-    Convert a time tuple to a string according to a format specification.
-    See the library reference manual for formatting codes. When the time tuple
-    is not present, current time as returned by localtime() is used."""
-    buf_value = _gettmarg(space, w_tup)
-
-    # Checks added to make sure strftime() does not crash Python by
-    # indexing blindly into some array for a textual representation
-    # by some bad index (fixes bug #897625).
-    # No check for year since handled in gettmarg().
-    if rffi.getintfield(buf_value, 'c_tm_mon') < 0 or rffi.getintfield(buf_value, 'c_tm_mon') > 11:
-        raise OperationError(space.w_ValueError,
-                             space.wrap("month out of range"))
-    if rffi.getintfield(buf_value, 'c_tm_mday') < 1 or rffi.getintfield(buf_value, 'c_tm_mday') > 31:
-        raise OperationError(space.w_ValueError,
-                             space.wrap("day of month out of range"))
-    if rffi.getintfield(buf_value, 'c_tm_hour') < 0 or rffi.getintfield(buf_value, 'c_tm_hour') > 23:
-        raise OperationError(space.w_ValueError,
-                             space.wrap("hour out of range"))
-    if rffi.getintfield(buf_value, 'c_tm_min') < 0 or rffi.getintfield(buf_value, 'c_tm_min') > 59:
-        raise OperationError(space.w_ValueError,
-                             space.wrap("minute out of range"))
-    if rffi.getintfield(buf_value, 'c_tm_sec') < 0 or rffi.getintfield(buf_value, 'c_tm_sec') > 61:
-        raise OperationError(space.w_ValueError,
-                             space.wrap("seconds out of range"))
-    if rffi.getintfield(buf_value, 'c_tm_yday') < 0 or rffi.getintfield(buf_value, 'c_tm_yday') > 365:
-        raise OperationError(space.w_ValueError,
-                             space.wrap("day of year out of range"))
-    if rffi.getintfield(buf_value, 'c_tm_isdst') < -1 or rffi.getintfield(buf_value, 'c_tm_isdst') > 1:
-        raise OperationError(space.w_ValueError,
-                             space.wrap("daylight savings flag out of range"))
-
-    if _WIN:
-        # check that the format string contains only valid directives
-        length = len(format)
-        i = 0
-        while i < length:
-            if format[i] == '%':
-                i += 1
-                if i < length and format[i] == '#':
-                    # not documented by python
-                    i += 1
-                if i >= length or format[i] not in "aAbBcdHIjmMpSUwWxXyYzZ%":
-                    raise OperationError(space.w_ValueError,
-                                         space.wrap("invalid format string"))
-            i += 1
-
-    i = 1024
-    while True:
-        outbuf = lltype.malloc(rffi.CCHARP.TO, i, flavor='raw')
-        try:
-            buflen = c_strftime(outbuf, i, format, buf_value)
-            if buflen > 0 or i >= 256 * len(format):
-                # if the buffer is 256 times as long as the format,
-                # it's probably not failing for lack of room!
-                # More likely, the format yields an empty result,
-                # e.g. an empty format, or %Z when the timezone
-                # is unknown.
-                result = rffi.charp2strn(outbuf, intmask(buflen))
-                return space.wrap(result)
-        finally:
-            lltype.free(outbuf, flavor='raw')
-        i += i
diff --git a/pypy/module/rctime/test/test_rctime.py b/pypy/module/rctime/test/test_rctime.py
deleted file mode 100644
--- a/pypy/module/rctime/test/test_rctime.py
+++ /dev/null
@@ -1,333 +0,0 @@
-class AppTestRCTime:
-    spaceconfig = {
-        "usemodules": ['rctime', 'struct', 'binascii'],
-    }
-
-    def test_attributes(self):
-        import time as rctime
-        assert isinstance(rctime.accept2dyear, int)
-        assert isinstance(rctime.altzone, int)
-        assert isinstance(rctime.daylight, int)
-        assert isinstance(rctime.timezone, int)
-        assert isinstance(rctime.tzname, tuple)
-        assert isinstance(rctime.__doc__, str)
-
-    def test_sleep(self):
-        import time as rctime
-        import sys
-        import os
-        raises(TypeError, rctime.sleep, "foo")
-        rctime.sleep(0.12345)
-        raises(IOError, rctime.sleep, -1.0)
-
-    def test_clock(self):
-        import time as rctime
-        rctime.clock()
-        assert isinstance(rctime.clock(), float)
-
-    def test_time(self):
-        import time as rctime
-        t1 = rctime.time()
-        assert isinstance(rctime.time(), float)
-        assert rctime.time() != 0.0 # 0.0 means failure
-        rctime.sleep(0.02)
-        t2 = rctime.time()
-        assert t1 != t2       # the resolution should be at least 0.01 secs
-
-    def test_ctime(self):
-        import time as rctime
-        raises(TypeError, rctime.ctime, "foo")
-        rctime.ctime(None)
-        rctime.ctime()
-        res = rctime.ctime(0)
-        assert isinstance(res, str)
-        rctime.ctime(rctime.time())
-        raises(ValueError, rctime.ctime, 1E200)
-        raises(OverflowError, rctime.ctime, 10**900)
-
-    def test_gmtime(self):
-        import time as rctime
-        raises(TypeError, rctime.gmtime, "foo")
-        rctime.gmtime()
-        rctime.gmtime(None)
-        rctime.gmtime(0)
-        res = rctime.gmtime(rctime.time())
-        assert isinstance(res, rctime.struct_time)
-        assert res[-1] == 0 # DST is always zero in gmtime()
-        t0 = rctime.mktime(rctime.gmtime())
-        t1 = rctime.mktime(rctime.gmtime(None))
-        assert 0 <= (t1 - t0) < 1.2
-        t = rctime.time()
-        assert rctime.gmtime(t) == rctime.gmtime(t)
-        raises(ValueError, rctime.gmtime, 2**64)
-        raises(ValueError, rctime.gmtime, -2**64)
-
-    def test_localtime(self):
-        import time as rctime
-        import os
-        raises(TypeError, rctime.localtime, "foo")
-        rctime.localtime()
-        rctime.localtime(None)
-        rctime.localtime(0)
-        res = rctime.localtime(rctime.time())
-        assert isinstance(res, rctime.struct_time)
-        t0 = rctime.mktime(rctime.localtime())
-        t1 = rctime.mktime(rctime.localtime(None))
-        assert 0 <= (t1 - t0) < 1.2
-        t = rctime.time()
-        assert rctime.localtime(t) == rctime.localtime(t)
-        if os.name == 'nt':
-            raises(ValueError, rctime.localtime, -1)
-        else:
-            rctime.localtime(-1)
-
-    def test_mktime(self):
-        import time as rctime
-        import os, sys
-        raises(TypeError, rctime.mktime, "foo")
-        raises(TypeError, rctime.mktime, None)
-        raises(TypeError, rctime.mktime, (1, 2))
-        raises(TypeError, rctime.mktime, (1, 2, 3, 4, 5, 6, 'f', 8, 9))
-        res = rctime.mktime(rctime.localtime())
-        assert isinstance(res, float)
-
-        ltime = rctime.localtime()
-        rctime.accept2dyear == 0
-        ltime = list(ltime)
-        ltime[0] = 1899
-        raises(ValueError, rctime.mktime, tuple(ltime))
-        rctime.accept2dyear == 1
-
-        ltime = list(ltime)
-        ltime[0] = 67
-        ltime = tuple(ltime)
-        if os.name != "nt" and sys.maxint < 1<<32:   # time_t may be 64bit
-            raises(OverflowError, rctime.mktime, ltime)
-
-        ltime = list(ltime)
-        ltime[0] = 100
-        raises(ValueError, rctime.mktime, tuple(ltime))
-
-        t = rctime.time()
-        assert long(rctime.mktime(rctime.localtime(t))) == long(t)
-        assert long(rctime.mktime(rctime.gmtime(t))) - rctime.timezone == long(t)
-        ltime = rctime.localtime()
-        assert rctime.mktime(tuple(ltime)) == rctime.mktime(ltime)
-        if os.name != 'nt':
-            assert rctime.mktime(rctime.localtime(-1)) == -1
-
-        res = rctime.mktime((2000, 1, 1, 0, 0, 0, -1, -1, -1))
-        if os.name == 'nt':
-            assert rctime.ctime(res) == 'Sat Jan 01 00:00:00 2000'
-        else:
-            assert rctime.ctime(res) == 'Sat Jan  1 00:00:00 2000'
-
-    def test_asctime(self):
-        import time as rctime
-        rctime.asctime()
-        # raises(TypeError, rctime.asctime, None)
-        raises(TypeError, rctime.asctime, ())
-        raises(TypeError, rctime.asctime, (1,))
-        raises(TypeError, rctime.asctime, range(8))
-        raises(TypeError, rctime.asctime, (1, 2))
-        raises(TypeError, rctime.asctime, (1, 2, 3, 4, 5, 6, 'f', 8, 9))
-        raises(TypeError, rctime.asctime, "foo")
-        res = rctime.asctime()
-        assert isinstance(res, str)
-        rctime.asctime(rctime.localtime())
-        t = rctime.time()
-        assert rctime.ctime(t) == rctime.asctime(rctime.localtime(t))
-        if rctime.timezone:
-            assert rctime.ctime(t) != rctime.asctime(rctime.gmtime(t))
-        ltime = rctime.localtime()
-        assert rctime.asctime(tuple(ltime)) == rctime.asctime(ltime)
-        try:
-            rctime.asctime((12345,) + (0,) * 8)  # assert this doesn't crash
-        except ValueError:
-            pass  # some OS (ie POSIXes besides Linux) reject year > 9999
-
-    def test_accept2dyear_access(self):
-        import time as rctime
-
-        accept2dyear = rctime.accept2dyear
-        del rctime.accept2dyear
-        try:
-            # with year >= 1900 this shouldn't need to access accept2dyear
-            assert rctime.asctime((2000,) + (0,) * 8).split()[-1] == '2000'
-        finally:
-            rctime.accept2dyear = accept2dyear
-
-    def test_struct_time(self):
-        import time as rctime
-        raises(TypeError, rctime.struct_time)
-        raises(TypeError, rctime.struct_time, "foo")
-        raises(TypeError, rctime.struct_time, (1, 2, 3))
-        tup = (1, 2, 3, 4, 5, 6, 7, 8, 9)
-        st_time = rctime.struct_time(tup)
-        assert str(st_time).startswith('time.struct_time(tm_year=1, ')
-        assert len(st_time) == len(tup)
-
-    def test_tzset(self):
-        import time as rctime
-        import os
-
-        if not os.name == "posix":
-            skip("tzset available only under Unix")
-
-        # epoch time of midnight Dec 25th 2002. Never DST in northern
-        # hemisphere.
-        xmas2002 = 1040774400.0
-
-        # these formats are correct for 2002, and possibly future years
-        # this format is the 'standard' as documented at:
-        # http://www.opengroup.org/onlinepubs/007904975/basedefs/xbd_chap08.html
-        # They are also documented in the tzset(3) man page on most Unix
-        # systems.
-        eastern = 'EST+05EDT,M4.1.0,M10.5.0'
-        victoria = 'AEST-10AEDT-11,M10.5.0,M3.5.0'
-        utc = 'UTC+0'
-
-        org_TZ = os.environ.get('TZ', None)
-        try:
-            # Make sure we can switch to UTC time and results are correct
-            # Note that unknown timezones default to UTC.
-            # Note that altzone is undefined in UTC, as there is no DST
-            os.environ['TZ'] = eastern
-            rctime.tzset()
-            os.environ['TZ'] = utc
-            rctime.tzset()
-            assert rctime.gmtime(xmas2002) == rctime.localtime(xmas2002)
-            assert rctime.daylight == 0
-            assert rctime.timezone == 0
-            assert rctime.localtime(xmas2002).tm_isdst == 0
-
-            # make sure we can switch to US/Eastern
-            os.environ['TZ'] = eastern
-            rctime.tzset()
-            assert rctime.gmtime(xmas2002) != rctime.localtime(xmas2002)
-            assert rctime.tzname == ('EST', 'EDT')
-            assert len(rctime.tzname) == 2
-            assert rctime.daylight == 1
-            assert rctime.timezone == 18000
-            assert rctime.altzone == 14400
-            assert rctime.localtime(xmas2002).tm_isdst == 0
-
-            # now go to the southern hemisphere.
-            os.environ['TZ'] = victoria
-            rctime.tzset()
-            assert rctime.gmtime(xmas2002) != rctime.localtime(xmas2002)
-            assert rctime.tzname[0] == 'AEST'
-            assert rctime.tzname[1] == 'AEDT'
-            assert len(rctime.tzname) == 2
-            assert rctime.daylight == 1
-            assert rctime.timezone == -36000
-            assert rctime.altzone == -39600
-            assert rctime.localtime(xmas2002).tm_isdst == 1
-        finally:
-            # repair TZ environment variable in case any other tests
-            # rely on it.
-            if org_TZ is not None:
-                os.environ['TZ'] = org_TZ
-            elif os.environ.has_key('TZ'):
-                del os.environ['TZ']
-            rctime.tzset()
-
-    def test_strftime(self):
-        import time as rctime
-        import os, sys
-
-        t = rctime.time()
-        tt = rctime.gmtime(t)
-        for directive in ('a', 'A', 'b', 'B', 'c', 'd', 'H', 'I',
-                          'j', 'm', 'M', 'p', 'S',
-                          'U', 'w', 'W', 'x', 'X', 'y', 'Y', 'Z', '%'):
-            format = ' %' + directive
-            rctime.strftime(format, tt)
-
-        raises(TypeError, rctime.strftime, ())
-        raises(TypeError, rctime.strftime, (1,))
-        raises(TypeError, rctime.strftime, range(8))
-        exp = '2000 01 01 00 00 00 1 001'
-        assert rctime.strftime("%Y %m %d %H %M %S %w %j", (0,)*9) == exp
-
-        # Guard against invalid/non-supported format string
-        # so that Python don't crash (Windows crashes when the format string
-        # input to [w]strftime is not kosher.
-        if os.name == 'nt':
-            raises(ValueError, rctime.strftime, '%f')
-        elif sys.platform == 'darwin' or 'bsd' in sys.platform:
-            # darwin strips % of unknown format codes
-            # http://bugs.python.org/issue9811
-            assert rctime.strftime('%f') == 'f'
-        else:
-            assert rctime.strftime('%f') == '%f'
-
-    def test_strftime_ext(self):
-        import time as rctime
-
-        tt = rctime.gmtime()
-        try:
-            result = rctime.strftime('%D', tt)
-        except ValueError:
-            pass
-        else:
-            assert result == rctime.strftime('%m/%d/%y', tt)
-
-    def test_strftime_bounds_checking(self):
-        import time as rctime
-
-        # make sure that strftime() checks the bounds of the various parts
-        # of the time tuple.
-
-        # check year
-        raises(ValueError, rctime.strftime, '', (1899, 1, 1, 0, 0, 0, 0, 1, -1))
-        if rctime.accept2dyear:
-            raises(ValueError, rctime.strftime, '', (-1, 1, 1, 0, 0, 0, 0, 1, -1))
-            raises(ValueError, rctime.strftime, '', (100, 1, 1, 0, 0, 0, 0, 1, -1))
-        # check month
-        raises(ValueError, rctime.strftime, '', (1900, 13, 1, 0, 0, 0, 0, 1, -1))
-        # check day of month
-        raises(ValueError, rctime.strftime, '', (1900, 1, 32, 0, 0, 0, 0, 1, -1))
-        # check hour
-        raises(ValueError, rctime.strftime, '', (1900, 1, 1, -1, 0, 0, 0, 1, -1))
-        raises(ValueError, rctime.strftime, '', (1900, 1, 1, 24, 0, 0, 0, 1, -1))
-        # check minute
-        raises(ValueError, rctime.strftime, '', (1900, 1, 1, 0, -1, 0, 0, 1, -1))
-        raises(ValueError, rctime.strftime, '', (1900, 1, 1, 0, 60, 0, 0, 1, -1))
-        # check second
-        raises(ValueError, rctime.strftime, '', (1900, 1, 1, 0, 0, -1, 0, 1, -1))
-        # C99 only requires allowing for one leap second, but Python's docs say
-        # allow two leap seconds (0..61)
-        raises(ValueError, rctime.strftime, '', (1900, 1, 1, 0, 0, 62, 0, 1, -1))
-        # no check for upper-bound day of week;
-        #  value forced into range by a "% 7" calculation.
-        # start check at -2 since gettmarg() increments value before taking
-        #  modulo.
-        raises(ValueError, rctime.strftime, '', (1900, 1, 1, 0, 0, 0, -2, 1, -1))
-        # check day of the year
-        raises(ValueError, rctime.strftime, '', (1900, 1, 1, 0, 0, 0, 0, 367, -1))
-        # check daylight savings flag
-        raises(ValueError, rctime.strftime, '', (1900, 1, 1, 0, 0, 0, 0, 1, -2))
-        raises(ValueError, rctime.strftime, '', (1900, 1, 1, 0, 0, 0, 0, 1, 2))
-
-    def test_strptime(self):
-        import time as rctime
-
-        t = rctime.time()
-        tt = rctime.gmtime(t)
-        assert isinstance(rctime.strptime("", ""), type(tt))
-
-        for directive in ('a', 'A', 'b', 'B', 'c', 'd', 'H', 'I',
-                          'j', 'm', 'M', 'p', 'S',
-                          'U', 'w', 'W', 'x', 'X', 'y', 'Y', 'Z', '%'):
-            format = ' %' + directive
-            print format
-            rctime.strptime(rctime.strftime(format, tt), format)
-
-    def test_pickle(self):
-        import pickle
-        import time as rctime
-        now = rctime.localtime()
-        new = pickle.loads(pickle.dumps(now))
-        assert new == now
-        assert type(new) is type(now)
diff --git a/pypy/module/select/test/test_epoll.py b/pypy/module/select/test/test_epoll.py
--- a/pypy/module/select/test/test_epoll.py
+++ b/pypy/module/select/test/test_epoll.py
@@ -7,7 +7,7 @@
 
 class AppTestEpoll(object):
     spaceconfig = {
-        "usemodules": ["select", "_socket", "posix", "rctime"],
+        "usemodules": ["select", "_socket", "posix", "time"],
     }
 
     def setup_class(cls):
diff --git a/pypy/module/select/test/test_kqueue.py b/pypy/module/select/test/test_kqueue.py
--- a/pypy/module/select/test/test_kqueue.py
+++ b/pypy/module/select/test/test_kqueue.py
@@ -4,7 +4,7 @@
 import sys
 
 class AppTestKqueue(object):
-    spaceconfig = dict(usemodules=["select", "_socket", "posix", "rctime"])
+    spaceconfig = dict(usemodules=["select", "_socket", "posix", "time"])
 
     def setup_class(cls):
         if not 'bsd' in sys.platform and \
diff --git a/pypy/module/select/test/test_select.py b/pypy/module/select/test/test_select.py
--- a/pypy/module/select/test/test_select.py
+++ b/pypy/module/select/test/test_select.py
@@ -249,7 +249,7 @@
 class AppTestSelectWithPipes(_AppTestSelect):
     "Use a pipe to get pairs of file descriptors"
     spaceconfig = {
-        "usemodules": ["select", "rctime", "thread"]
+        "usemodules": ["select", "time", "thread"]
     }
 
     def setup_class(cls):
@@ -325,7 +325,7 @@
     so we start our own server.
     """
     spaceconfig = {
-        "usemodules": ["select", "_socket", "rctime", "thread"],
+        "usemodules": ["select", "_socket", "time", "thread"],
     }
 
     def w_make_server(self):
diff --git a/pypy/module/signal/test/test_signal.py b/pypy/module/signal/test/test_signal.py
--- a/pypy/module/signal/test/test_signal.py
+++ b/pypy/module/signal/test/test_signal.py
@@ -36,7 +36,7 @@
 
 class AppTestSignal:
     spaceconfig = {
-        "usemodules": ['signal', 'rctime'] + (['fcntl'] if os.name != 'nt' else []),
+        "usemodules": ['signal', 'time'] + (['fcntl'] if os.name != 'nt' else []),
     }
 
     def setup_class(cls):
diff --git a/pypy/module/sys/test/test_sysmodule.py b/pypy/module/sys/test/test_sysmodule.py
--- a/pypy/module/sys/test/test_sysmodule.py
+++ b/pypy/module/sys/test/test_sysmodule.py
@@ -619,7 +619,7 @@
 
 class AppTestCurrentFramesWithThread(AppTestCurrentFrames):
     spaceconfig = {
-        "usemodules": ["rctime", "thread"],
+        "usemodules": ["time", "thread"],
     }
 
     def test_current_frames(self):
diff --git a/pypy/module/thread/test/support.py b/pypy/module/thread/test/support.py
--- a/pypy/module/thread/test/support.py
+++ b/pypy/module/thread/test/support.py
@@ -41,7 +41,7 @@
 
 
 class GenericTestThread:
-    spaceconfig = dict(usemodules=('thread', 'rctime', 'signal'))
+    spaceconfig = dict(usemodules=('thread', 'time', 'signal'))
 
     def setup_class(cls):
         cls.w_runappdirect = cls.space.wrap(cls.runappdirect)
diff --git a/pypy/module/time/__init__.py b/pypy/module/time/__init__.py
--- a/pypy/module/time/__init__.py
+++ b/pypy/module/time/__init__.py
@@ -1,13 +1,42 @@
+
 from pypy.interpreter.mixedmodule import MixedModule
+import os
+
+_WIN = os.name == "nt"
 
 class Module(MixedModule):
-    """time module"""
+    applevel_name = 'time'
+
+    interpleveldefs = {
+        'time': 'interp_time.time',
+        'clock': 'interp_time.clock',
+        'ctime': 'interp_time.ctime',
+        'asctime': 'interp_time.asctime',
+        'gmtime': 'interp_time.gmtime',
+        'localtime': 'interp_time.localtime',
+        'mktime': 'interp_time.mktime',
+        'strftime': 'interp_time.strftime',
+        'sleep' : 'interp_time.sleep',
+    }
+
+    if os.name == "posix":
+        interpleveldefs['tzset'] = 'interp_time.tzset'
 
     appleveldefs = {
+        'struct_time': 'app_time.struct_time',
+        '__doc__': 'app_time.__doc__',
+        'strptime': 'app_time.strptime',
     }
 
-    interpleveldefs = {
-        'clock'    : 'interp_time.clock',
-        'time'     : 'interp_time.time_',
-        'sleep'    : 'interp_time.sleep',
-    }
+    def startup(self, space):
+        if _WIN:
+            from pypy.module.time.interp_time import State
+            space.fromcache(State).startup(space)
+
+        # this machinery is needed to expose constants
+        # that have to be initialized one time only
+        from pypy.module.time import interp_time
+
+        interp_time._init_timezone(space)
+        interp_time._init_accept2dyear(space)
+
diff --git a/pypy/module/rctime/app_time.py b/pypy/module/time/app_time.py
rename from pypy/module/rctime/app_time.py
rename to pypy/module/time/app_time.py
diff --git a/pypy/module/time/interp_time.py b/pypy/module/time/interp_time.py
--- a/pypy/module/time/interp_time.py
+++ b/pypy/module/time/interp_time.py
@@ -1,20 +1,670 @@
-import time
+from rpython.rtyper.tool import rffi_platform as platform
+from rpython.rtyper.lltypesystem import rffi
+from pypy.interpreter.error import OperationError, oefmt
 from pypy.interpreter.gateway import unwrap_spec
+from rpython.rtyper.lltypesystem import lltype
+from rpython.rlib.rarithmetic import intmask
+from rpython.rlib import rposix
+from rpython.translator.tool.cbuild import ExternalCompilationInfo
+import os
+import sys
+import time as pytime
 
+_POSIX = os.name == "posix"
+_WIN = os.name == "nt"
+_CYGWIN = sys.platform == "cygwin"
+
+_time_zones = []
+if _CYGWIN:
+    _time_zones = ["GMT-12", "GMT-11", "GMT-10", "GMT-9", "GMT-8", "GMT-7",
+                   "GMT-6", "GMT-5", "GMT-4", "GMT-3", "GMT-2", "GMT-1",
+                   "GMT",  "GMT+1", "GMT+2", "GMT+3", "GMT+4", "GMT+5",
+                   "GMT+6",  "GMT+7", "GMT+8", "GMT+9", "GMT+10", "GMT+11",
+                   "GMT+12",  "GMT+13", "GMT+14"]
+
+if _WIN:
+    # Interruptible sleeps on Windows:
+    # We install a specific Console Ctrl Handler which sets an 'event'.
+    # time.sleep() will actually call WaitForSingleObject with the desired
+    # timeout.  On Ctrl-C, the signal handler is called, the event is set,
+    # and the wait function exits.
+    from rpython.rlib import rwin32
+    from pypy.interpreter.error import wrap_windowserror, wrap_oserror
+    from rpython.rlib import rthread as thread
+
+    eci = ExternalCompilationInfo(
+        includes = ['windows.h'],
+        post_include_bits = [
+            "RPY_EXTERN\n"
+            "BOOL pypy_timemodule_setCtrlHandler(HANDLE event);"],
+        separate_module_sources=['''
+            static HANDLE interrupt_event;
+
+            static BOOL WINAPI CtrlHandlerRoutine(
+              DWORD dwCtrlType)
+            {
+                SetEvent(interrupt_event);
+                /* allow other default handlers to be called.
+                 * Default Python handler will setup the
+                 * KeyboardInterrupt exception.
+                 */
+                return 0;
+            }
+
+            BOOL pypy_timemodule_setCtrlHandler(HANDLE event)
+            {
+                interrupt_event = event;
+                return SetConsoleCtrlHandler(CtrlHandlerRoutine, TRUE);
+            }
+
+        '''],
+        )
+    _setCtrlHandlerRoutine = rffi.llexternal(
+        'pypy_timemodule_setCtrlHandler',
+        [rwin32.HANDLE], rwin32.BOOL,
+        compilation_info=eci)
+
+    class GlobalState:
+        def __init__(self):
+            self.init()
+
+        def init(self):
+            self.interrupt_event = rwin32.NULL_HANDLE
+
+        def startup(self, space):
+            # Initialize the event handle used to signal Ctrl-C
+            try:
+                globalState.interrupt_event = rwin32.CreateEvent(
+                    rffi.NULL, True, False, rffi.NULL)
+            except WindowsError, e:
+                raise wrap_windowserror(space, e)
+            if not _setCtrlHandlerRoutine(globalState.interrupt_event):
+                raise wrap_windowserror(
+                    space, rwin32.lastWindowsError("SetConsoleCtrlHandler"))
+
+    globalState = GlobalState()
+
+    class State:
+        def __init__(self, space):
+            self.main_thread = 0
+
+        def _cleanup_(self):
+            self.main_thread = 0
+            globalState.init()
+
+        def startup(self, space):
+            self.main_thread = thread.get_ident()
+            globalState.startup(space)
+
+        def get_interrupt_event(self):
+            return globalState.interrupt_event
+
+
+_includes = ["time.h"]
+if _POSIX:
+    _includes.append('sys/time.h')
+
+class CConfig:
+    _compilation_info_ = ExternalCompilationInfo(
+        includes = _includes
+    )
+    CLOCKS_PER_SEC = platform.ConstantInteger("CLOCKS_PER_SEC")
+    clock_t = platform.SimpleType("clock_t", rffi.ULONG)
+    has_gettimeofday = platform.Has('gettimeofday')
+
+if _POSIX:
+    calling_conv = 'c'
+    CConfig.timeval = platform.Struct("struct timeval",
+                                      [("tv_sec", rffi.INT),
+                                       ("tv_usec", rffi.INT)])
+    if _CYGWIN:
+        CConfig.tm = platform.Struct("struct tm", [("tm_sec", rffi.INT),
+            ("tm_min", rffi.INT), ("tm_hour", rffi.INT), ("tm_mday", rffi.INT),
+            ("tm_mon", rffi.INT), ("tm_year", rffi.INT), ("tm_wday", rffi.INT),
+            ("tm_yday", rffi.INT), ("tm_isdst", rffi.INT)])
+    else:
+        CConfig.tm = platform.Struct("struct tm", [("tm_sec", rffi.INT),
+            ("tm_min", rffi.INT), ("tm_hour", rffi.INT), ("tm_mday", rffi.INT),
+            ("tm_mon", rffi.INT), ("tm_year", rffi.INT), ("tm_wday", rffi.INT),
+            ("tm_yday", rffi.INT), ("tm_isdst", rffi.INT), ("tm_gmtoff", rffi.LONG),
+            ("tm_zone", rffi.CCHARP)])
+elif _WIN:
+    calling_conv = 'win'
+    CConfig.tm = platform.Struct("struct tm", [("tm_sec", rffi.INT),
+        ("tm_min", rffi.INT), ("tm_hour", rffi.INT), ("tm_mday", rffi.INT),
+        ("tm_mon", rffi.INT), ("tm_year", rffi.INT), ("tm_wday", rffi.INT),
+        ("tm_yday", rffi.INT), ("tm_isdst", rffi.INT)])
+
+class cConfig:
+    pass
+
+for k, v in platform.configure(CConfig).items():
+    setattr(cConfig, k, v)
+cConfig.tm.__name__ = "_tm"
+
+def external(name, args, result, eci=CConfig._compilation_info_):
+    if _WIN and rffi.sizeof(rffi.TIME_T) == 8:
+        # Recent Microsoft compilers use 64bit time_t and
+        # the corresponding functions are named differently
+        if (rffi.TIME_T in args or rffi.TIME_TP in args
+            or result in (rffi.TIME_T, rffi.TIME_TP)):
+            name = '_' + name + '64'
+    return rffi.llexternal(name, args, result,
+                           compilation_info=eci,
+                           calling_conv=calling_conv,
+                           releasegil=False)
+
+if _POSIX:
+    cConfig.timeval.__name__ = "_timeval"
+    timeval = cConfig.timeval
+
+CLOCKS_PER_SEC = cConfig.CLOCKS_PER_SEC
+clock_t = cConfig.clock_t
+tm = cConfig.tm
+glob_buf = lltype.malloc(tm, flavor='raw', zero=True, immortal=True)
+
+if cConfig.has_gettimeofday:
+    c_gettimeofday = external('gettimeofday', [rffi.VOIDP, rffi.VOIDP], rffi.INT)
+TM_P = lltype.Ptr(tm)
+c_clock = external('clock', [rffi.TIME_TP], clock_t)
+c_time = external('time', [rffi.TIME_TP], rffi.TIME_T)
+c_ctime = external('ctime', [rffi.TIME_TP], rffi.CCHARP)
+c_gmtime = external('gmtime', [rffi.TIME_TP], TM_P)
+c_mktime = external('mktime', [TM_P], rffi.TIME_T)
+c_asctime = external('asctime', [TM_P], rffi.CCHARP)
+c_localtime = external('localtime', [rffi.TIME_TP], TM_P)
+if _POSIX:
+    c_tzset = external('tzset', [], lltype.Void)
+if _WIN:
+    win_eci = ExternalCompilationInfo(
+        includes = ["time.h"],
+        post_include_bits = ["RPY_EXTERN "
+                             "long pypy_get_timezone();\n"
+                             "RPY_EXTERN "
+                             "int pypy_get_daylight();\n"
+                             "RPY_EXTERN "
+                             "char** pypy_get_tzname();\n"
+                             "RPY_EXTERN "
+                             "void pypy__tzset();"],
+        separate_module_sources = ["""
+        long pypy_get_timezone() { return timezone; }
+        int pypy_get_daylight() { return daylight; }
+        char** pypy_get_tzname() { return tzname; }
+        void pypy__tzset() { _tzset(); }
+        """])
+    # Ensure sure that we use _tzset() and timezone from the same C Runtime.
+    c_tzset = external('pypy__tzset', [], lltype.Void, win_eci)
+    c_get_timezone = external('pypy_get_timezone', [], rffi.LONG, win_eci)
+    c_get_daylight = external('pypy_get_daylight', [], rffi.INT, win_eci)
+    c_get_tzname = external('pypy_get_tzname', [], rffi.CCHARPP, win_eci)
+
+c_strftime = external('strftime', [rffi.CCHARP, rffi.SIZE_T, rffi.CCHARP, TM_P],
+                      rffi.SIZE_T)
+
+def _init_accept2dyear(space):
+    if os.environ.get("PYTHONY2K"):
+        accept2dyear = 0
+    else:
+        accept2dyear = 1
+    _set_module_object(space, "accept2dyear", space.wrap(accept2dyear))
+
+def _init_timezone(space):
+    timezone = daylight = altzone = 0
+    tzname = ["", ""]
+
+    if _WIN:
+        c_tzset()
+        timezone = c_get_timezone()
+        altzone = timezone - 3600
+        daylight = c_get_daylight()
+        tzname_ptr = c_get_tzname()
+        tzname = rffi.charp2str(tzname_ptr[0]), rffi.charp2str(tzname_ptr[1])
+
+    if _POSIX:
+        if _CYGWIN:
+            YEAR = (365 * 24 + 6) * 3600
+
+            # about January 11th
+            t = (((c_time(lltype.nullptr(rffi.TIME_TP.TO))) / YEAR) * YEAR + 10 * 24 * 3600)
+            # we cannot have reference to stack variable, put it on the heap
+            t_ref = lltype.malloc(rffi.TIME_TP.TO, 1, flavor='raw')
+            t_ref[0] = rffi.cast(rffi.TIME_T, t)
+            p = c_localtime(t_ref)
+            q = c_gmtime(t_ref)
+            janzone = (p.c_tm_hour + 24 * p.c_tm_mday) - (q.c_tm_hour + 24 * q.c_tm_mday)
+            if janzone < -12:
+                janname = "   "
+            elif janzone > 14:
+                janname = "   "
+            else:
+                janname = _time_zones[janzone - 12]
+            janzone = janzone * 3600
+            # about July 11th
+            tt = t + YEAR / 2
+            t_ref[0] = rffi.cast(rffi.TIME_T, tt)
+            p = c_localtime(t_ref)
+            q = c_gmtime(t_ref)
+            julyzone = (p.c_tm_hour + 24 * p.c_tm_mday) - (q.c_tm_hour + 24 * q.c_tm_mday)
+            if julyzone < -12:
+                julyname = "   "
+            elif julyzone > 14:
+                julyname = "   "
+            else:
+                julyname = _time_zones[julyzone - 12]
+            julyzone = julyzone * 3600
+            lltype.free(t_ref, flavor='raw')
+
+            if janzone < julyzone:
+                # DST is reversed in the southern hemisphere
+                timezone = julyzone
+                altzone = janzone
+                daylight = int(janzone != julyzone)
+                tzname = [julyname, janname]
+            else:
+                timezone = janzone
+                altzone = julyzone
+                daylight = int(janzone != julyzone)
+                tzname = [janname, julyname]
+
+        else:
+            YEAR = (365 * 24 + 6) * 3600
+
+            t = (((c_time(lltype.nullptr(rffi.TIME_TP.TO))) / YEAR) * YEAR)
+            # we cannot have reference to stack variable, put it on the heap
+            t_ref = lltype.malloc(rffi.TIME_TP.TO, 1, flavor='raw')
+            t_ref[0] = rffi.cast(rffi.TIME_T, t)
+            p = c_localtime(t_ref)
+            janzone = -p.c_tm_gmtoff
+            tm_zone = rffi.charp2str(p.c_tm_zone)
+            janname = ["   ", tm_zone][bool(tm_zone)]
+            tt = t + YEAR / 2
+            t_ref[0] = rffi.cast(rffi.TIME_T, tt)
+            p = c_localtime(t_ref)
+            lltype.free(t_ref, flavor='raw')
+            tm_zone = rffi.charp2str(p.c_tm_zone)
+            julyzone = -p.c_tm_gmtoff
+            julyname = ["   ", tm_zone][bool(tm_zone)]
+
+            if janzone < julyzone:
+                # DST is reversed in the southern hemisphere
+                timezone = julyzone
+                altzone = janzone
+                daylight = int(janzone != julyzone)
+                tzname = [julyname, janname]
+            else:
+                timezone = janzone
+                altzone = julyzone
+                daylight = int(janzone != julyzone)
+                tzname = [janname, julyname]
+
+    _set_module_object(space, "timezone", space.wrap(timezone))
+    _set_module_object(space, 'daylight', space.wrap(daylight))
+    tzname_w = [space.wrap(tzname[0]), space.wrap(tzname[1])]
+    _set_module_object(space, 'tzname', space.newtuple(tzname_w))
+    _set_module_object(space, 'altzone', space.wrap(altzone))
+
+def _get_error_msg():
+    errno = rposix.get_errno()
+    return os.strerror(errno)
+
+if sys.platform != 'win32':
+    @unwrap_spec(secs=float)
+    def sleep(space, secs):
+        if secs < 0:
+            raise OperationError(space.w_IOError,
+                                 space.wrap("Invalid argument: negative time in sleep"))
+        pytime.sleep(secs)
+else:
+    from rpython.rlib import rwin32
+    from errno import EINTR
+    def _simple_sleep(space, secs, interruptible):
+        if secs == 0.0 or not interruptible:
+            pytime.sleep(secs)
+        else:
+            millisecs = int(secs * 1000)
+            interrupt_event = space.fromcache(State).get_interrupt_event()
+            rwin32.ResetEvent(interrupt_event)
+            rc = rwin32.WaitForSingleObject(interrupt_event, millisecs)
+            if rc == rwin32.WAIT_OBJECT_0:
+                # Yield to make sure real Python signal handler
+                # called.
+                pytime.sleep(0.001)
+                raise wrap_oserror(space,
+                                   OSError(EINTR, "sleep() interrupted"))
+    @unwrap_spec(secs=float)
+    def sleep(space, secs):
+        if secs < 0:
+            raise OperationError(space.w_IOError,
+                                 space.wrap("Invalid argument: negative time in sleep"))
+        # as decreed by Guido, only the main thread can be
+        # interrupted.
+        main_thread = space.fromcache(State).main_thread
+        interruptible = (main_thread == thread.get_ident())
+        MAX = sys.maxint / 1000.0 # > 24 days
+        while secs > MAX:
+            _simple_sleep(space, MAX, interruptible)
+            secs -= MAX
+        _simple_sleep(space, secs, interruptible)
+
+def _get_module_object(space, obj_name):
+    w_module = space.getbuiltinmodule('time')
+    w_obj = space.getattr(w_module, space.wrap(obj_name))
+    return w_obj
+
+def _set_module_object(space, obj_name, w_obj_value):
+    w_module = space.getbuiltinmodule('time')
+    space.setattr(w_module, space.wrap(obj_name), w_obj_value)
+
+def _get_inttime(space, w_seconds):
+    # w_seconds can be a wrapped None (it will be automatically wrapped
+    # in the callers, so we never get a real None here).
+    if space.is_none(w_seconds):
+        seconds = pytime.time()
+    else:
+        seconds = space.float_w(w_seconds)
+    #
+    t = rffi.cast(rffi.TIME_T, seconds)
+    #
+    # Logic from CPython: How much info did we lose?  We assume that
+    # time_t is an integral type.  If we lost a second or more, the
+    # input doesn't fit in a time_t; call it an error.
+    diff = seconds - rffi.cast(lltype.Float, t)
+    if diff <= -1.0 or diff >= 1.0:
+        raise OperationError(space.w_ValueError,
+                      space.wrap("timestamp out of range for platform time_t"))
+    return t
+
+def _tm_to_tuple(space, t):
+    time_tuple = [
+        space.wrap(rffi.getintfield(t, 'c_tm_year') + 1900),
+        space.wrap(rffi.getintfield(t, 'c_tm_mon') + 1), # want january == 1
+        space.wrap(rffi.getintfield(t, 'c_tm_mday')),
+        space.wrap(rffi.getintfield(t, 'c_tm_hour')),
+        space.wrap(rffi.getintfield(t, 'c_tm_min')),
+        space.wrap(rffi.getintfield(t, 'c_tm_sec')),
+        space.wrap((rffi.getintfield(t, 'c_tm_wday') + 6) % 7), # want monday == 0
+        space.wrap(rffi.getintfield(t, 'c_tm_yday') + 1), # want january, 1 == 1
+        space.wrap(rffi.getintfield(t, 'c_tm_isdst'))]
+
+    w_struct_time = _get_module_object(space, 'struct_time')
+    w_time_tuple = space.newtuple(time_tuple)
+    return space.call_function(w_struct_time, w_time_tuple)
+
+def _gettmarg(space, w_tup, allowNone=True):
+    if space.is_none(w_tup):
+        if not allowNone:
+            raise OperationError(space.w_TypeError,
+                                 space.wrap("tuple expected"))
+        # default to the current local time
+        tt = rffi.r_time_t(int(pytime.time()))
+        t_ref = lltype.malloc(rffi.TIME_TP.TO, 1, flavor='raw')
+        t_ref[0] = tt
+        pbuf = c_localtime(t_ref)
+        lltype.free(t_ref, flavor='raw')
+        if not pbuf:
+            raise OperationError(space.w_ValueError,
+                space.wrap(_get_error_msg()))
+        return pbuf
+
+    tup_w = space.fixedview(w_tup)
+    if len(tup_w) != 9:
+        raise oefmt(space.w_TypeError,
+                    "argument must be sequence of length 9, not %d",
+                    len(tup_w))
+
+    y = space.int_w(tup_w[0])
+    tm_mon = space.int_w(tup_w[1])
+    if tm_mon == 0:
+        tm_mon = 1
+    tm_mday = space.int_w(tup_w[2])
+    if tm_mday == 0:
+        tm_mday = 1
+    tm_yday = space.int_w(tup_w[7])
+    if tm_yday == 0:
+        tm_yday = 1
+    rffi.setintfield(glob_buf, 'c_tm_mon', tm_mon)
+    rffi.setintfield(glob_buf, 'c_tm_mday', tm_mday)
+    rffi.setintfield(glob_buf, 'c_tm_hour', space.int_w(tup_w[3]))
+    rffi.setintfield(glob_buf, 'c_tm_min', space.int_w(tup_w[4]))
+    rffi.setintfield(glob_buf, 'c_tm_sec', space.int_w(tup_w[5]))
+    rffi.setintfield(glob_buf, 'c_tm_wday', space.int_w(tup_w[6]))
+    rffi.setintfield(glob_buf, 'c_tm_yday', tm_yday)
+    rffi.setintfield(glob_buf, 'c_tm_isdst', space.int_w(tup_w[8]))
+    if _POSIX:
+        if _CYGWIN:
+            pass
+        else:
+            # actually never happens, but makes annotator happy
+            glob_buf.c_tm_zone = lltype.nullptr(rffi.CCHARP.TO)
+            rffi.setintfield(glob_buf, 'c_tm_gmtoff', 0)
+
+    if y < 1900:
+        w_accept2dyear = _get_module_object(space, "accept2dyear")
+        accept2dyear = space.int_w(w_accept2dyear)
+
+        if not accept2dyear:
+            raise OperationError(space.w_ValueError,
+                space.wrap("year >= 1900 required"))
+
+        if 69 <= y <= 99:
+            y += 1900
+        elif 0 <= y <= 68:
+            y += 2000
+        else:
+            raise OperationError(space.w_ValueError,
+                space.wrap("year out of range"))
+
+    # tm_wday does not need checking of its upper-bound since taking "%
+    #  7" in gettmarg() automatically restricts the range.
+    if rffi.getintfield(glob_buf, 'c_tm_wday') < -1:
+        raise OperationError(space.w_ValueError,
+                             space.wrap("day of week out of range"))
+
+    rffi.setintfield(glob_buf, 'c_tm_year', y - 1900)
+    rffi.setintfield(glob_buf, 'c_tm_mon',
+                     rffi.getintfield(glob_buf, 'c_tm_mon') - 1)
+    rffi.setintfield(glob_buf, 'c_tm_wday',
+                     (rffi.getintfield(glob_buf, 'c_tm_wday') + 1) % 7)
+    rffi.setintfield(glob_buf, 'c_tm_yday',
+                     rffi.getintfield(glob_buf, 'c_tm_yday') - 1)
+

From noreply at buildbot.pypy.org  Wed Nov 12 01:32:22 2014
From: noreply at buildbot.pypy.org (rlamy)
Date: Wed, 12 Nov 2014 01:32:22 +0100 (CET)
Subject: [pypy-commit] pypy expressions: Clean up V_Type code, add tests
Message-ID: <20141112003222.84DD71C07BB@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy 
Branch: expressions
Changeset: r74469:194c5a7dbe21
Date: 2014-11-12 00:32 +0000
http://bitbucket.org/pypy/pypy/changeset/194c5a7dbe21/

Log:	Clean up V_Type code, add tests

diff --git a/rpython/flowspace/expression.py b/rpython/flowspace/expression.py
--- a/rpython/flowspace/expression.py
+++ b/rpython/flowspace/expression.py
@@ -1,9 +1,8 @@
 from rpython.flowspace.model import Variable
-from rpython.flowspace.operation import op
-from rpython.annotator.model import SomeType
 
 class V_Type(Variable):
     def __init__(self, v_obj):
+        from rpython.annotator.model import SomeType
         Variable.__init__(self)
         self.arg = v_obj
         s = SomeType()
@@ -11,11 +10,18 @@
         self.annotation = s
 
     def as_operation(self):
+        from rpython.flowspace.operation import op
         return op.type(self.arg)
 
     def __eq__(self, other):
         return isinstance(other, V_Type) and other.arg == self.arg
 
+    def __hash__(self):
+        return hash((type(self), self.arg))
+
+    def __repr__(self):
+        return 'type(%s)' % self.arg
+
     def replace(self, mapping):
         if self.arg in mapping:
             return V_Type(mapping[self.arg])
diff --git a/rpython/flowspace/generator.py b/rpython/flowspace/generator.py
--- a/rpython/flowspace/generator.py
+++ b/rpython/flowspace/generator.py
@@ -5,6 +5,7 @@
 from rpython.flowspace.pygraph import PyGraph
 from rpython.flowspace.model import (Block, Link, Variable,
     Constant, checkgraph, const)
+from rpython.flowspace.expression import V_Type
 from rpython.flowspace.operation import op
 from rpython.translator.unsimplify import insert_empty_startblock, split_block
 from rpython.translator.simplify import eliminate_empty_blocks, simplify_graph
@@ -106,7 +107,6 @@
 def tweak_generator_body_graph(Entry, graph):
     # First, always run simplify_graph in order to reduce the number of
     # variables passed around
-    from rpython.flowspace.expression import V_Type
     simplify_graph(graph)
     insert_empty_startblock(None, graph)
     _insert_reads(graph.startblock, Entry.varnames)
diff --git a/rpython/flowspace/operation.py b/rpython/flowspace/operation.py
--- a/rpython/flowspace/operation.py
+++ b/rpython/flowspace/operation.py
@@ -12,6 +12,7 @@
 from rpython.tool.sourcetools import compile2
 from rpython.flowspace.model import (Constant, WrapException, const, Variable,
                                      SpaceOperation)
+from rpython.flowspace.expression import V_Type
 from rpython.flowspace.specialcase import register_flow_sc
 from rpython.annotator.model import (
     SomeTuple, AnnotatorError, read_can_only_throw)
@@ -453,7 +454,6 @@
         if result is not None:
             return result
         ctx.merge_point()
-        from rpython.flowspace.expression import V_Type
         v_instance, = self.args
         return V_Type(v_instance)
 
diff --git a/rpython/flowspace/test/test_expression.py b/rpython/flowspace/test/test_expression.py
new file mode 100644
--- /dev/null
+++ b/rpython/flowspace/test/test_expression.py
@@ -0,0 +1,15 @@
+from rpython.flowspace.model import Variable, const
+from rpython.flowspace.expression import V_Type
+
+def test_type():
+    v1, v2 = Variable(), Variable()
+    assert V_Type(v1) == V_Type(v1)
+    assert V_Type(v1) != V_Type(v2)
+    assert hash(V_Type(v1)) == hash(V_Type(v1))
+    assert hash(V_Type(v1)) != hash(V_Type(v2))
+
+def test_type_replace():
+    v1, v2 = Variable(), Variable()
+    assert V_Type(v1).replace({v1: v2}) == V_Type(v2)
+    assert V_Type(const(1)).replace({v1: v2}) == V_Type(const(1))
+    assert V_Type(v1).replace({v2: v1}) == V_Type(v1)

From noreply at buildbot.pypy.org  Wed Nov 12 08:45:52 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Wed, 12 Nov 2014 08:45:52 +0100 (CET)
Subject: [pypy-commit] stmgc hashtable: Add a reference to a paper
Message-ID: <20141112074552.91A091C0548@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: hashtable
Changeset: r1504:38ce55475572
Date: 2014-11-12 08:46 +0100
http://bitbucket.org/pypy/stmgc/changeset/38ce55475572/

Log:	Add a reference to a paper

diff --git a/c7/stm/hashtable.c b/c7/stm/hashtable.c
--- a/c7/stm/hashtable.c
+++ b/c7/stm/hashtable.c
@@ -22,6 +22,11 @@
 itself, once created, stays around.  It is only removed by the next
 major GC if it points to NULL and its read/write markers are not set
 in any currently-running transaction.
+
+References
+----------
+
+Inspired by: http://ppl.stanford.edu/papers/podc011-bronson.pdf
 */
 
 

From noreply at buildbot.pypy.org  Wed Nov 12 10:48:17 2014
From: noreply at buildbot.pypy.org (ltratt)
Date: Wed, 12 Nov 2014 10:48:17 +0100 (CET)
Subject: [pypy-commit] pypy recursion_and_inlining: Stop tracing when
	inlining is detected.
Message-ID: <20141112094817.143AA1C3641@cobra.cs.uni-duesseldorf.de>

Author: Laurence Tratt 
Branch: recursion_and_inlining
Changeset: r74470:b60064f55316
Date: 2014-11-11 15:54 +0000
http://bitbucket.org/pypy/pypy/changeset/b60064f55316/

Log:	Stop tracing when inlining is detected.

	Currently, if a function called part way through a trace turns out
	to be recursively, it is endlessly inlined, often leading to a
	tracing abort. This is tremendously inefficient, and many seemingly
	innocent recursive functions have extremely bad performance
	characteristics.

	This patch is the first step in trying to address this. Notice that
	this breaks a few tests, in the sense that it changes what is
	traced. It should have no visible effect on end behaviour.

diff --git a/rpython/jit/metainterp/pyjitpl.py b/rpython/jit/metainterp/pyjitpl.py
--- a/rpython/jit/metainterp/pyjitpl.py
+++ b/rpython/jit/metainterp/pyjitpl.py
@@ -951,9 +951,31 @@
         assembler_call = False
         if warmrunnerstate.inlining:
             if warmrunnerstate.can_inline_callable(greenboxes):
+                # We've found a potentially inlinable function; now we need to
+                # see if it's already on the stack. In other words: are we about
+                # to enter recursion? If so, we don't want to inline the
+                # recursion, which would be equivalent to unrolling a while
+                # loop.
                 portal_code = targetjitdriver_sd.mainjitcode
-                return self.metainterp.perform_call(portal_code, allboxes,
-                                                    greenkey=greenboxes)
+                inline = True
+                if self.metainterp.is_main_jitcode(portal_code):
+                    for gk, _ in self.metainterp.portal_trace_positions:
+                        if gk is None:
+                            continue
+                        assert len(gk) == len(greenboxes)
+                        i = 0
+                        for i in range(len(gk)):
+                            if not gk[i].same_constant(greenboxes[i]):
+                                break
+                        else:
+                            # The greenkey of a trace position on the stack
+                            # matches what we have, which means we're definitely
+                            # about to recurse.
+                            inline = False
+                            break
+                if inline:
+                    return self.metainterp.perform_call(portal_code, allboxes,
+                                greenkey=greenboxes)
             assembler_call = True
             # verify that we have all green args, needed to make sure
             # that assembler that we call is still correct

From noreply at buildbot.pypy.org  Wed Nov 12 10:48:18 2014
From: noreply at buildbot.pypy.org (ltratt)
Date: Wed, 12 Nov 2014 10:48:18 +0100 (CET)
Subject: [pypy-commit] pypy recursion_and_inlining: We only need
	can_inline_callable.
Message-ID: <20141112094818.469341C3641@cobra.cs.uni-duesseldorf.de>

Author: Laurence Tratt 
Branch: recursion_and_inlining
Changeset: r74471:45bc61aafcd3
Date: 2014-11-11 16:02 +0000
http://bitbucket.org/pypy/pypy/changeset/45bc61aafcd3/

Log:	We only need can_inline_callable.

diff --git a/rpython/jit/metainterp/warmstate.py b/rpython/jit/metainterp/warmstate.py
--- a/rpython/jit/metainterp/warmstate.py
+++ b/rpython/jit/metainterp/warmstate.py
@@ -567,17 +567,14 @@
         jd = self.jitdriver_sd
         cpu = self.cpu
 
-        def can_inline_greenargs(*greenargs):
+        def can_inline_callable(greenkey):
+            greenargs = unwrap_greenkey(greenkey)
             if can_never_inline(*greenargs):
                 return False
             cell = JitCell.get_jitcell(*greenargs)
             if cell is not None and (cell.flags & JC_DONT_TRACE_HERE) != 0:
                 return False
             return True
-        def can_inline_callable(greenkey):
-            greenargs = unwrap_greenkey(greenkey)
-            return can_inline_greenargs(*greenargs)
-        self.can_inline_greenargs = can_inline_greenargs
         self.can_inline_callable = can_inline_callable
 
         if jd._should_unroll_one_iteration_ptr is None:

From noreply at buildbot.pypy.org  Wed Nov 12 10:48:19 2014
From: noreply at buildbot.pypy.org (ltratt)
Date: Wed, 12 Nov 2014 10:48:19 +0100 (CET)
Subject: [pypy-commit] pypy recursion_and_inlining: Force recursive
 functions to be (separately) traced sooner.
Message-ID: <20141112094819.8B4041C3641@cobra.cs.uni-duesseldorf.de>

Author: Laurence Tratt 
Branch: recursion_and_inlining
Changeset: r74472:ebc86588479f
Date: 2014-11-12 09:44 +0000
http://bitbucket.org/pypy/pypy/changeset/ebc86588479f/

Log:	Force recursive functions to be (separately) traced sooner.

	As soon as we've identified a recursive function, we know we don't
	to inline it into other functions. Instead, we want to have it
	traced separately. This patch simply uses the same mechanism as
	aborted traces to achieve this.

diff --git a/rpython/jit/metainterp/pyjitpl.py b/rpython/jit/metainterp/pyjitpl.py
--- a/rpython/jit/metainterp/pyjitpl.py
+++ b/rpython/jit/metainterp/pyjitpl.py
@@ -971,6 +971,7 @@
                             # The greenkey of a trace position on the stack
                             # matches what we have, which means we're definitely
                             # about to recurse.
+                            warmrunnerstate.dont_trace_here(greenboxes)
                             inline = False
                             break
                 if inline:
diff --git a/rpython/jit/metainterp/warmstate.py b/rpython/jit/metainterp/warmstate.py
--- a/rpython/jit/metainterp/warmstate.py
+++ b/rpython/jit/metainterp/warmstate.py
@@ -577,6 +577,16 @@
             return True
         self.can_inline_callable = can_inline_callable
 
+        def dont_trace_here(greenkey):
+            # Set greenkey as somewhere that tracing should not occur into;
+            # notice that, as per the description of JC_DONT_TRACE_HERE earlier,
+            # if greenkey hasn't been traced separately, setting
+            # JC_DONT_TRACE_HERE will force tracing the next time the function
+            # is encountered.
+            cell = JitCell.ensure_jit_cell_at_key(greenkey)
+            cell.flags |= JC_DONT_TRACE_HERE
+        self.dont_trace_here = dont_trace_here
+
         if jd._should_unroll_one_iteration_ptr is None:
             def should_unroll_one_iteration(greenkey):
                 return False

From noreply at buildbot.pypy.org  Wed Nov 12 11:11:14 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Wed, 12 Nov 2014 11:11:14 +0100 (CET)
Subject: [pypy-commit] pypy stmgc-c7: Re-add a line that was lost during
 merge (this line is the diff
Message-ID: <20141112101114.5254A1C3A61@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: stmgc-c7
Changeset: r74473:d327a0f4d6ba
Date: 2014-11-12 11:10 +0100
http://bitbucket.org/pypy/pypy/changeset/d327a0f4d6ba/

Log:	Re-add a line that was lost during merge (this line is the diff done
	to rtyper/lltypesystem/rclass.py, which was deleted)

diff --git a/rpython/rtyper/rclass.py b/rpython/rtyper/rclass.py
--- a/rpython/rtyper/rclass.py
+++ b/rpython/rtyper/rclass.py
@@ -523,6 +523,7 @@
             if hints is None:
                 hints = {}
             hints = self._check_for_immutable_hints(hints)
+            hints = self._check_for_stm_hints(hints)
             kwds = {}
             if self.gcflavor == 'gc':
                 kwds['rtti'] = True

From noreply at buildbot.pypy.org  Wed Nov 12 11:24:32 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Wed, 12 Nov 2014 11:24:32 +0100 (CET)
Subject: [pypy-commit] pypy stmgc-c7: With STM,
 lightweight finalizers must not contain any setfield.
Message-ID: <20141112102432.CCB9F1D288C@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: stmgc-c7
Changeset: r74474:0c90b2da771b
Date: 2014-11-12 11:24 +0100
http://bitbucket.org/pypy/pypy/changeset/0c90b2da771b/

Log:	With STM, lightweight finalizers must not contain any setfield.

diff --git a/rpython/translator/backendopt/finalizer.py b/rpython/translator/backendopt/finalizer.py
--- a/rpython/translator/backendopt/finalizer.py
+++ b/rpython/translator/backendopt/finalizer.py
@@ -33,11 +33,13 @@
         if (op.opname.startswith('int_') or op.opname.startswith('float_')
             or op.opname.startswith('uint_') or op.opname.startswith('cast_')):
             return self.bottom_result()
-        if op.opname == 'setfield' or op.opname == 'bare_setfield':
-            TP = op.args[2].concretetype
-            if not isinstance(TP, lltype.Ptr) or TP.TO._gckind == 'raw':
-                # primitive type
-                return self.bottom_result()
+        ## # All setfields are forbidden.  Commented out in test_finalizer too.
+        ## # This is needed for STM and could maybe be justified in general.
+        ## if op.opname == 'setfield' or op.opname == 'bare_setfield':
+        ##     TP = op.args[2].concretetype
+        ##     if not isinstance(TP, lltype.Ptr) or TP.TO._gckind == 'raw':
+        ##         # primitive type
+        ##         return self.bottom_result()
         if op.opname == 'getfield':
             TP = op.result.concretetype
             if not isinstance(TP, lltype.Ptr) or TP.TO._gckind == 'raw':
diff --git a/rpython/translator/backendopt/test/test_finalizer.py b/rpython/translator/backendopt/test/test_finalizer.py
--- a/rpython/translator/backendopt/test/test_finalizer.py
+++ b/rpython/translator/backendopt/test/test_finalizer.py
@@ -53,12 +53,12 @@
     v2 = varoftype(lltype.Signed)
     v3 = varoftype(X)
     v4 = varoftype(Z)
-    assert not f.analyze(SpaceOperation('bare_setfield', [v1, Constant('x'),
-                                                          v2], None))
+    ## assert not f.analyze(SpaceOperation('bare_setfield', [v1, Constant('x'),
+    ##                                                       v2], None))
     assert     f.analyze(SpaceOperation('bare_setfield', [v1, Constant('y'),
                                                           v3], None))
-    assert not f.analyze(SpaceOperation('bare_setfield', [v1, Constant('z'),
-                                                          v4], None))
+    ## assert not f.analyze(SpaceOperation('bare_setfield', [v1, Constant('z'),
+    ##                                                       v4], None))
 
     def test_malloc(self):
         S = lltype.GcStruct('S')

From noreply at buildbot.pypy.org  Wed Nov 12 11:29:47 2014
From: noreply at buildbot.pypy.org (Raemi)
Date: Wed, 12 Nov 2014 11:29:47 +0100 (CET)
Subject: [pypy-commit] stmgc c8-small-uniform: merge new page handling (WIP)
Message-ID: <20141112102947.90F441D288C@cobra.cs.uni-duesseldorf.de>

Author: Remi Meier 
Branch: c8-small-uniform
Changeset: r1505:f128b7906776
Date: 2014-11-12 10:31 +0100
http://bitbucket.org/pypy/stmgc/changeset/f128b7906776/

Log:	merge new page handling (WIP)

diff too long, truncating to 2000 out of 2471 lines

diff --git a/c8/PAGES b/c8/PAGES
new file mode 100644
--- /dev/null
+++ b/c8/PAGES
@@ -0,0 +1,52 @@
+Handling of pages in stmgc
+--------------------------
+
+(Proposal)
+
+Each segment corresponds to one range of virtual addresses, of NB_PAGES.
+
+There is additionally one file descriptor corresponding to a
+memory-mapped file of length "15/16 * NB_PAGES".  In each segment, the
+final 15/16th of it is normally remapped with MAP_SHARED to correspond
+to this file.
+
+For each page of the file and each segment, we store two status bits
+that give one of three possible states: the NO_ACCESS state means that
+the corresponding page was mprotected to PROT_NONE; the SHARED state
+means the page is mapped to the file; the PRIVATE state means the page
+was remapped to an anonymous MAP_PRIVATE page.
+
+When a segment allocates new objects (out of its nursery), they go into
+pages that are initially set to SHARED in this segment, and NO_ACCESS in
+all other segments.
+
+When we try to read the same object from another segment, we get a
+segfault.  In the segment that receives the segfault, the page is
+NO_ACCESS.  At that point, two cases: either we can use the same data,
+or we can't.  We can if the data in the shared page is the unmodified
+data from the current revision (or from a later revision, in which case
+we just update to this later revision).  We can't if the shared data
+comes from a past revision or if it contains currently-modified data.
+
+If we can: we mprotect our segment's page back to SHARED.
+
+If we can't: we remap the page to PRIVATE.
+
+Finally, here is the write barrier logic.  When we're about to write to
+a page in our segment: first, we make sure it's not a NO_ACCESS page (by
+forcing a segfault to occur, I suppose).  Then, if it's a PRIVATE page,
+nothing to do; but if it's a SHARED page, we first check the other
+segments.  If none of them has also the page in the SHARED status (all
+have NO_ACCESS or PRIVATE), then we don't do anything either.  Only if
+there is a different segment that also has the page SHARED do we need
+more care.  There are two solutions in this case:
+
+1. We can change our page to PRIVATE.
+
+2. We can change the other segment's pages to NO_ACCESS.
+
+Which solution to choose in which case is unclear so far.
+
+The end result is that we end with as few PRIVATE pages as reasonably
+possible: it occurs only if one segment has currently changes in the
+page *and* a different segment is currently viewing the same data.
diff --git a/c8/stm/core.c b/c8/stm/core.c
--- a/c8/stm/core.c
+++ b/c8/stm/core.c
@@ -2,8 +2,6 @@
 # error "must be compiled via stmgc.c"
 #endif
 
-#include 
-
 
 #ifdef NDEBUG
 #define EVENTUALLY(condition)    {/* nothing */}
@@ -19,231 +17,440 @@
     }
 #endif
 
+/* General helper: copies objects into our own segment, from some
+   source described by a range of 'struct stm_undo_s'.  Maybe later
+   we could specialize this function to avoid the checks in the
+   inner loop.
+*/
+static void import_objects(
+        int from_segnum,            /* or -1: from undo->backup,
+                                       or -2: from undo->backup if not modified */
+        uintptr_t pagenum,          /* or -1: "all accessible" */
+        struct stm_undo_s *undo,
+        struct stm_undo_s *end)
+{
+    char *src_segment_base = (from_segnum >= 0 ? get_segment_base(from_segnum)
+                                               : NULL);
+
+    assert(IMPLY(from_segnum >= 0, get_priv_segment(from_segnum)->modification_lock));
+    assert(STM_PSEGMENT->modification_lock);
+
+    for (; undo < end; undo++) {
+        object_t *obj = undo->object;
+        stm_char *oslice = ((stm_char *)obj) + SLICE_OFFSET(undo->slice);
+        uintptr_t current_page_num = ((uintptr_t)oslice) / 4096;
+
+        if (pagenum == -1) {
+            if (get_page_status_in(STM_SEGMENT->segment_num,
+                                   current_page_num) == PAGE_NO_ACCESS)
+                continue;
+        }
+        else {
+            if (current_page_num != pagenum)
+                continue;
+        }
+
+        if (from_segnum == -2 && _stm_was_read(obj) && (obj->stm_flags & GCFLAG_WB_EXECUTED)) {
+            /* called from stm_validate():
+                > if not was_read(), we certainly didn't modify
+                > if not WB_EXECUTED, we may have read from the obj in a different page but
+                  did not modify it (should not occur right now, but future proof!)
+               only the WB_EXECUTED alone is not enough, since we may have imported from a
+               segment's private page (which had the flag set) */
+            assert(IMPLY(_stm_was_read(obj), (obj->stm_flags & GCFLAG_WB_EXECUTED))); /* for now */
+            continue;           /* only copy unmodified */
+        }
+
+        dprintf(("import slice seg=%d obj=%p off=%lu sz=%d pg=%lu\n",
+                 from_segnum, obj, SLICE_OFFSET(undo->slice),
+                 SLICE_SIZE(undo->slice), current_page_num));
+        char *src, *dst;
+        if (src_segment_base != NULL)
+            src = REAL_ADDRESS(src_segment_base, oslice);
+        else
+            src = undo->backup + SLICE_OFFSET(undo->slice);
+        dst = REAL_ADDRESS(STM_SEGMENT->segment_base, oslice);
+        memcpy(dst, src, SLICE_SIZE(undo->slice));
+
+        if (src_segment_base == NULL) {
+            /* backups never should have WB_EXECUTED */
+            assert(!(obj->stm_flags & GCFLAG_WB_EXECUTED));
+        }
+    }
+}
+
+
+/* ############# signal handler ############# */
+
+static void copy_bk_objs_in_page_from(int from_segnum, uintptr_t pagenum,
+                                      bool only_if_not_modified)
+{
+    /* looks at all bk copies of objects overlapping page 'pagenum' and
+       copies the part in 'pagenum' back to the current segment */
+    dprintf(("copy_bk_objs_in_page_from(%d, %ld, %d)\n",
+             from_segnum, (long)pagenum, only_if_not_modified));
+
+    struct list_s *list = get_priv_segment(from_segnum)->modified_old_objects;
+    struct stm_undo_s *undo = (struct stm_undo_s *)list->items;
+    struct stm_undo_s *end = (struct stm_undo_s *)(list->items + list->count);
+
+    import_objects(only_if_not_modified ? -2 : -1,
+                   pagenum, undo, end);
+}
+
+static void go_to_the_past(uintptr_t pagenum,
+                           struct stm_commit_log_entry_s *from,
+                           struct stm_commit_log_entry_s *to)
+{
+    assert(STM_PSEGMENT->modification_lock);
+    assert(from->rev_num >= to->rev_num);
+    /* walk BACKWARDS the commit log and update the page 'pagenum',
+       initially at revision 'from', until we reach the revision 'to'. */
+
+    /* XXXXXXX Recursive algo for now, fix this! */
+    if (from != to) {
+        struct stm_commit_log_entry_s *cl = to->next;
+        go_to_the_past(pagenum, from, cl);
+
+        struct stm_undo_s *undo = cl->written;
+        struct stm_undo_s *end = cl->written + cl->written_count;
+
+        import_objects(-1, pagenum, undo, end);
+    }
+}
+
+
+
+static void handle_segfault_in_page(uintptr_t pagenum)
+{
+    /* assumes page 'pagenum' is ACCESS_NONE, privatizes it,
+       and validates to newest revision */
+
+    dprintf(("handle_segfault_in_page(%lu), seg %d\n", pagenum, STM_SEGMENT->segment_num));
+
+    /* XXX: bad, but no deadlocks: */
+    acquire_all_privatization_locks();
+
+    long i;
+    int my_segnum = STM_SEGMENT->segment_num;
+
+    assert(get_page_status_in(my_segnum, pagenum) == PAGE_NO_ACCESS);
+
+    /* find who has the most recent revision of our page */
+    int shared_page_holder = -1;
+    int shared_ref_count = 0;
+    int copy_from_segnum = -1;
+    uint64_t most_recent_rev = 0;
+    for (i = 0; i < NB_SEGMENTS; i++) {
+        if (i == my_segnum)
+            continue;
+
+        if (get_page_status_in(i, pagenum) == PAGE_SHARED) {
+            /* mostly for debugging now: */
+            shared_page_holder = i;
+            shared_ref_count++;
+        }
+
+        struct stm_commit_log_entry_s *log_entry;
+        log_entry = get_priv_segment(i)->last_commit_log_entry;
+        if (get_page_status_in(i, pagenum) != PAGE_NO_ACCESS
+            && (copy_from_segnum == -1 || log_entry->rev_num > most_recent_rev)) {
+            copy_from_segnum = i;
+            most_recent_rev = log_entry->rev_num;
+        }
+    }
+    OPT_ASSERT(shared_page_holder != -1);
+    OPT_ASSERT(copy_from_segnum != -1 && copy_from_segnum != my_segnum);
+
+    /* XXX: for now, we don't try to get the single shared page. We simply
+       regard it as private for its holder. */
+    /* this assert should be true for now... */
+    assert(shared_ref_count == 1);
+
+    /* make our page private */
+    page_privatize_in(my_segnum, pagenum);
+    assert(get_page_status_in(my_segnum, pagenum) == PAGE_PRIVATE);
+
+    /* before copying anything, acquire modification locks from our and
+       the other segment */
+    uint64_t to_lock = (1UL << copy_from_segnum)| (1UL << my_segnum);
+    acquire_modification_lock_set(to_lock);
+    pagecopy((char*)(get_virt_page_of(my_segnum, pagenum) * 4096UL),
+             (char*)(get_virt_page_of(copy_from_segnum, pagenum) * 4096UL));
+
+    /* if there were modifications in the page, revert them. */
+    copy_bk_objs_in_page_from(copy_from_segnum, pagenum, false);
+
+    /* we need to go from 'src_version' to 'target_version'.  This
+       might need a walk into the past. */
+    struct stm_commit_log_entry_s *src_version, *target_version;
+    src_version = get_priv_segment(copy_from_segnum)->last_commit_log_entry;
+    target_version = STM_PSEGMENT->last_commit_log_entry;
+
+
+    dprintf(("handle_segfault_in_page: rev %lu to rev %lu\n",
+             src_version->rev_num, target_version->rev_num));
+    /* adapt revision of page to our revision:
+       if our rev is higher than the page we copy from, everything
+       is fine as we never read/modified the page anyway
+     */
+    if (src_version->rev_num > target_version->rev_num)
+        go_to_the_past(pagenum, src_version, target_version);
+
+    release_modification_lock_set(to_lock);
+    release_all_privatization_locks();
+}
+
+static void _signal_handler(int sig, siginfo_t *siginfo, void *context)
+{
+    int saved_errno = errno;
+    char *addr = siginfo->si_addr;
+    dprintf(("si_addr: %p\n", addr));
+    if (addr == NULL || addr < stm_object_pages ||
+        addr >= stm_object_pages+TOTAL_MEMORY) {
+        /* actual segfault, unrelated to stmgc */
+        fprintf(stderr, "Segmentation fault: accessing %p\n", addr);
+        abort();
+    }
+
+    int segnum = get_segment_of_linear_address(addr);
+    if (segnum != STM_SEGMENT->segment_num) {
+        fprintf(stderr, "Segmentation fault: accessing %p (seg %d) from"
+                        " seg %d\n", addr, STM_SEGMENT->segment_num, segnum);
+        abort();
+    }
+    dprintf(("-> segment: %d\n", segnum));
+
+    char *seg_base = STM_SEGMENT->segment_base;
+    uintptr_t pagenum = ((char*)addr - seg_base) / 4096UL;
+    if (pagenum < END_NURSERY_PAGE) {
+        fprintf(stderr, "Segmentation fault: accessing %p (seg %d "
+                        "page %lu)\n", addr, segnum, pagenum);
+        abort();
+    }
+
+    handle_segfault_in_page(pagenum);
+
+    errno = saved_errno;
+    /* now return and retry */
+}
+
 /* ############# commit log ############# */
 
 
 void _dbg_print_commit_log()
 {
-    volatile struct stm_commit_log_entry_s *cl;
-    cl = (volatile struct stm_commit_log_entry_s *)&commit_log_root;
+    struct stm_commit_log_entry_s *cl = &commit_log_root;
 
-    fprintf(stderr, "root (%p, %d)\n", cl->next, cl->segment_num);
+    fprintf(stderr, "commit log:\n");
     while ((cl = cl->next)) {
-        if ((uintptr_t)cl == -1) {
-            fprintf(stderr, "INEVITABLE\n");
+        if (cl == INEV_RUNNING) {
+            fprintf(stderr, "  INEVITABLE\n");
             return;
         }
-        size_t i = 0;
-        fprintf(stderr, "  elem (%p, %d)\n", cl->next, cl->segment_num);
-        object_t *obj;
-        while ((obj = cl->written[i])) {
-            fprintf(stderr, "-> %p\n", obj);
-            i++;
-        };
+        fprintf(stderr, "  entry at %p: seg %d, rev %lu\n", cl, cl->segment_num, cl->rev_num);
+        struct stm_undo_s *undo = cl->written;
+        struct stm_undo_s *end = undo + cl->written_count;
+        for (; undo < end; undo++) {
+            fprintf(stderr, "    obj %p, size %d, ofs %lu: ", undo->object,
+                    SLICE_SIZE(undo->slice), SLICE_OFFSET(undo->slice));
+            /* long i; */
+            /* for (i=0; islice); i += 8) */
+            /*     fprintf(stderr, " 0x%016lx", *(long *)(undo->backup + i)); */
+            fprintf(stderr, "\n");
+        }
     }
 }
 
-static void _update_obj_from(int from_seg, object_t *obj)
+static void reset_modified_from_backup_copies(int segment_num);  /* forward */
+
+static void _stm_validate(void *free_if_abort)
 {
-    /* during validation this looks up the obj in the
-       from_seg (backup or normal) and copies the version
-       over the current segment's one */
-    size_t obj_size;
-    char *realobj = REAL_ADDRESS(STM_SEGMENT->segment_base, obj);
-    uintptr_t pagenum = (uintptr_t)obj / 4096UL;
-
-    OPT_ASSERT(!is_shared_log_page(pagenum));
-    assert(is_private_log_page_in(STM_SEGMENT->segment_num, pagenum));
-    assert(is_private_log_page_in(from_seg, pagenum));
-
-    /* look the obj up in the other segment's modified_old_objects to
-       get its backup copy: */
-    acquire_modified_objs_lock(from_seg);
-
-    wlog_t *item;
-    struct tree_s *tree = get_priv_segment(from_seg)->modified_old_objects;
-    TREE_FIND(tree, (uintptr_t)obj, item, goto not_found);
-
-    obj_size = stmcb_size_rounded_up((struct object_s*)item->val);
-    memcpy(realobj, (char*)item->val, obj_size);
-    assert(obj->stm_flags & GCFLAG_WRITE_BARRIER);
-    release_modified_objs_lock(from_seg);
-    return;
-
- not_found:
-    /* copy from page directly (obj is unmodified) */
-    obj_size = stmcb_size_rounded_up(
-        (struct object_s*)REAL_ADDRESS(get_segment_base(from_seg), obj));
-    memcpy(realobj,
-           REAL_ADDRESS(get_segment_base(from_seg), obj),
-           obj_size);
-    obj->stm_flags |= GCFLAG_WRITE_BARRIER; /* may already be gone */
-    release_modified_objs_lock(from_seg);
-}
-
-void stm_validate(void *free_if_abort)
-{
+    dprintf(("_stm_validate(%p)\n", free_if_abort));
     /* go from last known entry in commit log to the
        most current one and apply all changes done
        by other transactions. Abort if we read one of
        the committed objs. */
+    struct stm_commit_log_entry_s *first_cl = STM_PSEGMENT->last_commit_log_entry;
+    struct stm_commit_log_entry_s *next_cl, *last_cl, *cl;
+    int my_segnum = STM_SEGMENT->segment_num;
+    /* Don't check this 'cl'. This entry is already checked */
+
     if (STM_PSEGMENT->transaction_state == TS_INEVITABLE) {
-        assert((uintptr_t)STM_PSEGMENT->last_commit_log_entry->next == -1);
+        assert(first_cl->next == INEV_RUNNING);
         return;
     }
 
-    volatile struct stm_commit_log_entry_s *cl, *prev_cl;
-    cl = prev_cl = (volatile struct stm_commit_log_entry_s *)
-        STM_PSEGMENT->last_commit_log_entry;
-
-    bool needs_abort = false;
-    /* Don't check 'cl'. This entry is already checked */
-    while ((cl = cl->next)) {
-        if ((uintptr_t)cl == -1) {
-            /* there is an inevitable transaction running */
+    /* Find the set of segments we need to copy from and lock them: */
+    uint64_t segments_to_lock = 1UL << my_segnum;
+    cl = first_cl;
+    while ((next_cl = cl->next) != NULL) {
+        if (next_cl == INEV_RUNNING) {
 #if STM_TESTS
-            free(free_if_abort);
+            if (free_if_abort != (void *)-1)
+                free(free_if_abort);
             stm_abort_transaction();
 #endif
-            cl = prev_cl;
-            _stm_collectable_safe_point();
-            continue;
+            /* only validate entries up to INEV */
+            break;
         }
-        prev_cl = cl;
+        assert(next_cl->rev_num > cl->rev_num);
+        cl = next_cl;
 
-        OPT_ASSERT(cl->segment_num >= 0 && cl->segment_num < NB_SEGMENTS);
+        if (cl->written_count) {
+            segments_to_lock |= (1UL << cl->segment_num);
+        }
+    }
+    last_cl = cl;
+    acquire_modification_lock_set(segments_to_lock);
 
-        object_t *obj;
-        size_t i = 0;
-        while ((obj = cl->written[i])) {
-            _update_obj_from(cl->segment_num, obj);
 
-            if (_stm_was_read(obj)) {
-                needs_abort = true;
+    /* import objects from first_cl to last_cl: */
+    bool needs_abort = false;
+    if (first_cl != last_cl) {
+        uint64_t segment_really_copied_from = 0UL;
 
-                /* if we wrote this obj, we need to free its backup and
-                   remove it from modified_old_objects because
-                   we would otherwise overwrite the updated obj on abort */
-                acquire_modified_objs_lock(STM_SEGMENT->segment_num);
-                wlog_t *item;
-                struct tree_s *tree = STM_PSEGMENT->modified_old_objects;
-                TREE_FIND(tree, (uintptr_t)obj, item, goto not_found);
-
-                free((void*)item->val);
-                TREE_FIND_DELETE(tree, item);
-
-            not_found:
-                /* nothing todo */
-                release_modified_objs_lock(STM_SEGMENT->segment_num);
+        cl = first_cl;
+        while ((cl = cl->next) != NULL) {
+            if (!needs_abort) {
+                struct stm_undo_s *undo = cl->written;
+                struct stm_undo_s *end = cl->written + cl->written_count;
+                for (; undo < end; undo++) {
+                    if (_stm_was_read(undo->object)) {
+                        /* first reset all modified objects from the backup
+                           copies as soon as the first conflict is detected;
+                           then we will proceed below to update our segment from
+                           the old (but unmodified) version to the newer version.
+                        */
+                        reset_modified_from_backup_copies(my_segnum);
+                        needs_abort = true;
+                        break;
+                    }
+                }
             }
 
-            i++;
-        };
+            if (cl->written_count) {
+                struct stm_undo_s *undo = cl->written;
+                struct stm_undo_s *end = cl->written + cl->written_count;
 
-        /* last fully validated entry */
-        STM_PSEGMENT->last_commit_log_entry = (struct stm_commit_log_entry_s *)cl;
+                segment_really_copied_from |= (1UL << cl->segment_num);
+                import_objects(cl->segment_num, -1, undo, end);
+            }
+
+            /* last fully validated entry */
+            STM_PSEGMENT->last_commit_log_entry = cl;
+            if (cl == last_cl)
+                break;
+        }
+        assert(cl == last_cl);
+
+        OPT_ASSERT(segment_really_copied_from < (1 << NB_SEGMENTS));
+        int segnum;
+        for (segnum = 0; segnum < NB_SEGMENTS; segnum++) {
+            if (segment_really_copied_from & (1UL << segnum)) {
+                /* here we can actually have our own modified version, so
+                   make sure to only copy things that are not modified in our
+                   segment... (if we do not abort) */
+                copy_bk_objs_in_page_from(
+                    segnum, -1,     /* any page */
+                    !needs_abort);  /* if we abort, we still want to copy everything */
+            }
+        }
     }
 
+    /* done with modifications */
+    release_modification_lock_set(segments_to_lock);
+
     if (needs_abort) {
-        free(free_if_abort);
+        if (free_if_abort != (void *)-1)
+            free(free_if_abort);
+        /* pages may be inconsistent */
+
         stm_abort_transaction();
     }
 }
 
-static struct stm_commit_log_entry_s *_create_commit_log_entry()
+static struct stm_commit_log_entry_s *_create_commit_log_entry(void)
 {
     /* puts all modified_old_objects in a new commit log entry */
 
     // we don't need the privatization lock, as we are only
     // reading from modified_old_objs and nobody but us can change it
-    struct tree_s *tree = STM_PSEGMENT->modified_old_objects;
-    size_t count = tree_count(tree);
-    size_t byte_len = sizeof(struct stm_commit_log_entry_s) + (count + 1) * sizeof(object_t*);
+    struct list_s *list = STM_PSEGMENT->modified_old_objects;
+    OPT_ASSERT((list_count(list) % 3) == 0);
+    size_t count = list_count(list) / 3;
+    size_t byte_len = sizeof(struct stm_commit_log_entry_s) +
+        count * sizeof(struct stm_undo_s);
     struct stm_commit_log_entry_s *result = malloc(byte_len);
 
     result->next = NULL;
     result->segment_num = STM_SEGMENT->segment_num;
-
-    int i = 0;
-    wlog_t *item;
-    TREE_LOOP_FORWARD(tree, item); {
-        result->written[i] = (object_t*)item->addr;
-        i++;
-    } TREE_LOOP_END;
-
-    OPT_ASSERT(count == i);
-    result->written[count] = NULL;
-
+    result->rev_num = -1;       /* invalid */
+    result->written_count = count;
+    memcpy(result->written, list->items, count * sizeof(struct stm_undo_s));
     return result;
 }
 
-static void _validate_and_add_to_commit_log()
+static void _validate_and_attach(struct stm_commit_log_entry_s *new)
 {
-    struct stm_commit_log_entry_s *new;
-    volatile struct stm_commit_log_entry_s **to;
+    struct stm_commit_log_entry_s *old;
+
+    while (1) {
+        _stm_validate(/* free_if_abort =*/ new);
+
+        /* try to attach to commit log: */
+        old = STM_PSEGMENT->last_commit_log_entry;
+        if (old->next == NULL) {
+            if (new != INEV_RUNNING) /* INEVITABLE */
+                new->rev_num = old->rev_num + 1;
+
+            if (__sync_bool_compare_and_swap(&old->next, NULL, new))
+                break;   /* success! */
+        } else if (old->next == INEV_RUNNING) {
+            /* we failed because there is an INEV transaction running */
+            usleep(10);
+        }
+
+        /* check for requested safe point. otherwise an INEV transaction
+           may try to commit but cannot because of the busy-loop here. */
+        _stm_collectable_safe_point();
+    }
+}
+
+static void _validate_and_turn_inevitable(void)
+{
+    _validate_and_attach((struct stm_commit_log_entry_s *)INEV_RUNNING);
+}
+
+static void _validate_and_add_to_commit_log(void)
+{
+    struct stm_commit_log_entry_s *old, *new;
 
     new = _create_commit_log_entry();
     if (STM_PSEGMENT->transaction_state == TS_INEVITABLE) {
-        OPT_ASSERT((uintptr_t)STM_PSEGMENT->last_commit_log_entry->next == -1);
+        old = STM_PSEGMENT->last_commit_log_entry;
+        new->rev_num = old->rev_num + 1;
+        OPT_ASSERT(old->next == INEV_RUNNING);
 
-        to = &(STM_PSEGMENT->last_commit_log_entry->next);
-        bool yes = __sync_bool_compare_and_swap(to, (void*)-1, new);
+        bool yes = __sync_bool_compare_and_swap(&old->next, INEV_RUNNING, new);
         OPT_ASSERT(yes);
-        return;
+    }
+    else {
+        _validate_and_attach(new);
     }
 
-    /* regular transaction: */
-    do {
-        stm_validate(new);
-
-        /* try attaching to commit log: */
-        to = &(STM_PSEGMENT->last_commit_log_entry->next);
-    } while (!__sync_bool_compare_and_swap(to, NULL, new));
-}
-
-static void _validate_and_turn_inevitable()
-{
-    struct stm_commit_log_entry_s *new;
-    volatile struct stm_commit_log_entry_s **to;
-
-    new = (struct stm_commit_log_entry_s*)-1;
-    do {
-        stm_validate(NULL);
-
-        /* try attaching to commit log: */
-        to = &(STM_PSEGMENT->last_commit_log_entry->next);
-    } while (!__sync_bool_compare_and_swap(to, NULL, new));
+    acquire_modification_lock(STM_SEGMENT->segment_num);
+    list_clear(STM_PSEGMENT->modified_old_objects);
+    STM_PSEGMENT->last_commit_log_entry = new;
+    release_modification_lock(STM_SEGMENT->segment_num);
 }
 
 /* ############# STM ############# */
+void stm_validate()
+{
+    _stm_validate(NULL);
+}
 
-void _privatize_shared_page(uintptr_t pagenum)
-{
-    /* privatize pages of obj for our segment iff previously
-       the pages were fully shared. */
-#ifndef NDEBUG
-    long l;
-    for (l = 0; l < NB_SEGMENTS; l++) {
-        assert(get_priv_segment(l)->privatization_lock);
-    }
-#endif
-
-    uintptr_t i;
-    int my_segnum = STM_SEGMENT->segment_num;
-
-    assert(is_shared_log_page(pagenum));
-    char *src = (char*)(get_virt_page_of(0, pagenum) * 4096UL);
-
-    for (i = 1; i < NB_SEGMENTS; i++) {
-        assert(!is_private_log_page_in(i, pagenum));
-
-        page_privatize_in(i, pagenum, src);
-    }
-    set_page_private_in(0, pagenum);
-
-    OPT_ASSERT(is_private_log_page_in(my_segnum, pagenum));
-    assert(!is_shared_log_page(pagenum));
-}
 
 void _stm_write_slowpath(object_t *obj)
 {
@@ -268,50 +475,116 @@
     /* add to read set: */
     stm_read(obj);
 
-    /* create backup copy: */
+    if (obj->stm_flags & GCFLAG_WB_EXECUTED) {
+        /* already executed WB once in this transaction. do GC
+           part again: */
+        dprintf(("write_slowpath-fast(%p)\n", obj));
+        obj->stm_flags &= ~GCFLAG_WRITE_BARRIER;
+        LIST_APPEND(STM_PSEGMENT->objects_pointing_to_nursery, obj);
+        return;
+    }
+
+    /* create backup copy (this may cause several page faults
+       XXX: do backup later and maybe allow for having NO_ACCESS
+       pages around anyway (kind of card marking)): */
     struct object_s *bk_obj = malloc(obj_size);
     memcpy(bk_obj, realobj, obj_size);
+    assert(!(bk_obj->stm_flags & GCFLAG_WB_EXECUTED));
 
-    /* if there are shared pages, privatize them */
-    uintptr_t page = first_page;
+    dprintf(("write_slowpath(%p): sz=%lu, bk=%p\n", obj, obj_size, bk_obj));
+ retry:
+    /* privatize pages: */
+    /* XXX don't always acquire all locks... */
+    acquire_all_privatization_locks();
+
+    uintptr_t page;
     for (page = first_page; page <= end_page; page++) {
-        if (UNLIKELY(is_shared_log_page(page))) {
-            long i;
-            for (i = 0; i < NB_SEGMENTS; i++) {
-                acquire_privatization_lock(i);
-            }
-            if (is_shared_log_page(page))
-                _privatize_shared_page(page);
-            for (i = NB_SEGMENTS-1; i >= 0; i--) {
-                release_privatization_lock(i);
+        /* check if our page is private or we are the only shared-page holder */
+        switch (get_page_status_in(my_segnum, page)) {
+
+        case PAGE_PRIVATE:
+            continue;
+
+        case PAGE_NO_ACCESS:
+            /* happens if there is a concurrent WB between us making the backup
+               and acquiring the locks */
+            release_all_privatization_locks();
+
+            volatile char *dummy = REAL_ADDRESS(STM_SEGMENT->segment_base, page * 4096UL);
+            *dummy;            /* force segfault */
+
+            goto retry;
+
+        case PAGE_SHARED:
+            break;
+
+        default:
+            assert(0);
+        }
+        /* make sure all the others are NO_ACCESS
+           choosing to make us PRIVATE is harder because then nobody must ever
+           update the shared page in stm_validate() except if it is the sole
+           reader of it. But then we don't actually know which revision the page is at. */
+        /* XXX this is a temporary solution I suppose */
+        int i;
+        for (i = 0; i < NB_SEGMENTS; i++) {
+            if (i == my_segnum)
+                continue;
+
+            if (get_page_status_in(i, page) == PAGE_SHARED) {
+                /* xxx: unmap? */
+                set_page_status_in(i, page, PAGE_NO_ACCESS);
+                mprotect((char*)(get_virt_page_of(i, page) * 4096UL), 4096UL, PROT_NONE);
+                dprintf(("NO_ACCESS in seg %d page %lu\n", i, page));
             }
         }
     }
-    /* pages not shared anymore. but we still may have
-       only a read protected page ourselves: */
+    /* all pages are either private or we were the first to write to a shared
+       page and therefore got it as our private one */
 
-    acquire_privatization_lock(my_segnum);
-    OPT_ASSERT(is_private_log_page_in(my_segnum, first_page));
+    /* phew, now add the obj to the write-set and register the
+       backup copy. */
+    /* XXX: we should not be here at all fiddling with page status
+       if 'obj' is merely an overflow object.  FIX ME, likely by copying
+       the overflow number logic from c7. */
 
-    /* remove the WRITE_BARRIER flag */
+    acquire_modification_lock(STM_SEGMENT->segment_num);
+    uintptr_t slice_sz;
+    uintptr_t in_page_offset = (uintptr_t)obj % 4096UL;
+    uintptr_t remaining_obj_sz = obj_size;
+    for (page = first_page; page <= end_page; page++) {
+        /* XXX Maybe also use mprotect() again to mark pages of the object as read-only, and
+           only stick it into modified_old_objects page-by-page?  Maybe it's
+           possible to do card-marking that way, too. */
+        OPT_ASSERT(remaining_obj_sz);
+
+        slice_sz = remaining_obj_sz;
+        if (in_page_offset + slice_sz > 4096UL) {
+            /* not over page boundaries */
+            slice_sz = 4096UL - in_page_offset;
+        }
+
+        STM_PSEGMENT->modified_old_objects = list_append3(
+            STM_PSEGMENT->modified_old_objects,
+            (uintptr_t)obj,     /* obj */
+            (uintptr_t)bk_obj,  /* bk_addr */
+            NEW_SLICE(obj_size - remaining_obj_sz, slice_sz));
+
+        remaining_obj_sz -= slice_sz;
+        in_page_offset = (in_page_offset + slice_sz) % 4096UL; /* mostly 0 */
+    }
+    OPT_ASSERT(remaining_obj_sz == 0);
+
+    release_modification_lock(STM_SEGMENT->segment_num);
+    /* done fiddling with protection and privatization */
+    release_all_privatization_locks();
+
+    /* remove the WRITE_BARRIER flag and add WB_EXECUTED */
     obj->stm_flags &= ~GCFLAG_WRITE_BARRIER;
+    obj->stm_flags |= GCFLAG_WB_EXECUTED;
 
     /* also add it to the GC list for minor collections */
     LIST_APPEND(STM_PSEGMENT->objects_pointing_to_nursery, obj);
-
-    /* done fiddling with protection and privatization */
-    release_privatization_lock(my_segnum);
-
-    /* phew, now add the obj to the write-set and register the
-       backup copy. */
-    /* XXX: possibly slow check; try overflow objs again? */
-    if (!tree_contains(STM_PSEGMENT->modified_old_objects, (uintptr_t)obj)) {
-        acquire_modified_objs_lock(my_segnum);
-        tree_insert(STM_PSEGMENT->modified_old_objects,
-                    (uintptr_t)obj, (uintptr_t)bk_obj);
-        release_modified_objs_lock(my_segnum);
-    }
-
 }
 
 static void reset_transaction_read_version(void)
@@ -324,7 +597,7 @@
              (long)(NB_READMARKER_PAGES * 4096UL)));
     if (mmap(readmarkers, NB_READMARKER_PAGES * 4096UL,
              PROT_READ | PROT_WRITE,
-             MAP_FIXED | MAP_PAGES_FLAGS, -1, 0) != readmarkers) {
+             MAP_FIXED | MAP_SHARED | MAP_ANONYMOUS | MAP_NORESERVE, -1, 0) != readmarkers) {
         /* fall-back */
 #if STM_TESTS
         stm_fatalerror("reset_transaction_read_version: %m");
@@ -334,15 +607,24 @@
     STM_SEGMENT->transaction_read_version = 1;
 }
 
+static void reset_wb_executed_flags(void)
+{
+    struct list_s *list = STM_PSEGMENT->modified_old_objects;
+    struct stm_undo_s *undo = (struct stm_undo_s *)list->items;
+    struct stm_undo_s *end = (struct stm_undo_s *)(list->items + list->count);
+
+    for (; undo < end; undo++) {
+        object_t *obj = undo->object;
+        obj->stm_flags &= ~GCFLAG_WB_EXECUTED;
+    }
+}
+
 
 static void _stm_start_transaction(stm_thread_local_t *tl)
 {
     assert(!_stm_in_transaction(tl));
 
-  retry:
-
-    if (!acquire_thread_segment(tl))
-        goto retry;
+    while (!acquire_thread_segment(tl)) {}
     /* GS invalid before this point! */
 
     assert(STM_PSEGMENT->safe_point == SP_NO_TRANSACTION);
@@ -355,7 +637,7 @@
     STM_PSEGMENT->shadowstack_at_start_of_transaction = tl->shadowstack;
 
     enter_safe_point_if_requested();
-    dprintf(("start_transaction\n"));
+    dprintf(("> start_transaction\n"));
 
     s_mutex_unlock();
 
@@ -365,7 +647,7 @@
         reset_transaction_read_version();
     }
 
-    assert(tree_is_cleared(STM_PSEGMENT->modified_old_objects));
+    assert(list_is_empty(STM_PSEGMENT->modified_old_objects));
     assert(list_is_empty(STM_PSEGMENT->objects_pointing_to_nursery));
     assert(tree_is_cleared(STM_PSEGMENT->young_outside_nursery));
     assert(tree_is_cleared(STM_PSEGMENT->nursery_objects_shadows));
@@ -374,7 +656,7 @@
 
     check_nursery_at_transaction_start();
 
-    stm_validate(NULL);
+    stm_validate();
 }
 
 long stm_start_transaction(stm_thread_local_t *tl)
@@ -422,35 +704,42 @@
     /* cannot access STM_SEGMENT or STM_PSEGMENT from here ! */
 }
 
+static void check_all_write_barrier_flags(char *segbase, struct list_s *list)
+{
+#ifndef NDEBUG
+    struct stm_undo_s *undo = (struct stm_undo_s *)list->items;
+    struct stm_undo_s *end = (struct stm_undo_s *)(list->items + list->count);
+    for (; undo < end; undo++) {
+        object_t *obj = undo->object;
+        char *dst = REAL_ADDRESS(segbase, obj);
+        assert(((struct object_s *)dst)->stm_flags & GCFLAG_WRITE_BARRIER);
+        assert(!(((struct object_s *)dst)->stm_flags & GCFLAG_WB_EXECUTED));
+    }
+#endif
+}
+
 void stm_commit_transaction(void)
 {
     assert(!_has_mutex());
     assert(STM_PSEGMENT->safe_point == SP_RUNNING);
     assert(STM_PSEGMENT->running_pthread == pthread_self());
 
-    dprintf(("stm_commit_transaction()\n"));
+    dprintf(("> stm_commit_transaction()\n"));
     minor_collection(1);
 
+    reset_wb_executed_flags();
+
+    /* minor_collection() above should have set again all WRITE_BARRIER flags.
+       Check that again here for the objects that are about to be copied into
+       the commit log. */
+    check_all_write_barrier_flags(STM_SEGMENT->segment_base,
+                                  STM_PSEGMENT->modified_old_objects);
+
     _validate_and_add_to_commit_log();
 
-    /* clear WRITE_BARRIER flags, free all backup copies,
-       and clear the tree: */
-    acquire_modified_objs_lock(STM_SEGMENT->segment_num);
-
-    struct tree_s *tree = STM_PSEGMENT->modified_old_objects;
-    wlog_t *item;
-    TREE_LOOP_FORWARD(tree, item); {
-        object_t *obj = (object_t*)item->addr;
-        struct object_s* bk_obj = (struct object_s *)item->val;
-        free(bk_obj);
-        obj->stm_flags |= GCFLAG_WRITE_BARRIER;
-    } TREE_LOOP_END;
-    tree_clear(tree);
-
-    release_modified_objs_lock(STM_SEGMENT->segment_num);
-
     invoke_and_clear_user_callbacks(0);   /* for commit */
 
+    /* XXX do we still need a s_mutex_lock() section here? */
     s_mutex_lock();
     enter_safe_point_if_requested();
     assert(STM_SEGMENT->nursery_end == NURSERY_END);
@@ -468,35 +757,42 @@
     s_mutex_unlock();
 }
 
-void reset_modified_from_backup_copies(int segment_num)
+static void reset_modified_from_backup_copies(int segment_num)
 {
 #pragma push_macro("STM_PSEGMENT")
 #pragma push_macro("STM_SEGMENT")
 #undef STM_PSEGMENT
 #undef STM_SEGMENT
-    acquire_modified_objs_lock(segment_num);
+    assert(get_priv_segment(segment_num)->modification_lock);
 
     struct stm_priv_segment_info_s *pseg = get_priv_segment(segment_num);
-    struct tree_s *tree = pseg->modified_old_objects;
-    wlog_t *item;
-    TREE_LOOP_FORWARD(tree, item); {
-        object_t *obj = (object_t*)item->addr;
-        struct object_s* bk_obj = (struct object_s *)item->val;
-        size_t obj_size;
+    struct list_s *list = pseg->modified_old_objects;
+    struct stm_undo_s *undo = (struct stm_undo_s *)list->items;
+    struct stm_undo_s *end = (struct stm_undo_s *)(list->items + list->count);
 
-        obj_size = stmcb_size_rounded_up(bk_obj);
+    for (; undo < end; undo++) {
+        object_t *obj = undo->object;
+        char *dst = REAL_ADDRESS(pseg->pub.segment_base, obj);
 
-        memcpy(REAL_ADDRESS(pseg->pub.segment_base, obj),
-               bk_obj, obj_size);
-        assert(obj->stm_flags & GCFLAG_WRITE_BARRIER); /* not written */
+        memcpy(dst + SLICE_OFFSET(undo->slice),
+               undo->backup + SLICE_OFFSET(undo->slice),
+               SLICE_SIZE(undo->slice));
 
-        free(bk_obj);
-    } TREE_LOOP_END;
+        size_t obj_size = stmcb_size_rounded_up((struct object_s*)undo->backup);
+        dprintf(("reset_modified_from_backup_copies(%d): obj=%p off=%lu bk=%p obj_sz=%lu\n",
+                 segment_num, obj, SLICE_OFFSET(undo->slice), undo->backup, obj_size));
 
-    tree_clear(tree);
+        if (obj_size - SLICE_OFFSET(undo->slice) <= 4096UL) {
+            /* only free bk copy once (last slice): */
+            free(undo->backup);
+            dprintf(("-> free(%p)\n", undo->backup));
+        }
+    }
 
-    release_modified_objs_lock(segment_num);
+    /* check that all objects have the GCFLAG_WRITE_BARRIER afterwards */
+    check_all_write_barrier_flags(pseg->pub.segment_base, list);
 
+    list_clear(list);
 #pragma pop_macro("STM_SEGMENT")
 #pragma pop_macro("STM_PSEGMENT")
 }
@@ -521,7 +817,9 @@
 
     long bytes_in_nursery = throw_away_nursery(pseg);
 
+    acquire_modification_lock(segment_num);
     reset_modified_from_backup_copies(segment_num);
+    release_modification_lock(segment_num);
 
     stm_thread_local_t *tl = pseg->pub.running_thread;
 #ifdef STM_NO_AUTOMATIC_SETJMP
@@ -638,11 +936,36 @@
     assert(frag_size > 0);
     assert(frag_size + ((uintptr_t)frag & 4095) <= 4096);
 
-    /* if the page of the fragment is fully shared, nothing to do */
+    /* if the page of the fragment is fully shared, nothing to do:
+       |S|N|N|N| */
+
+    /* nobody must change the page mapping until we flush */
     assert(STM_PSEGMENT->privatization_lock);
-    if (is_shared_log_page((uintptr_t)frag / 4096))
+
+    int my_segnum = STM_SEGMENT->segment_num;
+    uintptr_t pagenum = (uintptr_t)frag / 4096;
+    bool fully_shared = false;
+
+    if (get_page_status_in(my_segnum, pagenum) == PAGE_SHARED) {
+        fully_shared = true;
+        int i;
+        for (i = 0; fully_shared && i < NB_SEGMENTS; i++) {
+            if (i == my_segnum)
+                continue;
+
+            /* XXX: works if never all pages use SHARED page */
+            if (get_page_status_in(i, pagenum) != PAGE_NO_ACCESS) {
+                fully_shared = false;
+                break;
+            }
+        }
+    }
+
+    if (fully_shared)
         return;                 /* nothing to do */
 
+    /* e.g. |P|S|N|P| */
+
     /* Enqueue this object (or fragemnt of object) */
     if (STM_PSEGMENT->sq_len == SYNC_QUEUE_SIZE)
         synchronize_objects_flush();
@@ -686,7 +1009,7 @@
 
 static void synchronize_objects_flush(void)
 {
-
+    /* XXX: not sure this applies anymore.  */
     /* Do a full memory barrier.  We must make sure that other
        CPUs see the changes we did to the shared page ("S", in
        synchronize_object_enqueue()) before we check the other segments
@@ -697,21 +1020,24 @@
        and copies the page; but it risks doing so before seeing the "S"
        writes.
     */
-    /* XXX: not sure this applies anymore.  */
     long j = STM_PSEGMENT->sq_len;
     if (j == 0)
         return;
     STM_PSEGMENT->sq_len = 0;
 
+    dprintf(("synchronize_objects_flush(): %ld fragments\n", j));
+
     __sync_synchronize();
+    assert(STM_PSEGMENT->privatization_lock);
 
     long i, myself = STM_SEGMENT->segment_num;
     do {
         --j;
         stm_char *frag = STM_PSEGMENT->sq_fragments[j];
         uintptr_t page = ((uintptr_t)frag) / 4096UL;
-        if (is_shared_log_page(page))
-            continue;
+        /* XXX: necessary? */
+        /* if (is_shared_log_page(page)) */
+        /*     continue; */
 
         ssize_t frag_size = STM_PSEGMENT->sq_fragsizes[j];
 
@@ -720,11 +1046,11 @@
             if (i == myself)
                 continue;
 
-            char *dst = REAL_ADDRESS(get_segment_base(i), frag);
-            if (is_private_log_page_in(i, page))
+            if (get_page_status_in(i, page) != PAGE_NO_ACCESS) {
+                /* shared or private, but never segfault */
+                char *dst = REAL_ADDRESS(get_segment_base(i), frag);
                 memcpy(dst, src, frag_size);
-            else
-                EVENTUALLY(memcmp(dst, src, frag_size) == 0);  /* same page */
+            }
         }
     } while (j > 0);
 }
diff --git a/c8/stm/core.h b/c8/stm/core.h
--- a/c8/stm/core.h
+++ b/c8/stm/core.h
@@ -6,6 +6,8 @@
 #include 
 #include 
 #include 
+#include 
+
 
 /************************************************************/
 
@@ -17,7 +19,6 @@
 #define NB_PAGES            (2500*256)    // 2500MB
 #define NB_SEGMENTS         STM_NB_SEGMENTS
 #define NB_SEGMENTS_MAX     240    /* don't increase NB_SEGMENTS past this */
-#define MAP_PAGES_FLAGS     (MAP_SHARED | MAP_ANONYMOUS | MAP_NORESERVE)
 #define NB_NURSERY_PAGES    (STM_GC_NURSERY/4)
 
 #define TOTAL_MEMORY          (NB_PAGES * 4096UL * NB_SEGMENTS)
@@ -25,6 +26,7 @@
 #define FIRST_OBJECT_PAGE     ((READMARKER_END + 4095) / 4096UL)
 #define FIRST_NURSERY_PAGE    FIRST_OBJECT_PAGE
 #define END_NURSERY_PAGE      (FIRST_NURSERY_PAGE + NB_NURSERY_PAGES)
+#define NB_SHARED_PAGES       (NB_PAGES - END_NURSERY_PAGE)
 
 #define READMARKER_START      ((FIRST_OBJECT_PAGE * 4096UL) >> 4)
 #define FIRST_READMARKER_PAGE (READMARKER_START / 4096UL)
@@ -32,11 +34,10 @@
 #define FIRST_OLD_RM_PAGE     (OLD_RM_START / 4096UL)
 #define NB_READMARKER_PAGES   (FIRST_OBJECT_PAGE - FIRST_READMARKER_PAGE)
 
-#define TMP_COPY_PAGE         1 /* HACK */
-
 enum /* stm_flags */ {
     GCFLAG_WRITE_BARRIER = _STM_GCFLAG_WRITE_BARRIER,
     GCFLAG_HAS_SHADOW = 0x02,
+    GCFLAG_WB_EXECUTED = 0x04,
 };
 
 
@@ -54,8 +55,20 @@
 struct stm_priv_segment_info_s {
     struct stm_segment_info_s pub;
 
-    uint8_t modified_objs_lock;
-    struct tree_s *modified_old_objects;
+    /* lock protecting from concurrent modification of
+       'modified_old_objects', page-revision-changes, ...
+       Always acquired in global order of segments to avoid deadlocks. */
+    uint8_t modification_lock;
+
+    /* All the old objects (older than the current transaction) that
+       the current transaction attempts to modify.  This is used to
+       track the STM status: these are old objects that where written
+       to and that will need to be recorded in the commit log.  The
+       list contains three entries for every such object, in the same
+       format as 'struct stm_undo_s' below.
+    */
+    struct list_s *modified_old_objects;
+
     struct list_s *objects_pointing_to_nursery;
     struct tree_s *young_outside_nursery;
     struct tree_s *nursery_objects_shadows;
@@ -86,7 +99,6 @@
     int sq_len;
 };
 
-
 enum /* safe_point */ {
     SP_NO_TRANSACTION=0,
     SP_RUNNING,
@@ -104,17 +116,41 @@
 };
 
 /* Commit Log things */
+struct stm_undo_s {
+    object_t *object;   /* the object that is modified */
+    char *backup;       /* some backup data (a slice of the original obj) */
+    uint64_t slice;     /* location and size of this slice (cannot cross
+                           pages).  The size is in the lower 2 bytes, and
+                           the offset in the remaining 6 bytes. */
+};
+#define SLICE_OFFSET(slice)  ((slice) >> 16)
+#define SLICE_SIZE(slice)    ((int)((slice) & 0xFFFF))
+#define NEW_SLICE(offset, size) (((uint64_t)(offset)) << 16 | (size))
+
+/* The model is: we have a global chained list, from 'commit_log_root',
+   of 'struct stm_commit_log_entry_s' entries.  Every one is fully
+   read-only apart from the 'next' field.  Every one stands for one
+   commit that occurred.  It lists the old objects that were modified
+   in this commit, and their attached "undo logs" --- that is, the
+   data from 'written[n].backup' is the content of (slices of) the
+   object as they were *before* that commit occurred.
+*/
+#define INEV_RUNNING ((void*)-1)
 struct stm_commit_log_entry_s {
-    volatile struct stm_commit_log_entry_s *next;
+    struct stm_commit_log_entry_s *volatile next;
     int segment_num;
-    object_t *written[];        /* terminated with a NULL ptr */
+    uint64_t rev_num;
+    size_t written_count;
+    struct stm_undo_s written[];
 };
-static struct stm_commit_log_entry_s commit_log_root = {NULL, -1};
+static struct stm_commit_log_entry_s commit_log_root = {NULL, -1, 0, 0};
+
 
 #ifndef STM_TESTS
 static
 #endif
        char *stm_object_pages;
+static char *stm_file_pages;
 static int stm_object_pages_fd;
 static stm_thread_local_t *stm_all_thread_locals = NULL;
 
@@ -154,6 +190,8 @@
 static void synchronize_object_enqueue(object_t *obj);
 static void synchronize_objects_flush(void);
 
+static void _signal_handler(int sig, siginfo_t *siginfo, void *context);
+static void _stm_validate(void *free_if_abort);
 
 static inline void _duck(void) {
     /* put a call to _duck() between two instructions that set 0 into
@@ -174,12 +212,82 @@
     spinlock_release(get_priv_segment(segnum)->privatization_lock);
 }
 
-static inline void acquire_modified_objs_lock(int segnum)
+static inline bool all_privatization_locks_acquired()
 {
-    spinlock_acquire(get_priv_segment(segnum)->modified_objs_lock);
+#ifndef NDEBUG
+    long l;
+    for (l = 0; l < NB_SEGMENTS; l++) {
+        if (!get_priv_segment(l)->privatization_lock)
+            return false;
+    }
+    return true;
+#else
+    abort();
+#endif
 }
 
-static inline void release_modified_objs_lock(int segnum)
+static inline void acquire_all_privatization_locks()
 {
-    spinlock_release(get_priv_segment(segnum)->modified_objs_lock);
+    long l;
+    for (l = 0; l < NB_SEGMENTS; l++) {
+        acquire_privatization_lock(l);
+    }
 }
+
+static inline void release_all_privatization_locks()
+{
+    long l;
+    for (l = NB_SEGMENTS-1; l >= 0; l--) {
+        release_privatization_lock(l);
+    }
+}
+
+
+
+/* Modification locks are used to prevent copying from a segment
+   where either the revision of some pages is inconsistent with the
+   rest, or the modified_old_objects list is being modified (bk_copys).
+
+   Lock ordering: acquire privatization lock around acquiring a set
+   of modification locks!
+*/
+
+static inline void acquire_modification_lock(int segnum)
+{
+    spinlock_acquire(get_priv_segment(segnum)->modification_lock);
+}
+
+static inline void release_modification_lock(int segnum)
+{
+    spinlock_release(get_priv_segment(segnum)->modification_lock);
+}
+
+static inline void acquire_modification_lock_set(uint64_t seg_set)
+{
+    assert(NB_SEGMENTS <= 64);
+    OPT_ASSERT(seg_set < (1 << NB_SEGMENTS));
+
+    /* acquire locks in global order */
+    int i;
+    for (i = 0; i < NB_SEGMENTS; i++) {
+        if ((seg_set & (1 << i)) == 0)
+            continue;
+
+        spinlock_acquire(get_priv_segment(i)->modification_lock);
+    }
+}
+
+static inline void release_modification_lock_set(uint64_t seg_set)
+{
+    assert(NB_SEGMENTS <= 64);
+    OPT_ASSERT(seg_set < (1 << NB_SEGMENTS));
+
+    int i;
+    for (i = 0; i < NB_SEGMENTS; i++) {
+        if ((seg_set & (1 << i)) == 0)
+            continue;
+
+        assert(get_priv_segment(i)->modification_lock);
+        spinlock_release(get_priv_segment(i)->modification_lock);
+    }
+}
diff --git a/c8/stm/gcpage.c b/c8/stm/gcpage.c
--- a/c8/stm/gcpage.c
+++ b/c8/stm/gcpage.c
@@ -5,6 +5,7 @@
 
 static void setup_gcpage(void)
 {
+    /* XXXXXXX should use stm_file_pages, no? */
     uninitialized_page_start = stm_object_pages + END_NURSERY_PAGE * 4096UL;
     uninitialized_page_stop  = stm_object_pages + NB_PAGES * 4096UL;
 }
@@ -15,11 +16,13 @@
 
 static void setup_N_pages(char *pages_addr, uint64_t num)
 {
+    /* initialize to |S|N|N|N| */
     long i;
     for (i = 0; i < NB_SEGMENTS; i++) {
         acquire_privatization_lock(i);
     }
-    pages_initialize_shared((pages_addr - stm_object_pages) / 4096UL, num);
+    pages_initialize_shared_for(STM_SEGMENT->segment_num,
+                                (pages_addr - stm_object_pages) / 4096UL, num);
     for (i = NB_SEGMENTS-1; i >= 0; i--) {
         release_privatization_lock(i);
     }
diff --git a/c8/stm/list.h b/c8/stm/list.h
--- a/c8/stm/list.h
+++ b/c8/stm/list.h
@@ -45,6 +45,19 @@
     return lst;
 }
 
+static inline struct list_s *list_append3(struct list_s *lst, uintptr_t item0,
+                                          uintptr_t item1, uintptr_t item2)
+{
+    uintptr_t index = lst->count;
+    lst->count += 3;
+    if (UNLIKELY(index + 2 > lst->last_allocated))
+        lst = _list_grow(lst, index + 2);
+    lst->items[index + 0] = item0;
+    lst->items[index + 1] = item1;
+    lst->items[index + 2] = item2;
+    return lst;
+}
+
 
 static inline void list_clear(struct list_s *lst)
 {
diff --git a/c8/stm/misc.c b/c8/stm/misc.c
--- a/c8/stm/misc.c
+++ b/c8/stm/misc.c
@@ -46,8 +46,9 @@
 long _stm_count_modified_old_objects(void)
 {
     assert(STM_PSEGMENT->modified_old_objects);
-    assert(tree_count(STM_PSEGMENT->modified_old_objects) < 10000);
-    return tree_count(STM_PSEGMENT->modified_old_objects);
+    assert(list_count(STM_PSEGMENT->modified_old_objects) < 30000);
+    assert((list_count(STM_PSEGMENT->modified_old_objects) % 3) == 0);
+    return list_count(STM_PSEGMENT->modified_old_objects) / 3;
 }
 
 long _stm_count_objects_pointing_to_nursery(void)
@@ -59,8 +60,8 @@
 
 object_t *_stm_enum_modified_old_objects(long index)
 {
-    wlog_t* entry = tree_item(STM_PSEGMENT->modified_old_objects, index);
-    return (object_t*)entry->addr;
+    return (object_t *)list_item(
+        STM_PSEGMENT->modified_old_objects, index * 3);
 }
 
 object_t *_stm_enum_objects_pointing_to_nursery(long index)
@@ -69,13 +70,12 @@
         STM_PSEGMENT->objects_pointing_to_nursery, index);
 }
 
-static volatile struct stm_commit_log_entry_s *_last_cl_entry;
+static struct stm_commit_log_entry_s *_last_cl_entry;
 static long _last_cl_entry_index;
 void _stm_start_enum_last_cl_entry()
 {
     _last_cl_entry = &commit_log_root;
-    volatile struct stm_commit_log_entry_s *cl = (volatile struct stm_commit_log_entry_s *)
-        &commit_log_root;
+    struct stm_commit_log_entry_s *cl = &commit_log_root;
 
     while ((cl = cl->next)) {
         _last_cl_entry = cl;
@@ -85,8 +85,10 @@
 
 object_t *_stm_next_last_cl_entry()
 {
-    if (_last_cl_entry != &commit_log_root)
-        return _last_cl_entry->written[_last_cl_entry_index++];
-    return NULL;
+    if (_last_cl_entry == &commit_log_root)
+        return NULL;
+    if (_last_cl_entry_index >= _last_cl_entry->written_count)
+        return NULL;
+    return _last_cl_entry->written[_last_cl_entry_index++].object;
 }
 #endif
diff --git a/c8/stm/nursery.c b/c8/stm/nursery.c
--- a/c8/stm/nursery.c
+++ b/c8/stm/nursery.c
@@ -183,6 +183,8 @@
         uintptr_t obj_sync_now = list_pop_item(lst);
         object_t *obj = (object_t *)(obj_sync_now & ~FLAG_SYNC_LARGE);
 
+        assert(!_is_in_nursery(obj));
+
         _collect_now(obj);
 
         if (obj_sync_now & FLAG_SYNC_LARGE) {
diff --git a/c8/stm/pages.c b/c8/stm/pages.c
--- a/c8/stm/pages.c
+++ b/c8/stm/pages.c
@@ -1,8 +1,8 @@
 #ifndef _STM_CORE_H_
 # error "must be compiled via stmgc.c"
 #endif
-#include 
 
+#include 
 /************************************************************/
 
 static void setup_pages(void)
@@ -11,83 +11,56 @@
 
 static void teardown_pages(void)
 {
-    memset(pages_privatized, 0, sizeof(pages_privatized));
+    memset(pages_status, 0, sizeof(pages_status));
 }
 
 /************************************************************/
 
-static void d_remap_file_pages(char *addr, size_t size, ssize_t pgoff)
-{
-    dprintf(("remap_file_pages: 0x%lx bytes: (seg%ld %p) --> (seg%ld %p)\n",
-             (long)size,
-             (long)((addr - stm_object_pages) / 4096UL) / NB_PAGES,
-             (void *)((addr - stm_object_pages) % (4096UL * NB_PAGES)),
-             (long)pgoff / NB_PAGES,
-             (void *)((pgoff % NB_PAGES) * 4096UL)));
-    assert(size % 4096 == 0);
-    assert(size <= TOTAL_MEMORY);
-    assert(((uintptr_t)addr) % 4096 == 0);
-    assert(addr >= stm_object_pages);
-    assert(addr <= stm_object_pages + TOTAL_MEMORY - size);
-    assert(pgoff >= 0);
-    assert(pgoff <= (TOTAL_MEMORY - size) / 4096UL);
-
-    /* assert remappings follow the rule that page N in one segment
-       can only be remapped to page N in another segment */
-    assert(IMPLY(((addr - stm_object_pages) / 4096UL) != TMP_COPY_PAGE,
-                 ((addr - stm_object_pages) / 4096UL - pgoff) % NB_PAGES == 0));
-
-#ifdef USE_REMAP_FILE_PAGES
-    int res = remap_file_pages(addr, size, 0, pgoff, 0);
-    if (UNLIKELY(res < 0))
-        stm_fatalerror("remap_file_pages: %m");
-#else
-    char *res = mmap(addr, size,
-                     PROT_READ | PROT_WRITE,
-                     (MAP_PAGES_FLAGS & ~MAP_ANONYMOUS) | MAP_FIXED,
-                     stm_object_pages_fd, pgoff * 4096UL);
-    if (UNLIKELY(res != addr))
-        stm_fatalerror("mmap (remapping page): %m");
-#endif
-}
-
-
-static void pages_initialize_shared(uintptr_t pagenum, uintptr_t count)
+static void pages_initialize_shared_for(long segnum, uintptr_t pagenum, uintptr_t count)
 {
     /* call remap_file_pages() to make all pages in the range(pagenum,
-       pagenum+count) refer to the same physical range of pages from
-       segment 0. */
-    dprintf(("pages_initialize_shared: 0x%ld - 0x%ld\n", pagenum,
-             pagenum + count));
-#ifndef NDEBUG
-    long l;
-    for (l = 0; l < NB_SEGMENTS; l++) {
-        assert(get_priv_segment(l)->privatization_lock);
-    }
-#endif
+       pagenum+count) PAGE_SHARED in segnum, and PAGE_NO_ACCESS in other segments
+       initialize to |S|N|N|N| */
+
+    dprintf(("pages_initialize_shared: 0x%ld - 0x%ld\n", pagenum, pagenum + count));
+
+    assert(all_privatization_locks_acquired());
+
     assert(pagenum < NB_PAGES);
     if (count == 0)
         return;
+
+    /* already shared after setup.c (also for the other count-1 pages) */
+    assert(get_page_status_in(segnum, pagenum) == PAGE_SHARED);
+
+    /* make other segments NO_ACCESS: */
     uintptr_t i;
-    for (i = 1; i < NB_SEGMENTS; i++) {
-        char *segment_base = get_segment_base(i);
-        d_remap_file_pages(segment_base + pagenum * 4096UL,
-                           count * 4096UL, pagenum);
-    }
+    for (i = 0; i < NB_SEGMENTS; i++) {
+        if (i != segnum) {
+            char *segment_base = get_segment_base(i);
+            mprotect(segment_base + pagenum * 4096UL,
+                     count * 4096UL, PROT_NONE);
 
-    for (i = 0; i < NB_SEGMENTS; i++) {
-        uintptr_t amount = count;
-        while (amount-->0) {
-            volatile struct page_shared_s *ps2 = (volatile struct page_shared_s *)
-                &pages_privatized[pagenum + amount - PAGE_FLAG_START];
+            /* char *result = mmap( */
+            /*     segment_base + pagenum * 4096UL, */
+            /*     count * 4096UL, */
+            /*     PROT_NONE, */
+            /*     MAP_FIXED|MAP_NORESERVE|MAP_PRIVATE|MAP_ANONYMOUS, */
+            /*     -1, 0); */
+            /* if (result == MAP_FAILED) */
+            /*     stm_fatalerror("pages_initialize_shared failed (mmap): %m"); */
 
-            ps2->by_segment = 0; /* not private */
+
+            long amount = count;
+            while (amount-->0) {
+                set_page_status_in(i, pagenum + amount, PAGE_NO_ACCESS);
+            }
         }
     }
 }
 
 
-static void page_privatize_in(int segnum, uintptr_t pagenum, char *initialize_from)
+static void page_privatize_in(int segnum, uintptr_t pagenum)
 {
 #ifndef NDEBUG
     long l;
@@ -95,34 +68,19 @@
         assert(get_priv_segment(l)->privatization_lock);
     }
 #endif
-
-    /* check this thread's 'pages_privatized' bit */
-    uint64_t bitmask = 1UL << segnum;
-    volatile struct page_shared_s *ps = (volatile struct page_shared_s *)
-        &pages_privatized[pagenum - PAGE_FLAG_START];
-    if (ps->by_segment & bitmask) {
-        /* the page is already privatized; nothing to do */
-        return;
-    }
-
+    assert(get_page_status_in(segnum, pagenum) == PAGE_NO_ACCESS);
     dprintf(("page_privatize(%lu) in seg:%d\n", pagenum, segnum));
 
-    /* add this thread's 'pages_privatized' bit */
-    ps->by_segment |= bitmask;
+    char *addr = (char*)(get_virt_page_of(segnum, pagenum) * 4096UL);
+    char *result = mmap(
+        addr, 4096UL, PROT_READ | PROT_WRITE,
+        MAP_FIXED | MAP_PRIVATE | MAP_NORESERVE,
+        stm_object_pages_fd, get_file_page_of(pagenum) * 4096UL);
+    if (result == MAP_FAILED)
+        stm_fatalerror("page_privatize_in failed (mmap): %m");
 
-    /* "unmaps" the page to make the address space location correspond
-       again to its underlying file offset (XXX later we should again
-       attempt to group together many calls to d_remap_file_pages() in
-       succession) */
-    uintptr_t pagenum_in_file = NB_PAGES * segnum + pagenum;
-    char *tmp_page = stm_object_pages + TMP_COPY_PAGE * 4096UL;
-    /* first remap to TMP_PAGE, then copy stuff there (to the underlying
-       file page), then remap this file-page hopefully atomically to the
-       segnum's virtual page */
-    d_remap_file_pages(tmp_page, 4096, pagenum_in_file);
-    pagecopy(tmp_page, initialize_from);
-    write_fence();
+    set_page_status_in(segnum, pagenum, PAGE_PRIVATE);
 
-    char *new_page = stm_object_pages + pagenum_in_file * 4096UL;
-    d_remap_file_pages(new_page, 4096, pagenum_in_file);
+    volatile char *dummy = REAL_ADDRESS(get_segment_base(segnum), pagenum*4096UL);
+    *dummy = *dummy;            /* force copy-on-write from shared page */
 }
diff --git a/c8/stm/pages.h b/c8/stm/pages.h
--- a/c8/stm/pages.h
+++ b/c8/stm/pages.h
@@ -20,27 +20,35 @@
 
 #define PAGE_FLAG_START   END_NURSERY_PAGE
 #define PAGE_FLAG_END     NB_PAGES
-
-#define USE_REMAP_FILE_PAGES
+/* == NB_SHARED_PAGES */
 
 struct page_shared_s {
-#if NB_SEGMENTS <= 8
+#if NB_SEGMENTS <= 4
     uint8_t by_segment;
+#elif NB_SEGMENTS <= 8
+    uint16_t by_segment;
 #elif NB_SEGMENTS <= 16
-    uint16_t by_segment;
+    uint32_t by_segment;
 #elif NB_SEGMENTS <= 32
-    uint32_t by_segment;
-#elif NB_SEGMENTS <= 64
     uint64_t by_segment;
 #else
-#   error "NB_SEGMENTS > 64 not supported right now"
+#   error "NB_SEGMENTS > 32 not supported right now"
 #endif
 };
 
-static struct page_shared_s pages_privatized[PAGE_FLAG_END - PAGE_FLAG_START];
+enum {
+    PAGE_SHARED = 0,
+    PAGE_PRIVATE = 1,
+    PAGE_NO_ACCESS = 2,
+};
 
-static void pages_initialize_shared(uintptr_t pagenum, uintptr_t count);
-static void page_privatize_in(int segnum, uintptr_t pagenum, char *initialize_from);
+static struct page_shared_s pages_status[NB_SHARED_PAGES];
+
+static void pages_initialize_shared_for(long segnum, uintptr_t pagenum, uintptr_t count);
+static void page_privatize_in(int segnum, uintptr_t pagenum);
+
+
+
 
 static inline uintptr_t get_virt_page_of(long segnum, uintptr_t pagenum)
 {
@@ -48,24 +56,33 @@
     return (uintptr_t)get_segment_base(segnum) / 4096UL + pagenum;
 }
 
-static inline bool is_shared_log_page(uintptr_t pagenum)
+static inline uintptr_t get_file_page_of(uintptr_t pagenum)
 {
-    assert(pagenum >= PAGE_FLAG_START);
-    return pages_privatized[pagenum - PAGE_FLAG_START].by_segment == 0;
+    /* logical page -> file page */
+    return pagenum - PAGE_FLAG_START;
 }
 
-static inline void set_page_private_in(long segnum, uintptr_t pagenum)
+
+static inline uint8_t get_page_status_in(long segnum, uintptr_t pagenum)
 {
-    uint64_t bitmask = 1UL << segnum;
+    int seg_shift = segnum * 2;
+    OPT_ASSERT(seg_shift < 8 * sizeof(struct page_shared_s));
     volatile struct page_shared_s *ps = (volatile struct page_shared_s *)
-        &pages_privatized[pagenum - PAGE_FLAG_START];
-    assert(!(ps->by_segment & bitmask));
-    ps->by_segment |= bitmask;
+        &pages_status[get_file_page_of(pagenum)];
+
+    return (ps->by_segment >> seg_shift) & 3;
 }
 
-static inline bool is_private_log_page_in(long segnum, uintptr_t pagenum)
+static inline void set_page_status_in(long segnum, uintptr_t pagenum, uint8_t status)
 {
-    assert(pagenum >= PAGE_FLAG_START);
-    uint64_t bitmask = 1UL << segnum;
-    return (pages_privatized[pagenum - PAGE_FLAG_START].by_segment & bitmask);
+    OPT_ASSERT(status < 3);
+
+    int seg_shift = segnum * 2;
+    OPT_ASSERT(seg_shift < 8 * sizeof(struct page_shared_s));
+    volatile struct page_shared_s *ps = (volatile struct page_shared_s *)
+        &pages_status[get_file_page_of(pagenum)];
+
+    assert(status != get_page_status_in(segnum, pagenum));
+    ps->by_segment &= ~(3UL << seg_shift); /* clear */
+    ps->by_segment |= status << seg_shift; /* set */
 }
diff --git a/c8/stm/setup.c b/c8/stm/setup.c
--- a/c8/stm/setup.c
+++ b/c8/stm/setup.c
@@ -3,55 +3,61 @@
 #endif
 
 #include 
+#include            /* For O_* constants */
 
-#ifdef USE_REMAP_FILE_PAGES
-static char *setup_mmap(char *reason, int *ignored)
+static void setup_mmap(char *reason)
 {
-    char *result = mmap(NULL, TOTAL_MEMORY,
-                        PROT_READ | PROT_WRITE,
-                        MAP_PAGES_FLAGS, -1, 0);
-    if (result == MAP_FAILED)
-        stm_fatalerror("%s failed: %m", reason);
-
-    return result;
-}
-static void close_fd_mmap(int ignored)
-{
-}
-#else
-#include            /* For O_* constants */
-static char *setup_mmap(char *reason, int *map_fd)
-{
-    char name[128];
+    char name[] = "/__stmgc_c8__";
 
     /* Create the big shared memory object, and immediately unlink it.
        There is a small window where if this process is killed the
        object is left around.  It doesn't seem possible to do anything
        about it...
     */
-    int fd = shm_open(name, O_RDWR | O_CREAT | O_EXCL, 0600);
+    stm_object_pages_fd = shm_open(name, O_RDWR | O_CREAT | O_EXCL, 0600);
     shm_unlink(name);
 
-    if (fd == -1) {
+    if (stm_object_pages_fd == -1)
         stm_fatalerror("%s failed (stm_open): %m", reason);
+
+    if (ftruncate(stm_object_pages_fd, NB_SHARED_PAGES * 4096UL) != 0)
+        stm_fatalerror("%s failed (ftruncate): %m", reason);
+
+    stm_file_pages = mmap(NULL, NB_SHARED_PAGES * 4096UL,
+                          PROT_READ | PROT_WRITE,
+                          MAP_SHARED | MAP_NORESERVE,
+                          stm_object_pages_fd, 0);
+
+    if (stm_file_pages == MAP_FAILED)
+        stm_fatalerror("%s failed (mmap): %m", reason);
+
+
+    /* reserve the whole virtual memory space of the program for
+       all segments: */
+    stm_object_pages = mmap(NULL, TOTAL_MEMORY,
+                            PROT_READ | PROT_WRITE,
+                            MAP_PRIVATE | MAP_NORESERVE | MAP_ANONYMOUS,
+                            -1, 0);
+    if (stm_object_pages == MAP_FAILED)
+        stm_fatalerror("%s failed (mmap): %m", reason);
+
+    /* remap the shared part of the segments to the file pages */
+    long l;
+    for (l = 0; l < NB_SEGMENTS; l++) {
+        char *result = mmap(
+            stm_object_pages + (l * NB_PAGES + END_NURSERY_PAGE) * 4096UL, /* addr */
+            NB_SHARED_PAGES * 4096UL, /* len */
+            PROT_READ | PROT_WRITE,
+            MAP_FIXED | MAP_SHARED | MAP_NORESERVE,
+            stm_object_pages_fd, 0); /* file & offset */
+        if (result == MAP_FAILED)
+            stm_fatalerror("%s failed (mmap): %m", reason);
     }
-    if (ftruncate(fd, TOTAL_MEMORY) != 0) {
-        stm_fatalerror("%s failed (ftruncate): %m", reason);
-    }
-    char *result = mmap(NULL, TOTAL_MEMORY,
-                        PROT_READ | PROT_WRITE,
-                        MAP_PAGES_FLAGS & ~MAP_ANONYMOUS, fd, 0);
-    if (result == MAP_FAILED) {
-        stm_fatalerror("%s failed (mmap): %m", reason);
-    }
-    *map_fd = fd;
-    return result;
 }
 static void close_fd_mmap(int map_fd)
 {
     close(map_fd);
 }
-#endif
 
 static void setup_protection_settings(void)
 {
@@ -63,33 +69,40 @@
            NULL accesses land.  We mprotect it so that accesses fail. */
         mprotect(segment_base, 4096, PROT_NONE);
 
-        /* TMP_COPY_PAGE is used for atomic privatization */
-        mprotect(segment_base + TMP_COPY_PAGE * 4096UL,
-                 4096UL, PROT_READ|PROT_WRITE);
-
         /* Pages in range(2, FIRST_READMARKER_PAGE) are never used */
-        if (FIRST_READMARKER_PAGE > TMP_COPY_PAGE + 1)
-            mprotect(segment_base + (TMP_COPY_PAGE + 1) * 4096,
-                     (FIRST_READMARKER_PAGE - TMP_COPY_PAGE - 1) * 4096UL,
+        if (FIRST_READMARKER_PAGE > 2)
+            mprotect(segment_base + 2 * 4096,
+                     (FIRST_READMARKER_PAGE - 2) * 4096UL,
                      PROT_NONE);
 
-        /* STM_SEGMENT */
-        mprotect(segment_base + ((uintptr_t)STM_SEGMENT / 4096UL) * 4096UL,
-                 4096UL, PROT_READ|PROT_WRITE);
+        /* STM_SEGMENT-TL is in page 1 */
     }
 }
 
 
+static void setup_signal_handler(void)
+{
+    struct sigaction act;
+    memset(&act, 0, sizeof(act));
+
+	act.sa_sigaction = &_signal_handler;
+	/* The SA_SIGINFO flag tells sigaction() to use the sa_sigaction field, not sa_handler. */
+	act.sa_flags = SA_SIGINFO | SA_NODEFER;
+
+	if (sigaction(SIGSEGV, &act, NULL) < 0) {
+		perror ("sigaction");
+		abort();
+	}
+}
+
 void stm_setup(void)
 {
     /* Check that some values are acceptable */
-    assert(TMP_COPY_PAGE > 0 && TMP_COPY_PAGE <= 1);
-    assert(TMP_COPY_PAGE * 4096 + 4096 <= ((uintptr_t)STM_SEGMENT));
+    assert(4096 <= ((uintptr_t)STM_SEGMENT));
     assert((uintptr_t)STM_SEGMENT == (uintptr_t)STM_PSEGMENT);
     assert(((uintptr_t)STM_PSEGMENT) + sizeof(*STM_PSEGMENT) <= FIRST_READMARKER_PAGE*4096);
 
     assert(NB_SEGMENTS <= NB_SEGMENTS_MAX);
-    assert(TMP_COPY_PAGE < FIRST_READMARKER_PAGE);
     assert(FIRST_READMARKER_PAGE * 4096UL <= READMARKER_START);
     assert(READMARKER_START < READMARKER_END);
     assert(READMARKER_END <= 4096UL * FIRST_OBJECT_PAGE);
@@ -99,9 +112,14 @@
            (FIRST_READMARKER_PAGE * 4096UL));
     assert(_STM_FAST_ALLOC <= NB_NURSERY_PAGES * 4096);
 
-    stm_object_pages = setup_mmap("initial stm_object_pages mmap()",
-                                  &stm_object_pages_fd);
+    setup_mmap("initial stm_object_pages mmap()");
+
+    assert(stm_object_pages_fd);
+    assert(stm_object_pages);
+    assert(stm_file_pages);
+
     setup_protection_settings();
+    setup_signal_handler();
 
     long i;
     for (i = 0; i < NB_SEGMENTS; i++) {
@@ -118,7 +136,7 @@
         assert(0 <= i && i < 255);   /* 255 is WL_VISITED in gcpage.c */
         pr->pub.segment_num = i;
         pr->pub.segment_base = segment_base;
-        pr->modified_old_objects = tree_create();
+        pr->modified_old_objects = list_create();
         pr->objects_pointing_to_nursery = list_create();
         pr->young_outside_nursery = tree_create();
         pr->nursery_objects_shadows = tree_create();
@@ -155,7 +173,7 @@
         struct stm_priv_segment_info_s *pr = get_priv_segment(i);
         assert(list_is_empty(pr->objects_pointing_to_nursery));
         list_free(pr->objects_pointing_to_nursery);
-        tree_free(pr->modified_old_objects);
+        list_free(pr->modified_old_objects);
         tree_free(pr->young_outside_nursery);
         tree_free(pr->nursery_objects_shadows);
         tree_free(pr->callbacks_on_commit_and_abort[0]);
@@ -243,6 +261,15 @@
     _init_shadow_stack(tl);
     set_gs_register(get_segment_base(num));
     s_mutex_unlock();
+
+    if (num == 0) {
+        dprintf(("STM_GC_NURSERY: %d\n", STM_GC_NURSERY));
+        dprintf(("NB_PAGES: %d\n", NB_PAGES));
+        dprintf(("NB_SEGMENTS: %d\n", NB_SEGMENTS));
+        dprintf(("FIRST_OBJECT_PAGE=FIRST_NURSERY_PAGE: %lu\n", FIRST_OBJECT_PAGE));
+        dprintf(("END_NURSERY_PAGE: %lu\n", END_NURSERY_PAGE));
+        dprintf(("NB_SHARED_PAGES: %lu\n", NB_SHARED_PAGES));
+    }
 }
 
 void stm_unregister_thread_local(stm_thread_local_t *tl)
diff --git a/c8/stm/setup.h b/c8/stm/setup.h
--- a/c8/stm/setup.h
+++ b/c8/stm/setup.h
@@ -1,4 +1,4 @@
-static char *setup_mmap(char *reason, int *map_fd);
+static void setup_mmap(char *reason);
 static void close_fd_mmap(int map_fd);
 static void setup_protection_settings(void);
 static pthread_t *_get_cpth(stm_thread_local_t *);
diff --git a/c8/stmgc.h b/c8/stmgc.h
--- a/c8/stmgc.h
+++ b/c8/stmgc.h
@@ -34,7 +34,7 @@
     uintptr_t nursery_end;
     struct stm_thread_local_s *running_thread;
 };
-#define STM_SEGMENT           ((stm_segment_info_t *)8192)
+#define STM_SEGMENT           ((stm_segment_info_t *)4352)
 
 
 struct stm_shadowentry_s {
@@ -71,7 +71,6 @@
 char *_stm_real_address(object_t *o);
 #ifdef STM_TESTS
 #include 
-void stm_validate(void *free_if_abort);
 bool _stm_was_read(object_t *obj);
 bool _stm_was_written(object_t *obj);
 
@@ -231,6 +230,7 @@
 }
 
 void stm_become_globally_unique_transaction(stm_thread_local_t *tl, const char *msg);
+void stm_validate(void);
 
 
 /* dummies for now: */
diff --git a/c8/test/support.py b/c8/test/support.py
--- a/c8/test/support.py
+++ b/c8/test/support.py
@@ -43,13 +43,14 @@
 void stm_teardown(void);
 void stm_register_thread_local(stm_thread_local_t *tl);
 void stm_unregister_thread_local(stm_thread_local_t *tl);
-void stm_validate(void *free_if_abort);
+void stm_validate();
 bool _check_stm_validate();
 
 object_t *stm_setup_prebuilt(object_t *);
 void _stm_start_safe_point(void);
 bool _check_stop_safe_point(void);
 
+ssize_t _checked_stmcb_size_rounded_up(struct object_s *obj);
 
 bool _checked_stm_write(object_t *obj);
 bool _stm_was_read(object_t *obj);
@@ -172,6 +173,20 @@
     return 1;
 }
 
+ssize_t _checked_stmcb_size_rounded_up(struct object_s *obj)
+{
+    stm_thread_local_t *_tl = STM_SEGMENT->running_thread;      \
+    void **jmpbuf = _tl->rjthread.jmpbuf;                       \
+    if (__builtin_setjmp(jmpbuf) == 0) { /* returned directly */\
+        ssize_t res = stmcb_size_rounded_up(obj);
+        clear_jmpbuf(_tl);
+        return res;
+    }
+    clear_jmpbuf(_tl);
+    return 1;
+}
+
+
 bool _check_stop_safe_point(void) {
     CHECKED(_stm_stop_safe_point());
 }
@@ -189,7 +204,7 @@
 }
 
 bool _check_stm_validate(void) {
-    CHECKED(stm_validate(NULL));
+    CHECKED(stm_validate());
 }
 
 #undef CHECKED
@@ -276,7 +291,7 @@
                     ],
      undef_macros=['NDEBUG'],
      include_dirs=[parent_dir],
-     extra_compile_args=['-g', '-O0', '-Wall', '-ferror-limit=1'],

From noreply at buildbot.pypy.org  Wed Nov 12 11:29:48 2014
From: noreply at buildbot.pypy.org (Raemi)
Date: Wed, 12 Nov 2014 11:29:48 +0100 (CET)
Subject: [pypy-commit] stmgc c8-small-uniform: switch allocation to using
 file pages instead of segment 0
Message-ID: <20141112102948.A3CBB1D288C@cobra.cs.uni-duesseldorf.de>

Author: Remi Meier 
Branch: c8-small-uniform
Changeset: r1506:4ce8b3189edc
Date: 2014-11-12 11:29 +0100
http://bitbucket.org/pypy/stmgc/changeset/4ce8b3189edc/

Log:	switch allocation to using file pages instead of segment 0

diff --git a/c8/stm/core.h b/c8/stm/core.h
--- a/c8/stm/core.h
+++ b/c8/stm/core.h
@@ -150,7 +150,10 @@
 static
 #endif
        char *stm_object_pages;
-static char *stm_file_pages;
+#ifndef STM_TESTS
+static
+#endif
+       char *stm_file_pages;
 static int stm_object_pages_fd;
 static stm_thread_local_t *stm_all_thread_locals = NULL;
 
diff --git a/c8/stm/gcpage.c b/c8/stm/gcpage.c
--- a/c8/stm/gcpage.c
+++ b/c8/stm/gcpage.c
@@ -6,8 +6,8 @@
 static void setup_gcpage(void)
 {
     /* XXXXXXX should use stm_file_pages, no? */
-    uninitialized_page_start = stm_object_pages + END_NURSERY_PAGE * 4096UL;
-    uninitialized_page_stop  = stm_object_pages + NB_PAGES * 4096UL;
+    uninitialized_page_start = stm_file_pages;
+    uninitialized_page_stop  = stm_file_pages + NB_SHARED_PAGES * 4096UL;
 }
 
 static void teardown_gcpage(void)
@@ -21,8 +21,10 @@
     for (i = 0; i < NB_SEGMENTS; i++) {
         acquire_privatization_lock(i);
     }
-    pages_initialize_shared_for(STM_SEGMENT->segment_num,
-                                (pages_addr - stm_object_pages) / 4096UL, num);
+    pages_initialize_shared_for(
+        STM_SEGMENT->segment_num,
+        get_page_of_file_page((pages_addr - stm_file_pages) / 4096UL),
+        num);
     for (i = NB_SEGMENTS-1; i >= 0; i--) {
         release_privatization_lock(i);
     }
@@ -31,7 +33,7 @@
 
 static int lock_growth_large = 0;
 
-static char *allocate_outside_nursery_large(uint64_t size)
+static stm_char *allocate_outside_nursery_large(uint64_t size)
 {
     /* XXX: real allocation */
     spinlock_acquire(lock_growth_large);
@@ -51,26 +53,27 @@
             stm_fatalerror("uninitialized_page_start changed?");
         }
     }
-    dprintf(("allocate_outside_nursery_large(%lu): %p, seg=%d, page=%lu\n",
-             size, addr, get_segment_of_linear_address(addr),
-             (addr - get_segment_base(get_segment_of_linear_address(addr))) / 4096UL));
+
+    dprintf(("allocate_outside_nursery_large(%lu): %p, page=%lu\n",
+             size, addr,
+             (uintptr_t)addr / 4096UL + END_NURSERY_PAGE));
 
     spinlock_release(lock_growth_large);
-    return addr;
+    return (stm_char*)(addr - stm_file_pages + END_NURSERY_PAGE * 4096UL);
 }
 
 object_t *_stm_allocate_old(ssize_t size_rounded_up)
 {
     /* only for tests xxx but stm_setup_prebuilt() uses this now too */
-    char *p = allocate_outside_nursery_large(size_rounded_up);
-    memset(p, 0, size_rounded_up);
+    stm_char *p = allocate_outside_nursery_large(size_rounded_up);
+    memset(stm_object_pages + (uintptr_t)p, 0, size_rounded_up);
 
-    object_t *o = (object_t *)(p - stm_object_pages);
+    object_t *o = (object_t *)p;
     o->stm_flags = GCFLAG_WRITE_BARRIER;
 
     dprintf(("allocate_old(%lu): %p, seg=%d, page=%lu\n",
              size_rounded_up, p,
-             get_segment_of_linear_address(p),
-             (p - STM_SEGMENT->segment_base) / 4096UL));
+             get_segment_of_linear_address(stm_object_pages + (uintptr_t)p),
+             (uintptr_t)p / 4096UL));
     return o;
 }
diff --git a/c8/stm/gcpage.h b/c8/stm/gcpage.h
--- a/c8/stm/gcpage.h
+++ b/c8/stm/gcpage.h
@@ -8,4 +8,4 @@
 static void setup_gcpage(void);
 static void teardown_gcpage(void);
 static void setup_N_pages(char *pages_addr, uint64_t num);
-static char *allocate_outside_nursery_large(uint64_t size);
+static stm_char *allocate_outside_nursery_large(uint64_t size);
diff --git a/c8/stm/nursery.c b/c8/stm/nursery.c
--- a/c8/stm/nursery.c
+++ b/c8/stm/nursery.c
@@ -94,13 +94,11 @@
         if (size > GC_LAST_SMALL_SIZE) {
             /* case 1: object is not small enough.
                Ask gcpage.c for an allocation via largemalloc. */
-            char *allocated = allocate_outside_nursery_large(size);
-            nobj = (object_t *)(allocated - stm_object_pages);
+            nobj = (object_t *)allocate_outside_nursery_large(size);
         }
         else {
             /* case "small enough" */
-            char *allocated = allocate_outside_nursery_small(size);
-            nobj = (object_t *)(allocated - stm_object_pages);
+            nobj = (object_t *)allocate_outside_nursery_small(size);
         }
 
         /* copy the object */
@@ -334,8 +332,7 @@
     /*     stm_collect(0); */
     /* } */
 
-    char *result = allocate_outside_nursery_large(size_rounded_up);
-    object_t *o = (object_t *)(result - stm_object_pages);
+    object_t *o = (object_t *)allocate_outside_nursery_large(size_rounded_up);
 
     tree_insert(STM_PSEGMENT->young_outside_nursery, (uintptr_t)o, 0);
 
@@ -387,8 +384,7 @@
     size_t size = stmcb_size_rounded_up((struct object_s *)realobj);
 
     /* always gets outside as a large object for now */
-    char *allocated = allocate_outside_nursery_large(size);
-    object_t *nobj = (object_t *)(allocated - stm_object_pages);
+    object_t *nobj = (object_t *)allocate_outside_nursery_large(size);
 
     /* Initialize the shadow enough to be considered a valid gc object.
        If the original object stays alive at the next minor collection,
diff --git a/c8/stm/pages.h b/c8/stm/pages.h
--- a/c8/stm/pages.h
+++ b/c8/stm/pages.h
@@ -62,6 +62,10 @@
     return pagenum - PAGE_FLAG_START;
 }
 
+static inline uintptr_t get_page_of_file_page(uintptr_t file_page)
+{
+    return file_page + END_NURSERY_PAGE;
+}
 
 static inline uint8_t get_page_status_in(long segnum, uintptr_t pagenum)
 {
diff --git a/c8/stm/smallmalloc.c b/c8/stm/smallmalloc.c
--- a/c8/stm/smallmalloc.c
+++ b/c8/stm/smallmalloc.c
@@ -3,8 +3,8 @@
 #endif
 
 
-#define PAGE_SMSIZE_START   END_NURSERY_PAGE
-#define PAGE_SMSIZE_END     NB_PAGES
+#define PAGE_SMSIZE_START   0
+#define PAGE_SMSIZE_END     NB_SHARED_PAGES
 
 typedef struct {
     uint8_t sz;
@@ -21,7 +21,7 @@
 
 static fpsz_t *get_fpsz(char *smallpage)
 {
-    uintptr_t pagenum = (((char *)smallpage) - stm_object_pages) / 4096;
+    uintptr_t pagenum = (((char *)smallpage) - stm_file_pages) / 4096;
     assert(PAGE_SMSIZE_START <= pagenum && pagenum < PAGE_SMSIZE_END);
     return &full_pages_object_size[pagenum - PAGE_SMSIZE_START];
 }
@@ -46,6 +46,7 @@
 
 static void grab_more_free_pages_for_small_allocations(void)
 {
+    dprintf(("grab_more_free_pages_for_small_allocation()\n"));
     /* Grab GCPAGE_NUM_PAGES pages out of the top addresses.  Use the
        lock of pages.c to prevent any remapping from occurring under our
        feet.
@@ -59,7 +60,7 @@
             goto out_of_memory;
 
         uninitialized_page_stop -= decrease_by;
-        first_small_uniform_loc = uninitialized_page_stop - stm_object_pages;
+        first_small_uniform_loc = uninitialized_page_stop - stm_file_pages;
 
         /* XXX: */
         /* char *base = stm_object_pages + END_NURSERY_PAGE * 4096UL; */
@@ -150,7 +151,7 @@
 }
 
 __attribute__((always_inline))
-static inline char *allocate_outside_nursery_small(uint64_t size)
+static inline stm_char *allocate_outside_nursery_small(uint64_t size)
 {
     OPT_ASSERT((size & 7) == 0);
     OPT_ASSERT(16 <= size && size <= GC_LAST_SMALL_SIZE);
@@ -161,16 +162,28 @@
     struct small_free_loc_s *result = *fl;
 
     if (UNLIKELY(result == NULL))
-        return _allocate_small_slowpath(size);
+        return (stm_char*)
+            (_allocate_small_slowpath(size) - stm_file_pages + END_NURSERY_PAGE * 4096UL);
 
     *fl = result->next;
-    return (char *)result;
+    return (stm_char*)
+        ((char *)result - stm_file_pages + END_NURSERY_PAGE * 4096UL);
 }
 
 object_t *_stm_allocate_old_small(ssize_t size_rounded_up)
 {
-    char *p = allocate_outside_nursery_small(size_rounded_up);
-    return (object_t *)(p - stm_object_pages);
+    stm_char *p = allocate_outside_nursery_small(size_rounded_up);
+    memset(stm_object_pages + (uintptr_t)p, 0, size_rounded_up);
+
+    object_t *o = (object_t *)p;
+    o->stm_flags = GCFLAG_WRITE_BARRIER;
+
+    dprintf(("allocate_old_small(%lu): %p, seg=%d, page=%lu\n",
+             size_rounded_up, p,
+             get_segment_of_linear_address(stm_object_pages + (uintptr_t)p),
+             (uintptr_t)p / 4096UL));
+
+    return o;
 }
 
 /************************************************************/
@@ -178,8 +191,11 @@
 static inline bool _smallmalloc_sweep_keep(char *p)
 {
 #ifdef STM_TESTS
-    if (_stm_smallmalloc_keep != NULL)
-        return _stm_smallmalloc_keep(p);
+    if (_stm_smallmalloc_keep != NULL) {
+        // test wants a TLPREFIXd address
+        return _stm_smallmalloc_keep(
+            p - stm_file_pages + (char*)(END_NURSERY_PAGE * 4096UL));
+    }
 #endif
     abort();
     //return smallmalloc_keep_object_at(p);
diff --git a/c8/stm/smallmalloc.h b/c8/stm/smallmalloc.h
--- a/c8/stm/smallmalloc.h
+++ b/c8/stm/smallmalloc.h
@@ -54,7 +54,7 @@
 
 /* Functions
  */
-static inline char *allocate_outside_nursery_small(uint64_t size)
+static inline stm_char *allocate_outside_nursery_small(uint64_t size)
      __attribute__((always_inline));
 
 void _stm_smallmalloc_sweep(void);
diff --git a/c8/stmgc.h b/c8/stmgc.h
--- a/c8/stmgc.h
+++ b/c8/stmgc.h
@@ -80,6 +80,7 @@
 void _push_obj_to_other_segments(object_t *obj);
 
 char *stm_object_pages;
+char *stm_file_pages;
 object_t *_stm_allocate_old_small(ssize_t size_rounded_up);
 bool (*_stm_smallmalloc_keep)(char *data);
 void _stm_smallmalloc_sweep(void);
diff --git a/c8/test/support.py b/c8/test/support.py
--- a/c8/test/support.py
+++ b/c8/test/support.py
@@ -34,6 +34,7 @@
 } stm_thread_local_t;
 
 char *stm_object_pages;
+char *stm_file_pages;
 
 void stm_read(object_t *obj);
 /*void stm_write(object_t *obj); use _checked_stm_write() instead */
diff --git a/c8/test/test_smallmalloc.py b/c8/test/test_smallmalloc.py
--- a/c8/test/test_smallmalloc.py
+++ b/c8/test/test_smallmalloc.py
@@ -12,7 +12,7 @@
         BaseTest.setup_method(self, method)
         @ffi.callback("bool(char *)")
         def keep(data):
-            p = ffi.cast("object_t *", data - lib.stm_object_pages)
+            p = ffi.cast("object_t *", data)
             self.has_been_asked_for.append(p)
             return p in self.keep_me
         lib._stm_smallmalloc_keep = keep

From noreply at buildbot.pypy.org  Wed Nov 12 11:32:01 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Wed, 12 Nov 2014 11:32:01 +0100 (CET)
Subject: [pypy-commit] pypy stmgc-c7: Hopefully finish the RPython
 interfacing with stm hashtables.
Message-ID: <20141112103201.CCB891D288C@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: stmgc-c7
Changeset: r74475:34d87a99d19a
Date: 2014-11-12 11:31 +0100
http://bitbucket.org/pypy/pypy/changeset/34d87a99d19a/

Log:	Hopefully finish the RPython interfacing with stm hashtables.

diff --git a/rpython/memory/gctransform/stmframework.py b/rpython/memory/gctransform/stmframework.py
--- a/rpython/memory/gctransform/stmframework.py
+++ b/rpython/memory/gctransform/stmframework.py
@@ -7,6 +7,7 @@
 from rpython.memory.gctypelayout import WEAKREF, WEAKREFPTR
 from rpython.memory.gc.stmgc import StmGC
 from rpython.rtyper import rmodel, llannotation
+from rpython.rtyper.annlowlevel import llhelper
 from rpython.translator.backendopt.support import var_needsgc
 from rpython.rlib.objectmodel import specialize
 from rpython.rlib import rstm
diff --git a/rpython/rlib/rstm.py b/rpython/rlib/rstm.py
--- a/rpython/rlib/rstm.py
+++ b/rpython/rlib/rstm.py
@@ -193,7 +193,12 @@
                                  adtmeths={'get': ll_hashtable_get,
                                            'set': ll_hashtable_set})
 
-#def ll_hashtable_trace(...)
+def ll_hashtable_trace(gc, obj, callback, arg):
+    from rpython.memory.gctransform.stmframework import get_visit_function
+    visit_fn = get_visit_function(callback, arg)
+    addr = obj + llmemory.offsetof(_HASHTABLE_OBJ, 'll_raw_hashtable')
+    llop.stm_hashtable_tracefn(lltype.Void, addr.address[0], visit_fn)
+lambda_hashtable_trace = lambda: ll_hashtable_trace
 
 def create_hashtable():
     if not we_are_translated():
@@ -206,6 +211,7 @@
         p = lltype.malloc(_STM_HASHTABLE_ENTRY)
     else:
         p = lltype.nullptr(_STM_HASHTABLE_ENTRY)
+    rgc.register_custom_trace_hook(_HASHTABLE_OBJ, lambda_hashtable_trace)
     h = lltype.malloc(_HASHTABLE_OBJ)
     h.ll_raw_hashtable = llop.stm_hashtable_create(_STM_HASHTABLE_P, p)
     return h
diff --git a/rpython/rtyper/annlowlevel.py b/rpython/rtyper/annlowlevel.py
--- a/rpython/rtyper/annlowlevel.py
+++ b/rpython/rtyper/annlowlevel.py
@@ -516,7 +516,7 @@
 @specialize.arg(0)
 def cast_gcref_to_instance(Class, ptr):
     """Reverse the hacking done in cast_instance_to_gcref()."""
-    from rpython.rtyper.lltypesystem.rclass import OBJECTPTR
+    from rpython.rtyper.rclass import OBJECTPTR
     ptr = lltype.cast_opaque_ptr(OBJECTPTR, ptr)
     return cast_base_ptr_to_instance(Class, ptr)
 
diff --git a/rpython/rtyper/lltypesystem/lloperation.py b/rpython/rtyper/lltypesystem/lloperation.py
--- a/rpython/rtyper/lltypesystem/lloperation.py
+++ b/rpython/rtyper/lltypesystem/lloperation.py
@@ -464,6 +464,7 @@
     'stm_hashtable_free':     LLOp(),
     'stm_hashtable_read':     LLOp(),
     'stm_hashtable_write':    LLOp(),
+    'stm_hashtable_tracefn':  LLOp(),
 
     # __________ address operations __________
 
diff --git a/rpython/translator/stm/funcgen.py b/rpython/translator/stm/funcgen.py
--- a/rpython/translator/stm/funcgen.py
+++ b/rpython/translator/stm/funcgen.py
@@ -311,3 +311,8 @@
     arg3 = funcgen.expr(op.args[3])
     return ('stm_hashtable_write((object_t *)%s, %s, %s, (object_t *)%s, '
             '&stm_thread_local);' % (arg0, arg1, arg2, arg3))
+
+def stm_hashtable_tracefn(funcgen, op):
+    arg0 = funcgen.expr(op.args[0])
+    arg1 = funcgen.expr(op.args[1])
+    return 'stm_hashtable_tracefn((stm_hashtable_t *)%s, %s);' % (arg0, arg1)
diff --git a/rpython/translator/stm/test/test_ztranslated.py b/rpython/translator/stm/test/test_ztranslated.py
--- a/rpython/translator/stm/test/test_ztranslated.py
+++ b/rpython/translator/stm/test/test_ztranslated.py
@@ -553,8 +553,6 @@
         assert '' in err
 
     def test_hashtable(self):
-        py.test.skip("missing: custom tracer on Hashtable")
-
         class X(object):
             pass
 

From noreply at buildbot.pypy.org  Wed Nov 12 11:36:53 2014
From: noreply at buildbot.pypy.org (Raemi)
Date: Wed, 12 Nov 2014 11:36:53 +0100 (CET)
Subject: [pypy-commit] stmgc c8-small-uniform: fix
Message-ID: <20141112103653.60DB81D291D@cobra.cs.uni-duesseldorf.de>

Author: Remi Meier 
Branch: c8-small-uniform
Changeset: r1507:d24aeecfe68c
Date: 2014-11-12 11:37 +0100
http://bitbucket.org/pypy/stmgc/changeset/d24aeecfe68c/

Log:	fix

diff --git a/c8/stm/nursery.c b/c8/stm/nursery.c
--- a/c8/stm/nursery.c
+++ b/c8/stm/nursery.c
@@ -197,7 +197,9 @@
         lst = STM_PSEGMENT->objects_pointing_to_nursery;
     }
 
+    acquire_privatization_lock(STM_SEGMENT->segment_num);
     synchronize_objects_flush();
+    release_privatization_lock(STM_SEGMENT->segment_num);
 }
 
 

From noreply at buildbot.pypy.org  Wed Nov 12 11:46:39 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Wed, 12 Nov 2014 11:46:39 +0100 (CET)
Subject: [pypy-commit] pypy stmgc-c7: Add _stm.hashtable().
Message-ID: <20141112104639.C4A341C0548@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: stmgc-c7
Changeset: r74476:f0c0bada6f6a
Date: 2014-11-12 11:46 +0100
http://bitbucket.org/pypy/pypy/changeset/f0c0bada6f6a/

Log:	Add _stm.hashtable().

diff --git a/pypy/module/_stm/__init__.py b/pypy/module/_stm/__init__.py
--- a/pypy/module/_stm/__init__.py
+++ b/pypy/module/_stm/__init__.py
@@ -9,4 +9,5 @@
     interpleveldefs = {
         'local': 'local.STMLocal',
         'count': 'count.count',
+        'hashtable': 'hashtable.W_Hashtable',
     }
diff --git a/pypy/module/_stm/hashtable.py b/pypy/module/_stm/hashtable.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/_stm/hashtable.py
@@ -0,0 +1,47 @@
+"""
+The class _stm.hashtable, mapping integers to objects.
+"""
+
+from pypy.interpreter.baseobjspace import W_Root
+from pypy.interpreter.typedef import TypeDef
+from pypy.interpreter.gateway import interp2app, unwrap_spec
+
+from rpython.rlib import rstm
+from rpython.rtyper.annlowlevel import cast_gcref_to_instance
+from rpython.rtyper.annlowlevel import cast_instance_to_gcref
+
+
+class W_Hashtable(W_Root):
+
+    def __init__(self):
+        self.h = rstm.create_hashtable()
+
+    @unwrap_spec(key=int)
+    def getitem_w(self, space, key):
+        gcref = self.h.get(key)
+        if not gcref:
+            space.raise_key_error(space.wrap(key))
+        return cast_gcref_to_instance(W_Root, gcref)
+
+    @unwrap_spec(key=int)
+    def setitem_w(self, key, w_value):
+        gcref = cast_instance_to_gcref(w_value)
+        self.h.set(key, gcref)
+
+    @unwrap_spec(key=int)
+    def delitem_w(self, key):
+        self.h.set(key, rstm.NULL_GCREF)
+
+
+def W_Hashtable___new__(space, w_subtype):
+    r = space.allocate_instance(W_Hashtable, w_subtype)
+    r.__init__()
+    return space.wrap(r)
+
+W_Hashtable.typedef = TypeDef(
+    '_stm.hashtable',
+    __new__ = interp2app(W_Hashtable___new__),
+    __getitem__ = interp2app(W_Hashtable.getitem_w),
+    __setitem__ = interp2app(W_Hashtable.setitem_w),
+    __delitem__ = interp2app(W_Hashtable.delitem_w),
+    )
diff --git a/pypy/module/_stm/test/test_hashtable.py b/pypy/module/_stm/test/test_hashtable.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/_stm/test/test_hashtable.py
@@ -0,0 +1,15 @@
+
+
+class AppTestHashtable:
+    spaceconfig = dict(usemodules=['_stm'])
+
+    def test_simple(self):
+        import _stm
+        h = _stm.hashtable()
+        h[42+65536] = "bar"
+        raises(KeyError, "h[42]")
+        h[42] = "foo"
+        assert h[42] == "foo"
+        del h[42]
+        raises(KeyError, "h[42]")
+        assert h[42+65536] == "bar"
diff --git a/rpython/rlib/rstm.py b/rpython/rlib/rstm.py
--- a/rpython/rlib/rstm.py
+++ b/rpython/rlib/rstm.py
@@ -216,9 +216,9 @@
     h.ll_raw_hashtable = llop.stm_hashtable_create(_STM_HASHTABLE_P, p)
     return h
 
+NULL_GCREF = lltype.nullptr(llmemory.GCREF.TO)
+
 class HashtableForTest(object):
-    _NULL = lltype.nullptr(llmemory.GCREF.TO)
-
     def __init__(self):
         self._content = {}      # dict {integer: GCREF}
 
@@ -227,7 +227,7 @@
 
     def get(self, key):
         assert type(key) is int
-        return self._content.get(key, self._NULL)
+        return self._content.get(key, NULL_GCREF)
 
     def set(self, key, value):
         assert type(key) is int

From noreply at buildbot.pypy.org  Wed Nov 12 12:04:07 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Wed, 12 Nov 2014 12:04:07 +0100 (CET)
Subject: [pypy-commit] pypy stmgc-c7: various fixes until llsupport/test/*
	pass again
Message-ID: <20141112110407.8FEE71C0548@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: stmgc-c7
Changeset: r74477:67cf1848a979
Date: 2014-11-12 12:03 +0100
http://bitbucket.org/pypy/pypy/changeset/67cf1848a979/

Log:	various fixes until llsupport/test/* pass again

diff --git a/rpython/jit/backend/llsupport/rewrite.py b/rpython/jit/backend/llsupport/rewrite.py
--- a/rpython/jit/backend/llsupport/rewrite.py
+++ b/rpython/jit/backend/llsupport/rewrite.py
@@ -70,7 +70,7 @@
                 continue
             if op.is_guard():
                 self.emit_pending_zeros()
-            elif op.can_malloc():
+            elif op.is_call():
                 self.emitting_an_operation_that_can_collect()
             elif op.getopnum() == rop.LABEL:
                 self.emitting_an_operation_that_can_collect()
@@ -547,7 +547,7 @@
 
     def must_apply_write_barrier(self, val, v):
         if val not in self.write_barrier_applied:
-            if isinstance(v, BoxPtr) or (isinstance(v, ConstPtr) and
+            if (isinstance(v, BoxPtr) or (isinstance(v, ConstPtr) and
                                rgc.needs_write_barrier(v.value))):
                 return True
         return False
diff --git a/rpython/jit/backend/llsupport/stmrewrite.py b/rpython/jit/backend/llsupport/stmrewrite.py
--- a/rpython/jit/backend/llsupport/stmrewrite.py
+++ b/rpython/jit/backend/llsupport/stmrewrite.py
@@ -84,7 +84,7 @@
             return
         # ----------  other ignored ops  ----------
         if opnum in (rop.STM_SHOULD_BREAK_TRANSACTION, rop.FORCE_TOKEN,
-                     rop.READ_TIMESTAMP, rop.MARK_OPAQUE_PTR,
+                     rop.MARK_OPAQUE_PTR,
                      rop.JIT_DEBUG, rop.KEEPALIVE,
                      rop.QUASIIMMUT_FIELD, rop.RECORD_KNOWN_CLASS,
                      ):
diff --git a/rpython/jit/backend/llsupport/test/test_rewrite.py b/rpython/jit/backend/llsupport/test/test_rewrite.py
--- a/rpython/jit/backend/llsupport/test/test_rewrite.py
+++ b/rpython/jit/backend/llsupport/test/test_rewrite.py
@@ -103,7 +103,6 @@
         signedframedescr = self.cpu.signedframedescr
         floatframedescr = self.cpu.floatframedescr
         casmdescr.compiled_loop_token = clt
-        tzdescr = None # noone cares
         #
         namespace.update(locals())
         #
diff --git a/rpython/jit/backend/llsupport/test/test_stmrewrite.py b/rpython/jit/backend/llsupport/test/test_stmrewrite.py
--- a/rpython/jit/backend/llsupport/test/test_stmrewrite.py
+++ b/rpython/jit/backend/llsupport/test/test_stmrewrite.py
@@ -4,7 +4,8 @@
 from rpython.jit.metainterp import resoperation
 from rpython.jit.backend.llsupport.test.test_rewrite import (
     RewriteTests, BaseFakeCPU)
-from rpython.rtyper.lltypesystem import lltype, rclass, rffi, llmemory
+from rpython.rtyper.lltypesystem import lltype, rffi, llmemory
+from rpython.rtyper import rclass
 
 
 def test_all_operations_with_gc_in_their_name():
@@ -72,6 +73,7 @@
         for name, value in self.gc_ll_descr.__dict__.items():
             if name.endswith('descr') and name[1] == '2' and len(name) == 8:
                 namespace[name] = value     # "X2Ydescr"
+        self.gc_ll_descr.malloc_zero_filled = True
         RewriteTests.check_rewrite(self, frm_operations, to_operations,
                                    **namespace)
 
@@ -340,7 +342,6 @@
             "i3 = int_add(i1, i2)",   # all pure operations
             "f3 = float_abs(f1)",
             "i3 = force_token()",
-            "i3 = read_timestamp()",
             "i3 = mark_opaque_ptr(p1)",
             "jit_debug(i1, i2)",
             "keepalive(i1)",
@@ -546,14 +547,14 @@
         self.check_rewrite("""
             [p1, p2, i2, p3, i3]
             setarrayitem_gc(p1, i2, p2, descr=adescr) #noptr
-            i4 = read_timestamp()
+            i4 = force_token()
             setarrayitem_gc(p1, i3, p3, descr=adescr) #noptr
             jump()
         """, """
             [p1, p2, i2, p3, i3]
             cond_call_gc_wb_array(p1, i2, descr=wbdescr)
             setarrayitem_gc(p1, i2, p2, descr=adescr)
-            i4 = read_timestamp()
+            i4 = force_token()
             cond_call_gc_wb_array(p1, i3, descr=wbdescr)
             setarrayitem_gc(p1, i3, p3, descr=adescr)
             $DUMMYALLOC
@@ -564,14 +565,14 @@
         self.check_rewrite("""
             [p1, p2, i2, p3, i3]
             setinteriorfield_gc(p1, i2, p2, descr=intzdescr)
-            i4 = read_timestamp()
+            i4 = force_token()
             setinteriorfield_gc(p1, i3, p3, descr=intzdescr)
             jump()
         """, """
             [p1, p2, i2, p3, i3]
             cond_call_gc_wb_array(p1, i2, descr=wbdescr)
             setinteriorfield_gc(p1, i2, p2, descr=intzdescr)
-            i4 = read_timestamp()
+            i4 = force_token()
             cond_call_gc_wb_array(p1, i3, descr=wbdescr)
             setinteriorfield_gc(p1, i3, p3, descr=intzdescr)
             $DUMMYALLOC
diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py
--- a/rpython/jit/backend/x86/assembler.py
+++ b/rpython/jit/backend/x86/assembler.py
@@ -1213,7 +1213,7 @@
                 if shadowstack_reg is None:
                     mc.MOV(ecx, self.heap_shadowstack_top())
                     shadowstack_reg = ecx
-                mc.MOV(ebp, mem(shadowstack_reg, -WORD))
+                mc.MOV(ebp, mem(self.SEGMENT_NO, shadowstack_reg, -WORD))
         wbdescr = self.cpu.gc_ll_descr.write_barrier_descr
         if gcrootmap and wbdescr:
             # frame never uses card marking, so we enforce this is not

From noreply at buildbot.pypy.org  Wed Nov 12 12:36:24 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Wed, 12 Nov 2014 12:36:24 +0100 (CET)
Subject: [pypy-commit] pypy stmgc-c7: Kill special lock support for now
Message-ID: <20141112113624.D5EBE1C3A61@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: stmgc-c7
Changeset: r74478:b19fcc6ab1a2
Date: 2014-11-12 12:17 +0100
http://bitbucket.org/pypy/pypy/changeset/b19fcc6ab1a2/

Log:	Kill special lock support for now

diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -714,11 +714,7 @@
         if not we_are_translated() and self.config.translating:
             raise CannotHaveLock()
         try:
-            if self.config.translation.stm:
-                from pypy.module.thread import stm
-                return stm.allocate_stm_lock(self)
-            else:
-                return rthread.allocate_lock()
+            return rthread.allocate_lock()
         except rthread.error:
             raise OperationError(self.w_RuntimeError,
                                  self.wrap("out of resources"))
diff --git a/pypy/module/_stm/lock.py b/pypy/module/_stm/lock.py
deleted file mode 100644
--- a/pypy/module/_stm/lock.py
+++ /dev/null
@@ -1,21 +0,0 @@
-from pypy.module.thread.error import wrap_thread_error
-
-
-class STMLock(rthread.Lock):
-    def __init__(self, space, ll_lock):
-        rthread.Lock.__init__(self, ll_lock)
-        self.space = space
-
-    def acquire(self, flag):
-        if rstm.is_atomic():
-            acquired = rthread.Lock.acquire(self, False)
-            if flag and not acquired:
-                raise wrap_thread_error(self.space,
-                    "deadlock: an atomic transaction tries to acquire "
-                    "a lock that is already acquired.  See pypy/doc/stm.rst.")
-        else:
-            acquired = rthread.Lock.acquire(self, flag)
-        return acquired
-
-def allocate_stm_lock(space):
-    return STMLock(space, rthread.allocate_ll_lock())

From noreply at buildbot.pypy.org  Wed Nov 12 12:50:05 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Wed, 12 Nov 2014 12:50:05 +0100 (CET)
Subject: [pypy-commit] pypy stmgc-c7: fix
Message-ID: <20141112115005.47B321C3474@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: stmgc-c7
Changeset: r74479:6c9902ffa056
Date: 2014-11-12 12:48 +0100
http://bitbucket.org/pypy/pypy/changeset/6c9902ffa056/

Log:	fix

diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py
--- a/rpython/jit/backend/x86/assembler.py
+++ b/rpython/jit/backend/x86/assembler.py
@@ -2736,7 +2736,8 @@
         null_reg_cleared = False
         i = 0
         while i < nbytes:
-            addr = addr_add(base_loc, startindex_loc, baseofs + i, scale)
+            addr = addr_add(self.SEGMENT_GC, base_loc, startindex_loc,
+                            baseofs + i, scale)
             current = nbytes - i
             if current >= 16:
                 current = 16

From noreply at buildbot.pypy.org  Wed Nov 12 13:41:24 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Wed, 12 Nov 2014 13:41:24 +0100 (CET)
Subject: [pypy-commit] pypy stmgc-c7: fixes
Message-ID: <20141112124124.174D91C093D@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: stmgc-c7
Changeset: r74480:5018a9c8a064
Date: 2014-11-12 13:24 +0100
http://bitbucket.org/pypy/pypy/changeset/5018a9c8a064/

Log:	fixes

diff --git a/rpython/jit/backend/x86/callbuilder.py b/rpython/jit/backend/x86/callbuilder.py
--- a/rpython/jit/backend/x86/callbuilder.py
+++ b/rpython/jit/backend/x86/callbuilder.py
@@ -109,7 +109,7 @@
         if gcrootmap:
             if gcrootmap.is_shadow_stack:
                 rst = gcrootmap.get_root_stack_top_addr()
-                self.mc.MOV(ebx, heap(rst))
+                self.mc.MOV(ebx, self.asm.heap_tl(rst))
         #
         if self.asm.cpu.gc_ll_descr.stm:
             self.call_stm_before_ex_call()
@@ -207,12 +207,12 @@
             # here, ecx is zero (so rpy_fastgil was not acquired)
             rst = gcrootmap.get_root_stack_top_addr()
             mc = self.mc
-            mc.CMP(ebx, heap(rst))
+            mc.CMP(ebx, self.asm.heap_tl(rst))
             mc.J_il8(rx86.Conditions['E'], 0)
             je_location = mc.get_relative_pos()
             # revert the rpy_fastgil acquired above, so that the
             # general 'reacqgil_addr' below can acquire it again...
-            mc.MOV(heap(fastgil), ecx)
+            mc.MOV(heap(self.asm.SEGMENT_NO, fastgil), ecx)
             # patch the JNE above
             offset = mc.get_relative_pos() - jne_location
             assert 0 < offset <= 127
diff --git a/rpython/jit/backend/x86/test/test_stm_integration.py b/rpython/jit/backend/x86/test/test_stm_integration.py
--- a/rpython/jit/backend/x86/test/test_stm_integration.py
+++ b/rpython/jit/backend/x86/test/test_stm_integration.py
@@ -1,5 +1,6 @@
 import py
-from rpython.rtyper.lltypesystem import lltype, llmemory, rffi, rstr, rclass
+from rpython.rtyper.lltypesystem import lltype, llmemory, rffi, rstr
+from rpython.rtyper import rclass
 from rpython.jit.metainterp.history import ResOperation, TargetToken,\
      JitCellToken
 from rpython.jit.metainterp.history import (BoxInt, BoxPtr, ConstInt,

From noreply at buildbot.pypy.org  Wed Nov 12 14:45:14 2014
From: noreply at buildbot.pypy.org (Raemi)
Date: Wed, 12 Nov 2014 14:45:14 +0100 (CET)
Subject: [pypy-commit] stmgc c8-small-uniform: another small fix
Message-ID: <20141112134514.94DA91C3641@cobra.cs.uni-duesseldorf.de>

Author: Remi Meier 
Branch: c8-small-uniform
Changeset: r1508:f9126a9cb713
Date: 2014-11-12 14:02 +0100
http://bitbucket.org/pypy/stmgc/changeset/f9126a9cb713/

Log:	another small fix

diff --git a/c8/stm/core.c b/c8/stm/core.c
--- a/c8/stm/core.c
+++ b/c8/stm/core.c
@@ -984,6 +984,7 @@
     OPT_ASSERT(obj_size >= 16);
 
     if (LIKELY(is_small_uniform(obj))) {
+        OPT_ASSERT(obj_size <= GC_LAST_SMALL_SIZE);
         _synchronize_fragment((stm_char *)obj, obj_size);
         return;
     }
diff --git a/c8/stm/smallmalloc.c b/c8/stm/smallmalloc.c
--- a/c8/stm/smallmalloc.c
+++ b/c8/stm/smallmalloc.c
@@ -60,7 +60,7 @@
             goto out_of_memory;
 
         uninitialized_page_stop -= decrease_by;
-        first_small_uniform_loc = uninitialized_page_stop - stm_file_pages;
+        first_small_uniform_loc = uninitialized_page_stop - stm_file_pages + END_NURSERY_PAGE * 4096UL;
 
         /* XXX: */
         /* char *base = stm_object_pages + END_NURSERY_PAGE * 4096UL; */

From noreply at buildbot.pypy.org  Wed Nov 12 16:09:38 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Wed, 12 Nov 2014 16:09:38 +0100 (CET)
Subject: [pypy-commit] pypy stmgc-c7: Temporarily ignore the
 @must_be_light_finalizer if it causes troubles
Message-ID: <20141112150938.F02141C0548@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: stmgc-c7
Changeset: r74481:eb9446d4bb2d
Date: 2014-11-12 15:21 +0100
http://bitbucket.org/pypy/pypy/changeset/eb9446d4bb2d/

Log:	Temporarily ignore the @must_be_light_finalizer if it causes
	troubles for STM

diff --git a/rpython/translator/backendopt/finalizer.py b/rpython/translator/backendopt/finalizer.py
--- a/rpython/translator/backendopt/finalizer.py
+++ b/rpython/translator/backendopt/finalizer.py
@@ -24,6 +24,12 @@
         result = self.analyze_direct_call(graph)
         if (result is self.top_result() and
             getattr(graph.func, '_must_be_light_finalizer_', False)):
+            # XXX temporary
+            if self.translator.config.translation.stm:
+                from rpython.memory.gctransform.log import log
+                log.WARNING("%s: %r" % (FinalizerError.__doc__, graph))
+                return result
+            #
             raise FinalizerError(FinalizerError.__doc__, graph)
         return result
 

From noreply at buildbot.pypy.org  Wed Nov 12 16:09:40 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Wed, 12 Nov 2014 16:09:40 +0100 (CET)
Subject: [pypy-commit] pypy stmgc-c7: Add _stm.time() and _stm.clock().
Message-ID: <20141112150940.2CE601C0548@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: stmgc-c7
Changeset: r74482:316291d870ae
Date: 2014-11-12 15:57 +0100
http://bitbucket.org/pypy/pypy/changeset/316291d870ae/

Log:	Add _stm.time() and _stm.clock().

diff --git a/pypy/module/_stm/__init__.py b/pypy/module/_stm/__init__.py
--- a/pypy/module/_stm/__init__.py
+++ b/pypy/module/_stm/__init__.py
@@ -10,4 +10,6 @@
         'local': 'local.STMLocal',
         'count': 'count.count',
         'hashtable': 'hashtable.W_Hashtable',
+        'time': 'time.time',
+        'clock': 'time.clock',
     }
diff --git a/pypy/module/_stm/test/test_time.py b/pypy/module/_stm/test/test_time.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/_stm/test/test_time.py
@@ -0,0 +1,13 @@
+
+
+class AppTestHashtable:
+    spaceconfig = dict(usemodules=['_stm'])
+
+    def test_simple(self):
+        import _stm
+        t1 = _stm.time()
+        t2 = _stm.time()
+        assert t1 < t2 < t1 + 1
+        t1 = _stm.clock()
+        t2 = _stm.clock()
+        assert t1 < t2 < t1 + 1
diff --git a/pypy/module/_stm/time.py b/pypy/module/_stm/time.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/_stm/time.py
@@ -0,0 +1,56 @@
+"""
+_stm.time(), _stm.clock()
+"""
+
+from rpython.rtyper.lltypesystem import lltype, rffi
+from rpython.translator.tool.cbuild import ExternalCompilationInfo
+
+
+# Linux-only for now
+
+eci = ExternalCompilationInfo(
+    includes=["time.h"],
+    libraries=["rt"],
+    post_include_bits = ["""
+double pypy_clock_get_time(void);
+double pypy_clock_get_clock(void);
+"""],
+    separate_module_sources = ["""
+double pypy_clock_get_time(void) {
+    struct timespec t = {-1, 0};
+    clock_gettime(CLOCK_REALTIME, &t);
+    return t.tv_sec + 0.000000001 * t.tv_nsec;
+}
+double pypy_clock_get_clock(void) {
+    struct timespec t = {-1, 0};
+    clock_gettime(CLOCK_PROCESS_CPUTIME_ID, &t);
+    return t.tv_sec + 0.000000001 * t.tv_nsec;
+}
+"""])
+
+
+pypy_clock_get_time = rffi.llexternal('pypy_clock_get_time',
+                                      [], lltype.Float,
+                                      compilation_info=eci,
+                                      releasegil=False, transactionsafe=True)
+pypy_clock_get_clock = rffi.llexternal('pypy_clock_get_clock',
+                                       [], lltype.Float,
+                                       compilation_info=eci,
+                                       releasegil=False, transactionsafe=True)
+
+
+def time(space):
+    """Similar to time.time(), but works without conflict.
+The drawback is that the returned times may appear out of order:
+this thread's transaction may commit before or after another thread's,
+while _stm.time() called by both may return results in the opposite
+order (or even exactly equal results if you are unlucky)."""
+    return space.wrap(pypy_clock_get_time())
+
+def clock(space):
+    """Similar to time.clock(), but works without conflict.
+The drawback is that the returned times may appear out of order:
+this thread's transaction may commit before or after another thread's,
+while _stm.time() called by both may return results in the opposite
+order (or even exactly equal results if you are unlucky)."""
+    return space.wrap(pypy_clock_get_clock())
diff --git a/rpython/rtyper/module/ll_time.py b/rpython/rtyper/module/ll_time.py
--- a/rpython/rtyper/module/ll_time.py
+++ b/rpython/rtyper/module/ll_time.py
@@ -85,13 +85,11 @@
             if self.GETTIMEOFDAY_NO_TZ:
                 c_gettimeofday = self.llexternal('gettimeofday',
                                  [self.TIMEVALP], rffi.INT,
-                                  _nowrapper=True, releasegil=False,
-                                  transactionsafe=True)
+                                  _nowrapper=True, releasegil=False)
             else:
                 c_gettimeofday = self.llexternal('gettimeofday',
                                  [self.TIMEVALP, rffi.VOIDP], rffi.INT,
-                                  _nowrapper=True, releasegil=False,
-                                  transactionsafe=True)
+                                  _nowrapper=True, releasegil=False)
             c_ftime = None # We have gettimeofday(2), so force ftime(3) OFF.
         else:
             c_gettimeofday = None
@@ -101,14 +99,12 @@
                 self.configure(CConfigForFTime)
                 c_ftime = self.llexternal(FTIME, [lltype.Ptr(self.TIMEB)],
                                           lltype.Void,
-                                          _nowrapper=True, releasegil=False,
-                                          transactionsafe=True)
+                                          _nowrapper=True, releasegil=False)
             else:
                 c_ftime = None    # to not confuse the flow space
 
         c_time = self.llexternal('time', [rffi.VOIDP], rffi.TIME_T,
-                                 _nowrapper=True, releasegil=False,
-                                 transactionsafe=True)
+                                 _nowrapper=True, releasegil=False)
 
         def time_time_llimpl():
             void = lltype.nullptr(rffi.VOIDP.TO)
@@ -146,10 +142,10 @@
             A = lltype.FixedSizeArray(lltype.SignedLongLong, 1)
             QueryPerformanceCounter = self.llexternal(
                 'QueryPerformanceCounter', [lltype.Ptr(A)], lltype.Void,
-                releasegil=False, transactionsafe=True)
+                releasegil=False)
             QueryPerformanceFrequency = self.llexternal(
                 'QueryPerformanceFrequency', [lltype.Ptr(A)], rffi.INT,
-                releasegil=False, transactionsafe=True)
+                releasegil=False)
             class State(object):
                 pass
             state = State()
@@ -172,8 +168,7 @@
             c_getrusage = self.llexternal('getrusage', 
                                           [rffi.INT, lltype.Ptr(RUSAGE)],
                                           lltype.Void,
-                                          releasegil=False,
-                                          transactionsafe=True)
+                                          releasegil=False)
             def time_clock_llimpl():
                 a = lltype.malloc(RUSAGE, flavor='raw')
                 c_getrusage(RUSAGE_SELF, a)

From noreply at buildbot.pypy.org  Wed Nov 12 16:09:41 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Wed, 12 Nov 2014 16:09:41 +0100 (CET)
Subject: [pypy-commit] pypy stmgc-c7: Use 'extern' here,
 just to make it easier to find after we merge with the current
 default.
Message-ID: <20141112150941.550241C0548@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: stmgc-c7
Changeset: r74483:3fc3f50e2523
Date: 2014-11-12 16:09 +0100
http://bitbucket.org/pypy/pypy/changeset/3fc3f50e2523/

Log:	Use 'extern' here, just to make it easier to find after we merge
	with the current default.

diff --git a/pypy/module/_stm/time.py b/pypy/module/_stm/time.py
--- a/pypy/module/_stm/time.py
+++ b/pypy/module/_stm/time.py
@@ -12,8 +12,8 @@
     includes=["time.h"],
     libraries=["rt"],
     post_include_bits = ["""
-double pypy_clock_get_time(void);
-double pypy_clock_get_clock(void);
+extern double pypy_clock_get_time(void);
+extern double pypy_clock_get_clock(void);
 """],
     separate_module_sources = ["""
 double pypy_clock_get_time(void) {

From noreply at buildbot.pypy.org  Wed Nov 12 18:02:50 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Wed, 12 Nov 2014 18:02:50 +0100 (CET)
Subject: [pypy-commit] pypy stmgc-c7: fix
Message-ID: <20141112170250.F39A41D288C@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: stmgc-c7
Changeset: r74484:f43c46bcf4be
Date: 2014-11-12 17:01 +0100
http://bitbucket.org/pypy/pypy/changeset/f43c46bcf4be/

Log:	fix

diff --git a/rpython/rtyper/lltypesystem/rstr.py b/rpython/rtyper/lltypesystem/rstr.py
--- a/rpython/rtyper/lltypesystem/rstr.py
+++ b/rpython/rtyper/lltypesystem/rstr.py
@@ -1251,6 +1251,14 @@
         SRC = typeOf(src).TO     # STR or UNICODE
         DST = typeOf(dst).TO     # GcArray
         assert DST.OF is SRC.chars.OF
+
+        if rgc.stm_is_enabled():
+            i = 0
+            while i < length:
+                dst[i] = src.chars[i]
+                i += 1
+            return lst
+
         # from here, no GC operations can happen
         asrc = llmemory.cast_ptr_to_adr(src) + (
             llmemory.offsetof(SRC, 'chars') +

From noreply at buildbot.pypy.org  Wed Nov 12 18:02:52 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Wed, 12 Nov 2014 18:02:52 +0100 (CET)
Subject: [pypy-commit] pypy stmgc-c7: Test and fix
Message-ID: <20141112170252.29AC21D288C@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: stmgc-c7
Changeset: r74485:5304fa92e01f
Date: 2014-11-12 18:02 +0100
http://bitbucket.org/pypy/pypy/changeset/5304fa92e01f/

Log:	Test and fix

diff --git a/rpython/rlib/rstm.py b/rpython/rlib/rstm.py
--- a/rpython/rlib/rstm.py
+++ b/rpython/rlib/rstm.py
@@ -200,6 +200,8 @@
     llop.stm_hashtable_tracefn(lltype.Void, addr.address[0], visit_fn)
 lambda_hashtable_trace = lambda: ll_hashtable_trace
 
+_false = CDefinedIntSymbolic('0', default=0)    # remains in the C code
+
 def create_hashtable():
     if not we_are_translated():
         return HashtableForTest()      # for tests
@@ -207,7 +209,7 @@
     # Make sure we see a malloc() of it, so that its typeid is correctly
     # initialized.  It can be done in a NonConstant(False) path so that
     # the C compiler will actually drop it.
-    if NonConstant(False):
+    if _false:
         p = lltype.malloc(_STM_HASHTABLE_ENTRY)
     else:
         p = lltype.nullptr(_STM_HASHTABLE_ENTRY)
diff --git a/rpython/translator/stm/test/test_ztranslated.py b/rpython/translator/stm/test/test_ztranslated.py
--- a/rpython/translator/stm/test/test_ztranslated.py
+++ b/rpython/translator/stm/test/test_ztranslated.py
@@ -584,3 +584,7 @@
         t, cbuilder = self.compile(main)
         data = cbuilder.cmdexec('')
         assert 'ok!\n' in data
+
+        t, cbuilder = self.compile(main, backendopt=True)
+        data = cbuilder.cmdexec('')
+        assert 'ok!\n' in data

From noreply at buildbot.pypy.org  Wed Nov 12 19:43:07 2014
From: noreply at buildbot.pypy.org (rlamy)
Date: Wed, 12 Nov 2014 19:43:07 +0100 (CET)
Subject: [pypy-commit] pypy ssa-flow: Renovate SSA_to_SSI() and use it
 instead of fixeggblocks().
Message-ID: <20141112184307.D11CD1C093D@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy 
Branch: ssa-flow
Changeset: r74486:ee2bf8c11af5
Date: 2013-08-15 04:21 +0100
http://bitbucket.org/pypy/pypy/changeset/ee2bf8c11af5/

Log:	Renovate SSA_to_SSI() and use it instead of fixeggblocks().

	This will allow build_flow() to use the SSA paradigm.

diff --git a/rpython/flowspace/flowcontext.py b/rpython/flowspace/flowcontext.py
--- a/rpython/flowspace/flowcontext.py
+++ b/rpython/flowspace/flowcontext.py
@@ -7,6 +7,7 @@
 import __builtin__
 
 from rpython.tool.error import source_lines
+from rpython.translator.backendopt.ssa import SSA_to_SSI
 from rpython.rlib import rstackovf
 from rpython.flowspace.argument import CallSpec
 from rpython.flowspace.model import (Constant, Variable, Block, Link,
@@ -86,27 +87,7 @@
                 if isinstance(w_value, Variable):
                     w_value.rename(name)
             del block.framestate     # memory saver
-
-    # EggBlocks reuse the variables of their previous block,
-    # which is deemed not acceptable for simplicity of the operations
-    # that will be performed later on the flow graph.
-    for link in list(graph.iterlinks()):
-        block = link.target
-        if isinstance(block, EggBlock):
-            if (not block.operations and len(block.exits) == 1 and
-                link.args == block.inputargs):   # not renamed
-                # if the variables are not renamed across this link
-                # (common case for EggBlocks) then it's easy enough to
-                # get rid of the empty EggBlock.
-                link2 = block.exits[0]
-                link.args = list(link2.args)
-                link.target = link2.target
-                assert link2.exitcase is None
-            else:
-                mapping = {}
-                for a in block.inputargs:
-                    mapping[a] = Variable(a)
-                block.renamevariables(mapping)
+    SSA_to_SSI(graph)
 
 # ____________________________________________________________
 
diff --git a/rpython/translator/backendopt/ssa.py b/rpython/translator/backendopt/ssa.py
--- a/rpython/translator/backendopt/ssa.py
+++ b/rpython/translator/backendopt/ssa.py
@@ -141,11 +141,9 @@
         return entrymap
 
 def variables_created_in(block):
-    result = {}
-    for v in block.inputargs:
-        result[v] = True
+    result = set(block.inputargs)
     for op in block.operations:
-        result[op.result] = True
+        result.add(op.result)
     return result
 
 
@@ -158,28 +156,34 @@
     'graph_or_blocks' can be a graph, or just a dict that lists some blocks
     from a graph, as follows: {block: reachable-from-outside-flag}.
     """
+    seen = set()
+    for link in graph_or_blocks.iterlinks():
+        mapping = {}
+        seen.update(link.args)
+        for arg in link.target.inputargs:
+            if arg in seen and isinstance(arg, Variable):
+                mapping[arg] = Variable(arg)
+        link.target.renamevariables(mapping)
+
     entrymap = mkinsideentrymap(graph_or_blocks)
     builder = DataFlowFamilyBuilder(graph_or_blocks)
     variable_families = builder.get_variable_families()
     del builder
 
     pending = []     # list of (block, var-used-but-not-defined)
-
     for block in entrymap:
         variables_created = variables_created_in(block)
-        variables_used = {}
+        variables_used = set()
         for op in block.operations:
-            for v in op.args:
-                variables_used[v] = True
-        variables_used[block.exitswitch] = True
+            variables_used.update(op.args)
+        variables_used.add(block.exitswitch)
         for link in block.exits:
-            for v in link.args:
-                variables_used[v] = True
+            variables_used.update(link.args)
 
         for v in variables_used:
-            if isinstance(v, Variable):
-                if v not in variables_created:
-                    pending.append((block, v))
+            if (isinstance(v, Variable) and v not in variables_created and
+                    v._name not in ('last_exception_', 'last_exc_value_')):
+                pending.append((block, v))
 
     while pending:
         block, v = pending.pop()
@@ -190,8 +194,7 @@
         for w in variables_created:
             w_rep = variable_families.find_rep(w)
             if v_rep is w_rep:
-                # 'w' is in the same family as 'v', so we can simply
-                # reuse its value for 'v'
+                # 'w' is in the same family as 'v', so we can reuse it
                 block.renamevariables({v: w})
                 break
         else:

From noreply at buildbot.pypy.org  Wed Nov 12 19:43:09 2014
From: noreply at buildbot.pypy.org (rlamy)
Date: Wed, 12 Nov 2014 19:43:09 +0100 (CET)
Subject: [pypy-commit] pypy ssa-flow: make SSA_to_SSI accept only graphs,
	fix tests
Message-ID: <20141112184309.055ED1C093D@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy 
Branch: ssa-flow
Changeset: r74487:77c7e2e136ee
Date: 2013-08-15 16:20 +0100
http://bitbucket.org/pypy/pypy/changeset/77c7e2e136ee/

Log:	make SSA_to_SSI accept only graphs, fix tests

diff --git a/rpython/translator/backendopt/ssa.py b/rpython/translator/backendopt/ssa.py
--- a/rpython/translator/backendopt/ssa.py
+++ b/rpython/translator/backendopt/ssa.py
@@ -16,7 +16,9 @@
         # [Block, blockvar, linkvar, linkvar, linkvar...]
         opportunities = []
         opportunities_with_const = []
-        for block, links in mkinsideentrymap(graph).items():
+        entrymap = mkentrymap(graph)
+        del entrymap[graph.startblock]
+        for block, links in entrymap.items():
             assert links
             for n, inputvar in enumerate(block.inputargs):
                 vars = [block, inputvar]
@@ -123,23 +125,6 @@
 
 # ____________________________________________________________
 
-def mkinsideentrymap(graph_or_blocks):
-    # graph_or_blocks can be a full FunctionGraph, or a mapping
-    # {block: reachable-from-outside-flag}.
-    if isinstance(graph_or_blocks, dict):
-        blocks = graph_or_blocks
-        entrymap = {}
-        for block in blocks:
-            for link in block.exits:
-                if link.target in blocks and not blocks[link.target]:
-                    entrymap.setdefault(link.target, []).append(link)
-        return entrymap
-    else:
-        graph = graph_or_blocks
-        entrymap = mkentrymap(graph)
-        del entrymap[graph.startblock]
-        return entrymap
-
 def variables_created_in(block):
     result = set(block.inputargs)
     for op in block.operations:
@@ -147,17 +132,14 @@
     return result
 
 
-def SSA_to_SSI(graph_or_blocks, annotator=None):
+def SSA_to_SSI(graph, annotator=None):
     """Turn a number of blocks belonging to a flow graph into valid (i.e. SSI)
     form, assuming that they are only in SSA form (i.e. they can use each
     other's variables directly, without having to pass and rename them along
     links).
-
-    'graph_or_blocks' can be a graph, or just a dict that lists some blocks
-    from a graph, as follows: {block: reachable-from-outside-flag}.
     """
     seen = set()
-    for link in graph_or_blocks.iterlinks():
+    for link in graph.iterlinks():
         mapping = {}
         seen.update(link.args)
         for arg in link.target.inputargs:
@@ -165,8 +147,9 @@
                 mapping[arg] = Variable(arg)
         link.target.renamevariables(mapping)
 
-    entrymap = mkinsideentrymap(graph_or_blocks)
-    builder = DataFlowFamilyBuilder(graph_or_blocks)
+    entrymap = mkentrymap(graph)
+    del entrymap[graph.startblock]
+    builder = DataFlowFamilyBuilder(graph)
     variable_families = builder.get_variable_families()
     del builder
 
diff --git a/rpython/translator/backendopt/test/test_ssa.py b/rpython/translator/backendopt/test/test_ssa.py
--- a/rpython/translator/backendopt/test/test_ssa.py
+++ b/rpython/translator/backendopt/test/test_ssa.py
@@ -1,7 +1,7 @@
 from rpython.translator.backendopt.ssa import *
 from rpython.translator.translator import TranslationContext
-from rpython.flowspace.model import Block, Link, Variable, Constant
-from rpython.flowspace.model import SpaceOperation
+from rpython.flowspace.model import (
+    Block, Link, Variable, Constant, SpaceOperation, FunctionGraph)
 
 
 def test_data_flow_families():
@@ -60,16 +60,14 @@
     b2 = Block([x])
     b3 = Block([])
 
+    graph = FunctionGraph('x', b1)
     b2.operations.append(SpaceOperation('add', [x, c], y))
     b2.exitswitch = y
 
     b1.closeblock(Link([Constant(0)], b2))
     b2.closeblock(Link([y], b2), Link([], b3))
-    b3.closeblock(Link([y, c], None))
-
-    SSA_to_SSI({b1: True,     # reachable from outside
-                b2: False,
-                b3: False})
+    b3.closeblock(Link([y, c], graph.exceptblock))
+    SSA_to_SSI(graph)
 
     assert len(b1.inputargs) == 1
     assert len(b2.inputargs) == 2
@@ -100,10 +98,8 @@
 
     b3.operations.append(SpaceOperation('hello', [y], z))
     b1.closeblock(Link([x], b2), Link([], b3))
-
-    SSA_to_SSI({b1: True,     # reachable from outside
-                b2: False,
-                b3: False})
+    graph = FunctionGraph('x', b1)
+    SSA_to_SSI(graph)
 
     assert b1.inputargs == [x]
     assert b2.inputargs == [y]

From noreply at buildbot.pypy.org  Wed Nov 12 19:43:10 2014
From: noreply at buildbot.pypy.org (rlamy)
Date: Wed, 12 Nov 2014 19:43:10 +0100 (CET)
Subject: [pypy-commit] pypy ssa-flow: Eliminate empty blocks before calling
	SSA_to_SSI()
Message-ID: <20141112184310.266231C093D@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy 
Branch: ssa-flow
Changeset: r74488:d6628fb03af8
Date: 2013-08-15 16:47 +0100
http://bitbucket.org/pypy/pypy/changeset/d6628fb03af8/

Log:	Eliminate empty blocks before calling SSA_to_SSI()

diff --git a/rpython/flowspace/bytecode.py b/rpython/flowspace/bytecode.py
--- a/rpython/flowspace/bytecode.py
+++ b/rpython/flowspace/bytecode.py
@@ -5,7 +5,6 @@
 from opcode import EXTENDED_ARG, HAVE_ARGUMENT
 import opcode
 from rpython.flowspace.argument import Signature
-from rpython.flowspace.flowcontext import BytecodeCorruption
 
 CO_GENERATOR = 0x0020
 CO_VARARGS = 0x0004
@@ -27,6 +26,11 @@
         kwargname = None
     return Signature(argnames, varargname, kwargname)
 
+
+class BytecodeCorruption(Exception):
+    pass
+
+
 class HostCode(object):
     """
     A wrapper around a native code object of the host interpreter
diff --git a/rpython/flowspace/flowcontext.py b/rpython/flowspace/flowcontext.py
--- a/rpython/flowspace/flowcontext.py
+++ b/rpython/flowspace/flowcontext.py
@@ -8,6 +8,7 @@
 
 from rpython.tool.error import source_lines
 from rpython.translator.backendopt.ssa import SSA_to_SSI
+from rpython.translator.simplify import eliminate_empty_blocks
 from rpython.rlib import rstackovf
 from rpython.flowspace.argument import CallSpec
 from rpython.flowspace.model import (Constant, Variable, Block, Link,
@@ -17,6 +18,7 @@
 from rpython.flowspace.specialcase import (rpython_print_item,
     rpython_print_newline)
 from rpython.flowspace.operation import op
+from rpython.flowspace.bytecode import BytecodeCorruption
 
 w_None = const(None)
 
@@ -31,11 +33,10 @@
         msg += source_lines(self.ctx.graph, None, offset=self.ctx.last_instr)
         return "\n".join(msg)
 
+
 class StopFlowing(Exception):
     pass
 
-class BytecodeCorruption(Exception):
-    pass
 
 class SpamBlock(Block):
     def __init__(self, framestate):
@@ -87,6 +88,7 @@
                 if isinstance(w_value, Variable):
                     w_value.rename(name)
             del block.framestate     # memory saver
+    eliminate_empty_blocks(graph)
     SSA_to_SSI(graph)
 
 # ____________________________________________________________

From noreply at buildbot.pypy.org  Wed Nov 12 19:43:11 2014
From: noreply at buildbot.pypy.org (rlamy)
Date: Wed, 12 Nov 2014 19:43:11 +0100 (CET)
Subject: [pypy-commit] pypy ssa-flow: give Variables the right name from the
	start
Message-ID: <20141112184311.427531C093D@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy 
Branch: ssa-flow
Changeset: r74489:390372be1886
Date: 2013-08-15 17:41 +0100
http://bitbucket.org/pypy/pypy/changeset/390372be1886/

Log:	give Variables the right name from the start

diff --git a/rpython/flowspace/flowcontext.py b/rpython/flowspace/flowcontext.py
--- a/rpython/flowspace/flowcontext.py
+++ b/rpython/flowspace/flowcontext.py
@@ -81,12 +81,8 @@
         self.last_exception = last_exception
 
 def fixeggblocks(graph):
-    varnames = graph.func.func_code.co_varnames
     for block in graph.iterblocks():
         if isinstance(block, SpamBlock):
-            for name, w_value in zip(varnames, block.framestate.mergeable):
-                if isinstance(w_value, Variable):
-                    w_value.rename(name)
             del block.framestate     # memory saver
     eliminate_empty_blocks(graph)
     SSA_to_SSI(graph)
@@ -494,6 +490,10 @@
             block = None
 
         newblock = SpamBlock(newstate)
+        varnames = self.pycode.co_varnames
+        for name, w_value in zip(varnames, newstate.mergeable):
+            if isinstance(w_value, Variable):
+                w_value.rename(name)
         # unconditionally link the current block to the newblock
         outputargs = currentstate.getoutputargs(newstate)
         link = Link(outputargs, newblock)
diff --git a/rpython/flowspace/pygraph.py b/rpython/flowspace/pygraph.py
--- a/rpython/flowspace/pygraph.py
+++ b/rpython/flowspace/pygraph.py
@@ -13,7 +13,7 @@
         from rpython.flowspace.flowcontext import SpamBlock
         data = [None] * code.co_nlocals
         for i in range(code.formalargcount):
-            data[i] = Variable()
+            data[i] = Variable(code.co_varnames[i])
         state = FrameState(data + [Constant(None), Constant(None)], [], 0)
         initialblock = SpamBlock(state)
         super(PyGraph, self).__init__(self._sanitize_funcname(func), initialblock)

From noreply at buildbot.pypy.org  Wed Nov 12 19:43:12 2014
From: noreply at buildbot.pypy.org (rlamy)
Date: Wed, 12 Nov 2014 19:43:12 +0100 (CET)
Subject: [pypy-commit] pypy ssa-flow: Don't pass unchanged Variable objects
	through links
Message-ID: <20141112184312.641B91C093D@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy 
Branch: ssa-flow
Changeset: r74490:43ff4ed9e9ec
Date: 2013-08-15 20:19 +0100
http://bitbucket.org/pypy/pypy/changeset/43ff4ed9e9ec/

Log:	Don't pass unchanged Variable objects through links

	Remove incorrect hack in SSA_to_SSI

diff --git a/rpython/flowspace/flowcontext.py b/rpython/flowspace/flowcontext.py
--- a/rpython/flowspace/flowcontext.py
+++ b/rpython/flowspace/flowcontext.py
@@ -111,12 +111,11 @@
 
     def guessbool(self, ctx, w_condition):
         block = self.crnt_block
-        vars = block.getvariables()
         links = []
         for case in [False, True]:
-            egg = EggBlock(vars, block, case)
+            egg = EggBlock([], block, case)
             ctx.pendingblocks.append(egg)
-            link = Link(vars, egg, case)
+            link = Link([], egg, case)
             links.append(link)
 
         block.exitswitch = w_condition
@@ -133,16 +132,16 @@
         links = []
         for case in [None] + list(cases):
             if case is not None:
-                assert block.operations[-1].result is bvars[-1]
-                vars = bvars[:-1]
-                vars2 = bvars[:-1]
                 if case is Exception:
                     last_exc = Variable('last_exception')
                 else:
                     last_exc = Constant(case)
                 last_exc_value = Variable('last_exc_value')
-                vars.extend([last_exc, last_exc_value])
-                vars2.extend([Variable(), Variable()])
+                vars = [last_exc, last_exc_value]
+                vars2 = [Variable(), Variable()]
+            else:
+                vars = []
+                vars2 = []
             egg = EggBlock(vars2, block, case)
             ctx.pendingblocks.append(egg)
             link = Link(vars, egg, case)
diff --git a/rpython/translator/backendopt/ssa.py b/rpython/translator/backendopt/ssa.py
--- a/rpython/translator/backendopt/ssa.py
+++ b/rpython/translator/backendopt/ssa.py
@@ -138,15 +138,6 @@
     other's variables directly, without having to pass and rename them along
     links).
     """
-    seen = set()
-    for link in graph.iterlinks():
-        mapping = {}
-        seen.update(link.args)
-        for arg in link.target.inputargs:
-            if arg in seen and isinstance(arg, Variable):
-                mapping[arg] = Variable(arg)
-        link.target.renamevariables(mapping)
-
     entrymap = mkentrymap(graph)
     del entrymap[graph.startblock]
     builder = DataFlowFamilyBuilder(graph)
diff --git a/rpython/translator/simplify.py b/rpython/translator/simplify.py
--- a/rpython/translator/simplify.py
+++ b/rpython/translator/simplify.py
@@ -75,10 +75,12 @@
                 outputargs = []
                 for v in exit.args:
                     if isinstance(v, Variable):
-                        # this variable is valid in the context of block1
-                        # but it must come from 'link'
-                        i = block1.inputargs.index(v)
-                        v = link.args[i]
+                        try:
+                            i = block1.inputargs.index(v)
+                            v = link.args[i]
+                        except ValueError:
+                            # the variable was passed implicitly to block1
+                            pass
                     outputargs.append(v)
                 link.args = outputargs
                 link.target = exit.target

From noreply at buildbot.pypy.org  Wed Nov 12 19:43:13 2014
From: noreply at buildbot.pypy.org (rlamy)
Date: Wed, 12 Nov 2014 19:43:13 +0100 (CET)
Subject: [pypy-commit] pypy ssa-flow: split two rather unrelated code paths
Message-ID: <20141112184313.8913E1C093D@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy 
Branch: ssa-flow
Changeset: r74491:c98fcc6242b4
Date: 2013-08-13 15:55 +0100
http://bitbucket.org/pypy/pypy/changeset/c98fcc6242b4/

Log:	split two rather unrelated code paths

diff --git a/rpython/flowspace/flowcontext.py b/rpython/flowspace/flowcontext.py
--- a/rpython/flowspace/flowcontext.py
+++ b/rpython/flowspace/flowcontext.py
@@ -486,7 +486,18 @@
                 break
         else:
             newstate = currentstate.copy()
-            block = None
+            newblock = SpamBlock(newstate)
+            varnames = self.pycode.co_varnames
+            for name, w_value in zip(varnames, newstate.mergeable):
+                if isinstance(w_value, Variable):
+                    w_value.rename(name)
+            # unconditionally link the current block to the newblock
+            outputargs = currentstate.getoutputargs(newstate)
+            link = Link(outputargs, newblock)
+            currentblock.closeblock(link)
+            candidates.insert(0, newblock)
+            self.pendingblocks.append(newblock)
+            return
 
         newblock = SpamBlock(newstate)
         varnames = self.pycode.co_varnames
@@ -498,15 +509,15 @@
         link = Link(outputargs, newblock)
         currentblock.closeblock(link)
 
-        if block is not None:
-            # to simplify the graph, we patch the old block to point
-            # directly at the new block which is its generalization
-            block.dead = True
-            block.operations = ()
-            block.exitswitch = None
-            outputargs = block.framestate.getoutputargs(newstate)
-            block.recloseblock(Link(outputargs, newblock))
-            candidates.remove(block)
+        # to simplify the graph, we patch the old block to point
+        # directly at the new block which is its generalization
+        block.dead = True
+        block.operations = ()
+        block.exitswitch = None
+        outputargs = block.framestate.getoutputargs(newstate)
+        block.recloseblock(Link(outputargs, newblock))
+        candidates.remove(block)
+
         candidates.insert(0, newblock)
         self.pendingblocks.append(newblock)
 

From noreply at buildbot.pypy.org  Wed Nov 12 19:43:14 2014
From: noreply at buildbot.pypy.org (rlamy)
Date: Wed, 12 Nov 2014 19:43:14 +0100 (CET)
Subject: [pypy-commit] pypy ssa-flow: Assign variable names more locally
Message-ID: <20141112184314.A3F4D1C093D@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy 
Branch: ssa-flow
Changeset: r74492:fa7224bd0d4e
Date: 2013-08-16 01:53 +0100
http://bitbucket.org/pypy/pypy/changeset/fa7224bd0d4e/

Log:	Assign variable names more locally

diff --git a/rpython/flowspace/flowcontext.py b/rpython/flowspace/flowcontext.py
--- a/rpython/flowspace/flowcontext.py
+++ b/rpython/flowspace/flowcontext.py
@@ -487,10 +487,6 @@
         else:
             newstate = currentstate.copy()
             newblock = SpamBlock(newstate)
-            varnames = self.pycode.co_varnames
-            for name, w_value in zip(varnames, newstate.mergeable):
-                if isinstance(w_value, Variable):
-                    w_value.rename(name)
             # unconditionally link the current block to the newblock
             outputargs = currentstate.getoutputargs(newstate)
             link = Link(outputargs, newblock)
@@ -929,6 +925,8 @@
         w_newvalue = self.popvalue()
         assert w_newvalue is not None
         self.locals_stack_w[varindex] = w_newvalue
+        if isinstance(w_newvalue, Variable):
+            w_newvalue.rename(self.getlocalvarname(varindex))
 
     def STORE_GLOBAL(self, nameindex):
         varname = self.getname_u(nameindex)
diff --git a/rpython/flowspace/framestate.py b/rpython/flowspace/framestate.py
--- a/rpython/flowspace/framestate.py
+++ b/rpython/flowspace/framestate.py
@@ -13,7 +13,7 @@
         newstate = []
         for w in self.mergeable:
             if isinstance(w, Variable):
-                w = Variable()
+                w = Variable(w)
             newstate.append(w)
         return FrameState(newstate, self.blocklist, self.next_instr)
 
@@ -66,14 +66,14 @@
 
 def union(w1, w2):
     "Union of two variables or constants."
+    if w1 == w2:
+        return w1
     if w1 is None or w2 is None:
         return None  # if w1 or w2 is an undefined local, we "kill" the value
                      # coming from the other path and return an undefined local
     if isinstance(w1, Variable) or isinstance(w2, Variable):
         return Variable()  # new fresh Variable
     if isinstance(w1, Constant) and isinstance(w2, Constant):
-        if w1 == w2:
-            return w1
         # FlowSignal represent stack unrollers in the stack.
         # They should not be merged because they will be unwrapped.
         # This is needed for try:except: and try:finally:, though

From noreply at buildbot.pypy.org  Wed Nov 12 19:43:15 2014
From: noreply at buildbot.pypy.org (rlamy)
Date: Wed, 12 Nov 2014 19:43:15 +0100 (CET)
Subject: [pypy-commit] pypy ssa-flow: Rename FrameState.__eq__ to .matches
Message-ID: <20141112184315.CAADA1C093D@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy 
Branch: ssa-flow
Changeset: r74493:5206983b39c8
Date: 2013-08-16 03:08 +0100
http://bitbucket.org/pypy/pypy/changeset/5206983b39c8/

Log:	Rename FrameState.__eq__ to .matches

diff --git a/rpython/flowspace/flowcontext.py b/rpython/flowspace/flowcontext.py
--- a/rpython/flowspace/flowcontext.py
+++ b/rpython/flowspace/flowcontext.py
@@ -476,13 +476,7 @@
         candidates = self.joinpoints.setdefault(next_instr, [])
         for block in candidates:
             newstate = block.framestate.union(currentstate)
-            if newstate is None:
-                continue
-            elif newstate == block.framestate:
-                outputargs = currentstate.getoutputargs(newstate)
-                currentblock.closeblock(Link(outputargs, block))
-                return
-            else:
+            if newstate is not None:
                 break
         else:
             newstate = currentstate.copy()
@@ -495,6 +489,11 @@
             self.pendingblocks.append(newblock)
             return
 
+        if newstate.matches(block.framestate):
+            outputargs = currentstate.getoutputargs(newstate)
+            currentblock.closeblock(Link(outputargs, block))
+            return
+
         newblock = SpamBlock(newstate)
         varnames = self.pycode.co_varnames
         for name, w_value in zip(varnames, newstate.mergeable):
diff --git a/rpython/flowspace/framestate.py b/rpython/flowspace/framestate.py
--- a/rpython/flowspace/framestate.py
+++ b/rpython/flowspace/framestate.py
@@ -20,13 +20,11 @@
     def getvariables(self):
         return [w for w in self.mergeable if isinstance(w, Variable)]
 
-    def __eq__(self, other):
-        """Two states are equal
-        if they only use different Variables at the same place"""
+    def matches(self, other):
+        """Two states match if they only differ by using different Variables
+        at the same place"""
         # safety check, don't try to compare states with different
         # nonmergeable states
-        assert isinstance(other, FrameState)
-        assert len(self.mergeable) == len(other.mergeable)
         assert self.blocklist == other.blocklist
         assert self.next_instr == other.next_instr
         for w1, w2 in zip(self.mergeable, other.mergeable):
@@ -35,9 +33,6 @@
                 return False
         return True
 
-    def __ne__(self, other):
-        return not (self == other)
-
     def union(self, other):
         """Compute a state that is at least as general as both self and other.
            A state 'a' is more general than a state 'b' if all Variables in 'b'
diff --git a/rpython/flowspace/test/test_framestate.py b/rpython/flowspace/test/test_framestate.py
--- a/rpython/flowspace/test/test_framestate.py
+++ b/rpython/flowspace/test/test_framestate.py
@@ -26,41 +26,41 @@
         ctx = self.get_context(self.func_simple)
         fs1 = ctx.getstate(0)
         fs2 = ctx.getstate(0)
-        assert fs1 == fs2
+        assert fs1.matches(fs2)
 
     def test_neq_hacked_framestate(self):
         ctx = self.get_context(self.func_simple)
         fs1 = ctx.getstate(0)
         ctx.locals_stack_w[ctx.pycode.co_nlocals-1] = Variable()
         fs2 = ctx.getstate(0)
-        assert fs1 != fs2
+        assert not fs1.matches(fs2)
 
     def test_union_on_equal_framestates(self):
         ctx = self.get_context(self.func_simple)
         fs1 = ctx.getstate(0)
         fs2 = ctx.getstate(0)
-        assert fs1.union(fs2) == fs1
+        assert fs1.union(fs2).matches(fs1)
 
     def test_union_on_hacked_framestates(self):
         ctx = self.get_context(self.func_simple)
         fs1 = ctx.getstate(0)
         ctx.locals_stack_w[ctx.pycode.co_nlocals-1] = Variable()
         fs2 = ctx.getstate(0)
-        assert fs1.union(fs2) == fs2  # fs2 is more general
-        assert fs2.union(fs1) == fs2  # fs2 is more general
+        assert fs1.union(fs2).matches(fs2)  # fs2 is more general
+        assert fs2.union(fs1).matches(fs2)  # fs2 is more general
 
     def test_restore_frame(self):
         ctx = self.get_context(self.func_simple)
         fs1 = ctx.getstate(0)
         ctx.locals_stack_w[ctx.pycode.co_nlocals-1] = Variable()
         ctx.setstate(fs1)
-        assert fs1 == ctx.getstate(0)
+        assert fs1.matches(ctx.getstate(0))
 
     def test_copy(self):
         ctx = self.get_context(self.func_simple)
         fs1 = ctx.getstate(0)
         fs2 = fs1.copy()
-        assert fs1 == fs2
+        assert fs1.matches(fs2)
 
     def test_getvariables(self):
         ctx = self.get_context(self.func_simple)

From noreply at buildbot.pypy.org  Thu Nov 13 01:31:40 2014
From: noreply at buildbot.pypy.org (rlamy)
Date: Thu, 13 Nov 2014 01:31:40 +0100 (CET)
Subject: [pypy-commit] pypy ssa-flow: clean up and kill dead code
Message-ID: <20141113003140.805AC1C3641@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy 
Branch: ssa-flow
Changeset: r74494:8da7301b25d7
Date: 2014-11-12 23:49 +0000
http://bitbucket.org/pypy/pypy/changeset/8da7301b25d7/

Log:	clean up and kill dead code

diff --git a/rpython/annotator/test/test_annrpython.py b/rpython/annotator/test/test_annrpython.py
--- a/rpython/annotator/test/test_annrpython.py
+++ b/rpython/annotator/test/test_annrpython.py
@@ -14,7 +14,6 @@
 from rpython.rlib.rarithmetic import r_uint, base_int, r_longlong, r_ulonglong
 from rpython.rlib.rarithmetic import r_singlefloat
 from rpython.rlib import objectmodel
-from rpython.flowspace.objspace import build_flow
 from rpython.flowspace.flowcontext import FlowingError
 from rpython.flowspace.operation import op
 
@@ -53,17 +52,6 @@
                 self.translator.view()
             return s
 
-    def make_fun(self, func):
-        import inspect
-        try:
-            func = func.im_func
-        except AttributeError:
-            pass
-        name = func.func_name
-        funcgraph = build_flow(func)
-        funcgraph.source = inspect.getsource(func)
-        return funcgraph
-
     def test_simple_func(self):
         """
         one test source:
diff --git a/rpython/flowspace/test/test_generator.py b/rpython/flowspace/test/test_generator.py
--- a/rpython/flowspace/test/test_generator.py
+++ b/rpython/flowspace/test/test_generator.py
@@ -67,7 +67,7 @@
             yield n
             yield n
         #
-        graph = build_flow(func)
+        graph = make_generator_entry_graph(func)
         if option.view:
             graph.show()
         block = graph.startblock
diff --git a/rpython/translator/tool/make_dot.py b/rpython/translator/tool/make_dot.py
--- a/rpython/translator/tool/make_dot.py
+++ b/rpython/translator/tool/make_dot.py
@@ -1,7 +1,6 @@
 import os
 import inspect, linecache
 from rpython.flowspace.model import *
-from rpython.flowspace.objspace import build_flow
 from rpython.tool.udir import udir
 from py.process import cmdexec
 from rpython.tool.error import offset2lineno
@@ -238,14 +237,3 @@
     # not a keyword
     name = ''.join([CHAR_MAP[c] for c in name])
     return '_' + name
-
-
-if __name__ == '__main__':
-    def f(x):
-        i = 0
-        while i < x:
-            i += 1
-        return i
-
-    graph = build_flow(f)
-    make_dot('f', graph)

From noreply at buildbot.pypy.org  Thu Nov 13 01:31:41 2014
From: noreply at buildbot.pypy.org (rlamy)
Date: Thu, 13 Nov 2014 01:31:41 +0100 (CET)
Subject: [pypy-commit] pypy ssa-flow: move SSA_to_SSI() call from
 build_flow() to simplify_graph()
Message-ID: <20141113003141.BD9341C3641@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy 
Branch: ssa-flow
Changeset: r74495:08a17311eb3a
Date: 2014-11-13 00:30 +0000
http://bitbucket.org/pypy/pypy/changeset/08a17311eb3a/

Log:	move SSA_to_SSI() call from build_flow() to simplify_graph()

diff --git a/rpython/flowspace/flowcontext.py b/rpython/flowspace/flowcontext.py
--- a/rpython/flowspace/flowcontext.py
+++ b/rpython/flowspace/flowcontext.py
@@ -7,7 +7,6 @@
 import __builtin__
 
 from rpython.tool.error import source_lines
-from rpython.translator.backendopt.ssa import SSA_to_SSI
 from rpython.translator.simplify import eliminate_empty_blocks
 from rpython.rlib import rstackovf
 from rpython.flowspace.argument import CallSpec
@@ -84,8 +83,6 @@
     for block in graph.iterblocks():
         if isinstance(block, SpamBlock):
             del block.framestate     # memory saver
-    eliminate_empty_blocks(graph)
-    SSA_to_SSI(graph)
 
 # ____________________________________________________________
 
diff --git a/rpython/flowspace/objspace.py b/rpython/flowspace/objspace.py
--- a/rpython/flowspace/objspace.py
+++ b/rpython/flowspace/objspace.py
@@ -30,7 +30,7 @@
 
 def build_flow(func):
     """
-    Create the flow graph for the function.
+    Create the flow graph (in SSA form) for the function.
     """
     _assert_rpythonic(func)
     if (isgeneratorfunction(func) and
@@ -41,7 +41,6 @@
     ctx = FlowContext(graph, code)
     ctx.build_flow()
     fixeggblocks(graph)
-    checkgraph(graph)
     if code.is_generator:
         tweak_generator_graph(graph)
     return graph
diff --git a/rpython/flowspace/test/test_objspace.py b/rpython/flowspace/test/test_objspace.py
--- a/rpython/flowspace/test/test_objspace.py
+++ b/rpython/flowspace/test/test_objspace.py
@@ -635,6 +635,7 @@
 
     def test_highly_branching_example(self):
         x = self.codetest(self.highly_branching_example)
+        simplify_graph(x)
         # roughly 20 blocks + 30 links
         assert len(list(x.iterblocks())) + len(list(x.iterlinks())) < 60
 
diff --git a/rpython/translator/simplify.py b/rpython/translator/simplify.py
--- a/rpython/translator/simplify.py
+++ b/rpython/translator/simplify.py
@@ -14,6 +14,7 @@
 from rpython.translator import unsimplify
 from rpython.translator.backendopt import ssa
 from rpython.rtyper.lltypesystem import lloperation, lltype
+from rpython.translator.backendopt.ssa import SSA_to_SSI
 
 def get_graph(arg, translator):
     if isinstance(arg, Variable):
@@ -961,6 +962,7 @@
 
 all_passes = [
     eliminate_empty_blocks,
+    SSA_to_SSI,
     remove_assertion_errors,
     join_blocks,
     coalesce_bool,
@@ -976,7 +978,6 @@
     """inplace-apply all the existing optimisations to the graph."""
     if passes is True:
         passes = all_passes
-    checkgraph(graph)
     for pass_ in passes:
         pass_(graph)
     checkgraph(graph)
@@ -984,6 +985,7 @@
 def cleanup_graph(graph):
     checkgraph(graph)
     eliminate_empty_blocks(graph)
+    SSA_to_SSI(graph)
     join_blocks(graph)
     remove_identical_vars(graph)
     checkgraph(graph)

From noreply at buildbot.pypy.org  Thu Nov 13 05:15:29 2014
From: noreply at buildbot.pypy.org (mattip)
Date: Thu, 13 Nov 2014 05:15:29 +0100 (CET)
Subject: [pypy-commit] pypy default: backout 3532a779652c and define as well
 as declare for MSVC, unbreaks -A tests
Message-ID: <20141113041529.B61DA1C0548@cobra.cs.uni-duesseldorf.de>

Author: mattip 
Branch: 
Changeset: r74496:6db23cc8fa0d
Date: 2014-11-13 05:11 +0200
http://bitbucket.org/pypy/pypy/changeset/6db23cc8fa0d/

Log:	backout 3532a779652c and define as well as declare for MSVC,
	unbreaks -A tests

diff --git a/pypy/module/test_lib_pypy/ctypes_tests/_ctypes_test.c b/pypy/module/test_lib_pypy/ctypes_tests/_ctypes_test.c
--- a/pypy/module/test_lib_pypy/ctypes_tests/_ctypes_test.c
+++ b/pypy/module/test_lib_pypy/ctypes_tests/_ctypes_test.c
@@ -316,8 +316,8 @@
 {
 }
 
-static LONG_LONG last_tf_arg_s;
-static unsigned LONG_LONG last_tf_arg_u;
+EXPORT(LONG_LONG) last_tf_arg_s = 0;
+EXPORT(unsigned LONG_LONG) last_tf_arg_u = 0;
 
 struct BITS {
 	int A: 1, B:2, C:3, D:4, E: 5, F: 6, G: 7, H: 8, I: 9;

From noreply at buildbot.pypy.org  Thu Nov 13 10:08:02 2014
From: noreply at buildbot.pypy.org (Raemi)
Date: Thu, 13 Nov 2014 10:08:02 +0100 (CET)
Subject: [pypy-commit] stmgc c8-small-uniform: more debugging
Message-ID: <20141113090802.684941C3140@cobra.cs.uni-duesseldorf.de>

Author: Remi Meier 
Branch: c8-small-uniform
Changeset: r1509:380ee5886e6b
Date: 2014-11-13 10:08 +0100
http://bitbucket.org/pypy/stmgc/changeset/380ee5886e6b/

Log:	more debugging

diff --git a/c8/stm/core.c b/c8/stm/core.c
--- a/c8/stm/core.c
+++ b/c8/stm/core.c
@@ -35,6 +35,7 @@
     assert(IMPLY(from_segnum >= 0, get_priv_segment(from_segnum)->modification_lock));
     assert(STM_PSEGMENT->modification_lock);
 
+    DEBUG_EXPECT_SEGFAULT(false);
     for (; undo < end; undo++) {
         object_t *obj = undo->object;
         stm_char *oslice = ((stm_char *)obj) + SLICE_OFFSET(undo->slice);
@@ -61,6 +62,11 @@
             continue;           /* only copy unmodified */
         }
 
+        /* XXX: if the next assert is always true, we should never get a segfault
+           in this function at all. So the DEBUG_EXPECT_SEGFAULT is correct. */
+        assert((get_page_status_in(STM_SEGMENT->segment_num,
+                                   current_page_num) != PAGE_NO_ACCESS));
+
         dprintf(("import slice seg=%d obj=%p off=%lu sz=%d pg=%lu\n",
                  from_segnum, obj, SLICE_OFFSET(undo->slice),
                  SLICE_SIZE(undo->slice), current_page_num));
@@ -77,6 +83,7 @@
             assert(!(obj->stm_flags & GCFLAG_WB_EXECUTED));
         }
     }
+    DEBUG_EXPECT_SEGFAULT(true);
 }
 
 
@@ -203,6 +210,8 @@
 
 static void _signal_handler(int sig, siginfo_t *siginfo, void *context)
 {
+    assert(_stm_segfault_expected);
+
     int saved_errno = errno;
     char *addr = siginfo->si_addr;
     dprintf(("si_addr: %p\n", addr));
@@ -229,7 +238,9 @@
         abort();
     }
 
+    DEBUG_EXPECT_SEGFAULT(false);
     handle_segfault_in_page(pagenum);
+    DEBUG_EXPECT_SEGFAULT(true);
 
     errno = saved_errno;
     /* now return and retry */
@@ -548,6 +559,8 @@
        if 'obj' is merely an overflow object.  FIX ME, likely by copying
        the overflow number logic from c7. */
 
+    DEBUG_EXPECT_SEGFAULT(false);
+
     acquire_modification_lock(STM_SEGMENT->segment_num);
     uintptr_t slice_sz;
     uintptr_t in_page_offset = (uintptr_t)obj % 4096UL;
@@ -575,14 +588,16 @@
     }
     OPT_ASSERT(remaining_obj_sz == 0);
 
+    /* remove the WRITE_BARRIER flag and add WB_EXECUTED */
+    obj->stm_flags &= ~GCFLAG_WRITE_BARRIER;
+    obj->stm_flags |= GCFLAG_WB_EXECUTED;
+
+    DEBUG_EXPECT_SEGFAULT(true);
+
     release_modification_lock(STM_SEGMENT->segment_num);
     /* done fiddling with protection and privatization */
     release_all_privatization_locks();
 
-    /* remove the WRITE_BARRIER flag and add WB_EXECUTED */
-    obj->stm_flags &= ~GCFLAG_WRITE_BARRIER;
-    obj->stm_flags |= GCFLAG_WB_EXECUTED;
-
     /* also add it to the GC list for minor collections */
     LIST_APPEND(STM_PSEGMENT->objects_pointing_to_nursery, obj);
 }
@@ -939,30 +954,32 @@
     /* if the page of the fragment is fully shared, nothing to do:
        |S|N|N|N| */
 
-    /* nobody must change the page mapping until we flush */
-    assert(STM_PSEGMENT->privatization_lock);
+    /* XXXXX: re-enable the following if completely sure that we always
+       copy the shared page when we privatize correctly. */
+    /* /\* nobody must change the page mapping until we flush *\/ */
+    /* assert(STM_PSEGMENT->privatization_lock); */
 
-    int my_segnum = STM_SEGMENT->segment_num;
-    uintptr_t pagenum = (uintptr_t)frag / 4096;
-    bool fully_shared = false;
+    /* int my_segnum = STM_SEGMENT->segment_num; */
+    /* uintptr_t pagenum = (uintptr_t)frag / 4096; */
+    /* bool fully_shared = false; */
 
-    if (get_page_status_in(my_segnum, pagenum) == PAGE_SHARED) {
-        fully_shared = true;
-        int i;
-        for (i = 0; fully_shared && i < NB_SEGMENTS; i++) {
-            if (i == my_segnum)
-                continue;
+    /* if (get_page_status_in(my_segnum, pagenum) == PAGE_SHARED) { */
+    /*     fully_shared = true; */
+    /*     int i; */
+    /*     for (i = 0; fully_shared && i < NB_SEGMENTS; i++) { */
+    /*         if (i == my_segnum) */
+    /*             continue; */
 
-            /* XXX: works if never all pages use SHARED page */
-            if (get_page_status_in(i, pagenum) != PAGE_NO_ACCESS) {
-                fully_shared = false;
-                break;
-            }
-        }
-    }
+    /*         /\* XXX: works if never all pages use SHARED page *\/ */
+    /*         if (get_page_status_in(i, pagenum) != PAGE_NO_ACCESS) { */
+    /*             fully_shared = false; */
+    /*             break; */
+    /*         } */
+    /*     } */
+    /* } */
 
-    if (fully_shared)
-        return;                 /* nothing to do */
+    /* if (fully_shared) */
+    /*     return;                 /\* nothing to do *\/ */
 
     /* e.g. |P|S|N|P| */
 
@@ -1030,6 +1047,7 @@
 
     __sync_synchronize();
     assert(STM_PSEGMENT->privatization_lock);
+    DEBUG_EXPECT_SEGFAULT(false);
 
     long i, myself = STM_SEGMENT->segment_num;
     do {
@@ -1054,4 +1072,6 @@
             }
         }
     } while (j > 0);
+
+    DEBUG_EXPECT_SEGFAULT(true);
 }
diff --git a/c8/stm/core.h b/c8/stm/core.h
--- a/c8/stm/core.h
+++ b/c8/stm/core.h
@@ -147,13 +147,12 @@
 
 
 #ifndef STM_TESTS
-static
+static char *stm_object_pages;
+static char *stm_file_pages;
+#else
+char *stm_object_pages;
+char *stm_file_pages;
 #endif
-       char *stm_object_pages;
-#ifndef STM_TESTS
-static
-#endif
-       char *stm_file_pages;
 static int stm_object_pages_fd;
 static stm_thread_local_t *stm_all_thread_locals = NULL;
 
diff --git a/c8/stm/nursery.c b/c8/stm/nursery.c
--- a/c8/stm/nursery.c
+++ b/c8/stm/nursery.c
@@ -385,7 +385,7 @@
     char *realobj = REAL_ADDRESS(STM_SEGMENT->segment_base, obj);
     size_t size = stmcb_size_rounded_up((struct object_s *)realobj);
 
-    /* always gets outside as a large object for now */
+    /* always gets outside as a large object for now (XXX?) */
     object_t *nobj = (object_t *)allocate_outside_nursery_large(size);
 
     /* Initialize the shadow enough to be considered a valid gc object.
diff --git a/c8/stm/setup.c b/c8/stm/setup.c
--- a/c8/stm/setup.c
+++ b/c8/stm/setup.c
@@ -262,6 +262,8 @@
     set_gs_register(get_segment_base(num));
     s_mutex_unlock();
 
+    DEBUG_EXPECT_SEGFAULT(true);
+
     if (num == 0) {
         dprintf(("STM_GC_NURSERY: %d\n", STM_GC_NURSERY));
         dprintf(("NB_PAGES: %d\n", NB_PAGES));
diff --git a/c8/stm/setup.h b/c8/stm/setup.h
--- a/c8/stm/setup.h
+++ b/c8/stm/setup.h
@@ -2,3 +2,10 @@
 static void close_fd_mmap(int map_fd);
 static void setup_protection_settings(void);
 static pthread_t *_get_cpth(stm_thread_local_t *);
+
+#ifndef NDEBUG
+static __thread long _stm_segfault_expected = false;
+#define DEBUG_EXPECT_SEGFAULT(v) do {_stm_segfault_expected = (v);} while (0)
+#else
+#define DEBUG_EXPECT_SEGFAULT(v) {}
+#endif
diff --git a/c8/test/test_hash_id.py b/c8/test/test_hash_id.py
--- a/c8/test/test_hash_id.py
+++ b/c8/test/test_hash_id.py
@@ -51,6 +51,7 @@
         assert len(set([h1, h2, h3, h4])) == 4     # guaranteed by the algo
 
     def test_hash_lower_bits(self):
+        py.test.skip("fails without a real large-malloc implementation")
         self.start_transaction()
         lp1 = stm_allocate(32)
         lp2 = stm_allocate(32)

From noreply at buildbot.pypy.org  Thu Nov 13 10:26:53 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 13 Nov 2014 10:26:53 +0100 (CET)
Subject: [pypy-commit] pypy default: Test and fix
Message-ID: <20141113092653.9BC3A1C0548@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r74497:f175883d10c1
Date: 2014-11-13 10:26 +0100
http://bitbucket.org/pypy/pypy/changeset/f175883d10c1/

Log:	Test and fix

diff --git a/pypy/objspace/std/strbufobject.py b/pypy/objspace/std/strbufobject.py
--- a/pypy/objspace/std/strbufobject.py
+++ b/pypy/objspace/std/strbufobject.py
@@ -5,6 +5,7 @@
 from pypy.objspace.std.bytesobject import (W_AbstractBytesObject,
     W_BytesObject, StringBuffer)
 from pypy.interpreter.gateway import interp2app, unwrap_spec
+from pypy.interpreter.error import OperationError
 from rpython.rlib.rstring import StringBuilder
 
 
@@ -46,15 +47,18 @@
         return space.wrap(self.length)
 
     def descr_add(self, space, w_other):
+        try:
+            other = W_BytesObject._op_val(space, w_other)
+        except OperationError as e:
+            if e.match(space, space.w_TypeError):
+                return space.w_NotImplemented
+            raise
         if self.builder.getlength() != self.length:
             builder = StringBuilder()
             builder.append(self.force())
         else:
             builder = self.builder
-        if isinstance(w_other, W_StringBufferObject):
-            builder.append(w_other.force())
-        else:
-            builder.append(w_other._value)
+        builder.append(other)
         return W_StringBufferObject(builder)
 
     def descr_str(self, space):
diff --git a/pypy/objspace/std/test/test_strbufobject.py b/pypy/objspace/std/test/test_strbufobject.py
--- a/pypy/objspace/std/test/test_strbufobject.py
+++ b/pypy/objspace/std/test/test_strbufobject.py
@@ -78,3 +78,8 @@
         c = '0'.__add__('1')
         x = c + a
         assert x == '01ab'
+
+    def test_add_non_string(self):
+        a = 'a'
+        a += 'b'
+        raises(TypeError, "a += 5")

From noreply at buildbot.pypy.org  Thu Nov 13 11:59:11 2014
From: noreply at buildbot.pypy.org (ltratt)
Date: Thu, 13 Nov 2014 11:59:11 +0100 (CET)
Subject: [pypy-commit] pypy recursion_and_inlining: Unroll a (customisable)
 fixed number of iterations of recursive functions.
Message-ID: <20141113105911.4A7791D35EB@cobra.cs.uni-duesseldorf.de>

Author: Laurence Tratt 
Branch: recursion_and_inlining
Changeset: r74498:2fa67aa20eea
Date: 2014-11-13 10:53 +0000
http://bitbucket.org/pypy/pypy/changeset/2fa67aa20eea/

Log:	Unroll a (customisable) fixed number of iterations of recursive
	functions.

	In essence, we count how instances of the function we're about to
	call are already on the meta-interpreter stack and only stop tracing
	when that count has been exceeded. Initial experiments suggest that
	7 is a reasonable number, though this shouldn't be considered fixed
	in stone, as it's heavily dependent on what benchmarks one uses.

diff --git a/rpython/jit/metainterp/pyjitpl.py b/rpython/jit/metainterp/pyjitpl.py
--- a/rpython/jit/metainterp/pyjitpl.py
+++ b/rpython/jit/metainterp/pyjitpl.py
@@ -959,6 +959,7 @@
                 portal_code = targetjitdriver_sd.mainjitcode
                 inline = True
                 if self.metainterp.is_main_jitcode(portal_code):
+                    count = 0
                     for gk, _ in self.metainterp.portal_trace_positions:
                         if gk is None:
                             continue
@@ -968,12 +969,16 @@
                             if not gk[i].same_constant(greenboxes[i]):
                                 break
                         else:
-                            # The greenkey of a trace position on the stack
-                            # matches what we have, which means we're definitely
-                            # about to recurse.
-                            warmrunnerstate.dont_trace_here(greenboxes)
-                            inline = False
-                            break
+                            count += 1
+                    memmgr = self.metainterp.staticdata.warmrunnerdesc.memory_manager
+                    if count >= memmgr.max_unroll_recursion:
+                        # This function is recursive and has exceeded the
+                        # maximum number of unrollings we allow. We want to stop
+                        # inlining it further and to make sure that, if it
+                        # hasn't happened already, the function is traced
+                        # separately as soon as possible.
+                        warmrunnerstate.dont_trace_here(greenboxes)
+                        inline = False
                 if inline:
                     return self.metainterp.perform_call(portal_code, allboxes,
                                 greenkey=greenboxes)
diff --git a/rpython/jit/metainterp/warmspot.py b/rpython/jit/metainterp/warmspot.py
--- a/rpython/jit/metainterp/warmspot.py
+++ b/rpython/jit/metainterp/warmspot.py
@@ -69,7 +69,8 @@
                     backendopt=False, trace_limit=sys.maxint,
                     inline=False, loop_longevity=0, retrace_limit=5,
                     function_threshold=4,
-                    enable_opts=ALL_OPTS_NAMES, max_retrace_guards=15, **kwds):
+                    enable_opts=ALL_OPTS_NAMES, max_retrace_guards=15, 
+                    max_unroll_recursion=7, **kwds):
     from rpython.config.config import ConfigError
     translator = interp.typer.annotator.translator
     try:
@@ -91,6 +92,7 @@
         jd.warmstate.set_param_retrace_limit(retrace_limit)
         jd.warmstate.set_param_max_retrace_guards(max_retrace_guards)
         jd.warmstate.set_param_enable_opts(enable_opts)
+        jd.warmstate.set_param_max_unroll_recursion(max_unroll_recursion)
     warmrunnerdesc.finish()
     if graph_and_interp_only:
         return interp, graph
diff --git a/rpython/jit/metainterp/warmstate.py b/rpython/jit/metainterp/warmstate.py
--- a/rpython/jit/metainterp/warmstate.py
+++ b/rpython/jit/metainterp/warmstate.py
@@ -291,6 +291,11 @@
             if self.warmrunnerdesc.memory_manager:
                 self.warmrunnerdesc.memory_manager.max_unroll_loops = value
 
+    def set_param_max_unroll_recursion(self, value):
+        if self.warmrunnerdesc:
+            if self.warmrunnerdesc.memory_manager:
+                self.warmrunnerdesc.memory_manager.max_unroll_recursion = value
+
     def disable_noninlinable_function(self, greenkey):
         cell = self.JitCell.ensure_jit_cell_at_key(greenkey)
         cell.flags |= JC_DONT_TRACE_HERE
diff --git a/rpython/rlib/jit.py b/rpython/rlib/jit.py
--- a/rpython/rlib/jit.py
+++ b/rpython/rlib/jit.py
@@ -463,6 +463,7 @@
     'max_unroll_loops': 'number of extra unrollings a loop can cause',
     'enable_opts': 'INTERNAL USE ONLY (MAY NOT WORK OR LEAD TO CRASHES): '
                    'optimizations to enable, or all = %s' % ENABLE_ALL_OPTS,
+    'max_unroll_recursion': 'how many levels deep to unroll a recursive function'
     }
 
 PARAMETERS = {'threshold': 1039, # just above 1024, prime
@@ -476,6 +477,7 @@
               'max_retrace_guards': 15,
               'max_unroll_loops': 0,
               'enable_opts': 'all',
+              'max_unroll_recursion': 7,
               }
 unroll_parameters = unrolling_iterable(PARAMETERS.items())
 

From noreply at buildbot.pypy.org  Thu Nov 13 14:23:37 2014
From: noreply at buildbot.pypy.org (fijal)
Date: Thu, 13 Nov 2014 14:23:37 +0100 (CET)
Subject: [pypy-commit] pypy optresult: type specialize instances of resop
Message-ID: <20141113132337.023D81C0548@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: optresult
Changeset: r74499:0ea26299d718
Date: 2014-11-12 08:38 +0200
http://bitbucket.org/pypy/pypy/changeset/0ea26299d718/

Log:	type specialize instances of resop

diff --git a/rpython/jit/metainterp/resoperation.py b/rpython/jit/metainterp/resoperation.py
--- a/rpython/jit/metainterp/resoperation.py
+++ b/rpython/jit/metainterp/resoperation.py
@@ -1,6 +1,7 @@
-from rpython.rlib.objectmodel import we_are_translated
+from rpython.rlib.objectmodel import we_are_translated, specialize
 
 
+ at specialize.argtype(2)
 def ResOperation(opnum, args, result, descr=None):
     cls = opclasses[opnum]
     op = cls(result)
@@ -12,6 +13,12 @@
         elif op.is_guard():
             assert not descr.final_descr
         op.setdescr(descr)
+    if isinstance(result, int):
+        op._resint = result
+    elif isinstance(result, float):
+        op._resfloat = result
+    else:
+        op._resref = result
     return op
 
 
@@ -24,7 +31,7 @@
     opnum = 0
     _cls_has_bool_result = False
 
-    _attrs_ = ('result',)
+    _attrs_ = ()
 
     def __init__(self, result):
         self.result = result
@@ -231,6 +238,37 @@
         newop.setfailargs(self.getfailargs())
         return newop
 
+# ===========
+# type mixins
+# ===========
+
+class IntOp(object):
+    _mixin_ = True
+
+    def getint(self):
+        return self._resint
+
+    def setint(self, intval):
+        self._resint = intval
+
+class FloatOp(object):
+    _mixin_ = True
+
+    def getfloat(self):
+        return self._resfloat
+
+    def setfloat(self, floatval):
+        self._resfloat = floatval
+
+class RefOp(object):
+    _mixin_ = True
+
+    def getref(self):
+        return self._resref
+
+    def setref(self, refval):
+        self._resref = refval
+
 # ============
 # arity mixins
 # ============
@@ -375,164 +413,174 @@
 
 # ____________________________________________________________
 
+""" All the operations are desribed like this:
+
+NAME/no-of-args-or-*[b][d]/type-of-result-or-none
+
+if b is present it means the operation produces a boolean
+if d is present it means there is a descr
+type of result can be one of r i f, * for anything, + for i or f or nothing
+"""
+
 _oplist = [
     '_FINAL_FIRST',
-    'JUMP/*d',
-    'FINISH/*d',
+    'JUMP/*d/',
+    'FINISH/*d/',
     '_FINAL_LAST',
 
-    'LABEL/*d',
+    'LABEL/*d/',
 
     '_GUARD_FIRST',
     '_GUARD_FOLDABLE_FIRST',
-    'GUARD_TRUE/1d',
-    'GUARD_FALSE/1d',
-    'GUARD_VALUE/2d',
-    'GUARD_CLASS/2d',
-    'GUARD_NONNULL/1d',
-    'GUARD_ISNULL/1d',
-    'GUARD_NONNULL_CLASS/2d',
+    'GUARD_TRUE/1d/',
+    'GUARD_FALSE/1d/',
+    'GUARD_VALUE/2d/',
+    'GUARD_CLASS/2d/',
+    'GUARD_NONNULL/1d/',
+    'GUARD_ISNULL/1d/',
+    'GUARD_NONNULL_CLASS/2d/',
     '_GUARD_FOLDABLE_LAST',
-    'GUARD_NO_EXCEPTION/0d',    # may be called with an exception currently set
-    'GUARD_EXCEPTION/1d',       # may be called with an exception currently set
-    'GUARD_NO_OVERFLOW/0d',
-    'GUARD_OVERFLOW/0d',
-    'GUARD_NOT_FORCED/0d',      # may be called with an exception currently set
-    'GUARD_NOT_FORCED_2/0d',    # same as GUARD_NOT_FORCED, but for finish()
-    'GUARD_NOT_INVALIDATED/0d',
-    'GUARD_FUTURE_CONDITION/0d', # is removable, may be patched by an optimization
+    'GUARD_NO_EXCEPTION/0d/',   # may be called with an exception currently set
+    'GUARD_EXCEPTION/1d/r',     # may be called with an exception currently set
+    'GUARD_NO_OVERFLOW/0d/',
+    'GUARD_OVERFLOW/0d/',
+    'GUARD_NOT_FORCED/0d/',      # may be called with an exception currently set
+    'GUARD_NOT_FORCED_2/0d/',    # same as GUARD_NOT_FORCED, but for finish()
+    'GUARD_NOT_INVALIDATED/0d/',
+    'GUARD_FUTURE_CONDITION/0d/',
+    # is removable, may be patched by an optimization
     '_GUARD_LAST', # ----- end of guard operations -----
 
     '_NOSIDEEFFECT_FIRST', # ----- start of no_side_effect operations -----
     '_ALWAYS_PURE_FIRST', # ----- start of always_pure operations -----
-    'INT_ADD/2',
-    'INT_SUB/2',
-    'INT_MUL/2',
-    'INT_FLOORDIV/2',
-    'UINT_FLOORDIV/2',
-    'INT_MOD/2',
-    'INT_AND/2',
-    'INT_OR/2',
-    'INT_XOR/2',
-    'INT_RSHIFT/2',
-    'INT_LSHIFT/2',
-    'UINT_RSHIFT/2',
-    'FLOAT_ADD/2',
-    'FLOAT_SUB/2',
-    'FLOAT_MUL/2',
-    'FLOAT_TRUEDIV/2',
-    'FLOAT_NEG/1',
-    'FLOAT_ABS/1',
-    'CAST_FLOAT_TO_INT/1',          # don't use for unsigned ints; we would
-    'CAST_INT_TO_FLOAT/1',          # need some messy code in the backend
-    'CAST_FLOAT_TO_SINGLEFLOAT/1',
-    'CAST_SINGLEFLOAT_TO_FLOAT/1',
-    'CONVERT_FLOAT_BYTES_TO_LONGLONG/1',
-    'CONVERT_LONGLONG_BYTES_TO_FLOAT/1',
+    'INT_ADD/2/i',
+    'INT_SUB/2/i',
+    'INT_MUL/2/i',
+    'INT_FLOORDIV/2/i',
+    'UINT_FLOORDIV/2/i',
+    'INT_MOD/2/i',
+    'INT_AND/2/i',
+    'INT_OR/2/i',
+    'INT_XOR/2/i',
+    'INT_RSHIFT/2/i',
+    'INT_LSHIFT/2/i',
+    'UINT_RSHIFT/2/i',
+    'FLOAT_ADD/2/f',
+    'FLOAT_SUB/2/f',
+    'FLOAT_MUL/2/f',
+    'FLOAT_TRUEDIV/2/f',
+    'FLOAT_NEG/1/f',
+    'FLOAT_ABS/1/f',
+    'CAST_FLOAT_TO_INT/1/i',          # don't use for unsigned ints; we would
+    'CAST_INT_TO_FLOAT/1/f',          # need some messy code in the backend
+    'CAST_FLOAT_TO_SINGLEFLOAT/1/f',
+    'CAST_SINGLEFLOAT_TO_FLOAT/1/f',
+    'CONVERT_FLOAT_BYTES_TO_LONGLONG/1/i',
+    'CONVERT_LONGLONG_BYTES_TO_FLOAT/1/f',
     #
-    'INT_LT/2b',
-    'INT_LE/2b',
-    'INT_EQ/2b',
-    'INT_NE/2b',
-    'INT_GT/2b',
-    'INT_GE/2b',
-    'UINT_LT/2b',
-    'UINT_LE/2b',
-    'UINT_GT/2b',
-    'UINT_GE/2b',
-    'FLOAT_LT/2b',
-    'FLOAT_LE/2b',
-    'FLOAT_EQ/2b',
-    'FLOAT_NE/2b',
-    'FLOAT_GT/2b',
-    'FLOAT_GE/2b',
+    'INT_LT/2b/i',
+    'INT_LE/2b/i',
+    'INT_EQ/2b/i',
+    'INT_NE/2b/i',
+    'INT_GT/2b/i',
+    'INT_GE/2b/i',
+    'UINT_LT/2b/i',
+    'UINT_LE/2b/i',
+    'UINT_GT/2b/i',
+    'UINT_GE/2b/i',
+    'FLOAT_LT/2b/i',
+    'FLOAT_LE/2b/i',
+    'FLOAT_EQ/2b/i',
+    'FLOAT_NE/2b/i',
+    'FLOAT_GT/2b/i',
+    'FLOAT_GE/2b/i',
     #
-    'INT_IS_ZERO/1b',
-    'INT_IS_TRUE/1b',
-    'INT_NEG/1',
-    'INT_INVERT/1',
-    'INT_FORCE_GE_ZERO/1',
+    'INT_IS_ZERO/1b/i',
+    'INT_IS_TRUE/1b/i',
+    'INT_NEG/1/i',
+    'INT_INVERT/1/i',
+    'INT_FORCE_GE_ZERO/1/i',
     #
-    'SAME_AS/1',      # gets a Const or a Box, turns it into another Box
-    'CAST_PTR_TO_INT/1',
-    'CAST_INT_TO_PTR/1',
+    'SAME_AS/1/*',      # gets a Const or a Box, turns it into another Box
+    'CAST_PTR_TO_INT/1/i',
+    'CAST_INT_TO_PTR/1/r',
     #
-    'PTR_EQ/2b',
-    'PTR_NE/2b',
-    'INSTANCE_PTR_EQ/2b',
-    'INSTANCE_PTR_NE/2b',
+    'PTR_EQ/2b/i',
+    'PTR_NE/2b/i',
+    'INSTANCE_PTR_EQ/2b/i',
+    'INSTANCE_PTR_NE/2b/i',
     #
-    'ARRAYLEN_GC/1d',
-    'STRLEN/1',
-    'STRGETITEM/2',
-    'GETFIELD_GC_PURE/1d',
-    'GETFIELD_RAW_PURE/1d',
-    'GETARRAYITEM_GC_PURE/2d',
-    'GETARRAYITEM_RAW_PURE/2d',
-    'UNICODELEN/1',
-    'UNICODEGETITEM/2',
+    'ARRAYLEN_GC/1d/i',
+    'STRLEN/1/i',
+    'STRGETITEM/2/i',
+    'GETFIELD_GC_PURE/1d/*',
+    'GETFIELD_RAW_PURE/1d/*',
+    'GETARRAYITEM_GC_PURE/2d/*',
+    'GETARRAYITEM_RAW_PURE/2d/*',
+    'UNICODELEN/1/i',
+    'UNICODEGETITEM/2/i',
     #
     '_ALWAYS_PURE_LAST',  # ----- end of always_pure operations -----
 
-    'GETARRAYITEM_GC/2d',
-    'GETARRAYITEM_RAW/2d',
-    'GETINTERIORFIELD_GC/2d',
-    'RAW_LOAD/2d',
-    'GETFIELD_GC/1d',
-    'GETFIELD_RAW/1d',
+    'GETARRAYITEM_GC/2d/*',
+    'GETARRAYITEM_RAW/2d/+',
+    'GETINTERIORFIELD_GC/2d/*',
+    'RAW_LOAD/2d/+',
+    'GETFIELD_GC/1d/*',
+    'GETFIELD_RAW/1d/+',
     '_MALLOC_FIRST',
-    'NEW/0d',             #-> GcStruct, gcptrs inside are zeroed (not the rest)
-    'NEW_WITH_VTABLE/1',  #-> GcStruct with vtable, gcptrs inside are zeroed
-    'NEW_ARRAY/1d',       #-> GcArray, not zeroed. only for arrays of primitives
-    'NEW_ARRAY_CLEAR/1d', #-> GcArray, fully zeroed
-    'NEWSTR/1',           #-> STR, the hash field is zeroed
-    'NEWUNICODE/1',       #-> UNICODE, the hash field is zeroed
+    'NEW/0d/r',           #-> GcStruct, gcptrs inside are zeroed (not the rest)
+    'NEW_WITH_VTABLE/1/r',#-> GcStruct with vtable, gcptrs inside are zeroed
+    'NEW_ARRAY/1d/r',     #-> GcArray, not zeroed. only for arrays of primitives
+    'NEW_ARRAY_CLEAR/1d/r',#-> GcArray, fully zeroed
+    'NEWSTR/1/r',         #-> STR, the hash field is zeroed
+    'NEWUNICODE/1/r',     #-> UNICODE, the hash field is zeroed
     '_MALLOC_LAST',
-    'FORCE_TOKEN/0',
-    'VIRTUAL_REF/2',         # removed before it's passed to the backend
-    'MARK_OPAQUE_PTR/1b',
+    'FORCE_TOKEN/0/i',
+    'VIRTUAL_REF/2/r',    # removed before it's passed to the backend
+    'MARK_OPAQUE_PTR/1b/',
     # this one has no *visible* side effect, since the virtualizable
     # must be forced, however we need to execute it anyway
     '_NOSIDEEFFECT_LAST', # ----- end of no_side_effect operations -----
 
-    'INCREMENT_DEBUG_COUNTER/1',
-    'SETARRAYITEM_GC/3d',
-    'SETARRAYITEM_RAW/3d',
-    'SETINTERIORFIELD_GC/3d',
-    'SETINTERIORFIELD_RAW/3d',    # right now, only used by tests
-    'RAW_STORE/3d',
-    'SETFIELD_GC/2d',
-    'ZERO_PTR_FIELD/2', # only emitted by the rewrite, clears a pointer field
+    'INCREMENT_DEBUG_COUNTER/1/',
+    'SETARRAYITEM_GC/3d/',
+    'SETARRAYITEM_RAW/3d/',
+    'SETINTERIORFIELD_GC/3d/',
+    'SETINTERIORFIELD_RAW/3d/',    # right now, only used by tests
+    'RAW_STORE/3d/',
+    'SETFIELD_GC/2d/',
+    'ZERO_PTR_FIELD/2/', # only emitted by the rewrite, clears a pointer field
                         # at a given constant offset, no descr
-    'ZERO_ARRAY/3d',    # only emitted by the rewrite, clears (part of) an array
+    'ZERO_ARRAY/3d/',   # only emitted by the rewrite, clears (part of) an array
                         # [arraygcptr, firstindex, length], descr=ArrayDescr
-    'SETFIELD_RAW/2d',
-    'STRSETITEM/3',
-    'UNICODESETITEM/3',
-    'COND_CALL_GC_WB/1d',       # [objptr] (for the write barrier)
-    'COND_CALL_GC_WB_ARRAY/2d', # [objptr, arrayindex] (write barr. for array)
-    'DEBUG_MERGE_POINT/*',      # debugging only
-    'JIT_DEBUG/*',              # debugging only
-    'VIRTUAL_REF_FINISH/2',   # removed before it's passed to the backend
-    'COPYSTRCONTENT/5',       # src, dst, srcstart, dststart, length
-    'COPYUNICODECONTENT/5',
-    'QUASIIMMUT_FIELD/1d',    # [objptr], descr=SlowMutateDescr
-    'RECORD_KNOWN_CLASS/2',   # [objptr, clsptr]
-    'KEEPALIVE/1',
+    'SETFIELD_RAW/2d/',
+    'STRSETITEM/3/',
+    'UNICODESETITEM/3/',
+    'COND_CALL_GC_WB/1d/',       # [objptr] (for the write barrier)
+    'COND_CALL_GC_WB_ARRAY/2d/', # [objptr, arrayindex] (write barr. for array)
+    'DEBUG_MERGE_POINT/*/',      # debugging only
+    'JIT_DEBUG/*/',              # debugging only
+    'VIRTUAL_REF_FINISH/2/',   # removed before it's passed to the backend
+    'COPYSTRCONTENT/5/',       # src, dst, srcstart, dststart, length
+    'COPYUNICODECONTENT/5/',
+    'QUASIIMMUT_FIELD/1d/',    # [objptr], descr=SlowMutateDescr
+    'RECORD_KNOWN_CLASS/2/',   # [objptr, clsptr]
+    'KEEPALIVE/1/',
 
     '_CANRAISE_FIRST', # ----- start of can_raise operations -----
     '_CALL_FIRST',
-    'CALL/*d',
-    'COND_CALL/*d', # a conditional call, with first argument as a condition
-    'CALL_ASSEMBLER/*d',  # call already compiled assembler
-    'CALL_MAY_FORCE/*d',
-    'CALL_LOOPINVARIANT/*d',
-    'CALL_RELEASE_GIL/*d',  # release the GIL and "close the stack" for asmgcc
-    'CALL_PURE/*d',             # removed before it's passed to the backend
-    'CALL_MALLOC_GC/*d',      # like CALL, but NULL => propagate MemoryError
-    'CALL_MALLOC_NURSERY/1',  # nursery malloc, const number of bytes, zeroed
-    'CALL_MALLOC_NURSERY_VARSIZE/3d',
-    'CALL_MALLOC_NURSERY_VARSIZE_FRAME/1',
+    'CALL/*d/*',
+    'COND_CALL/*d/*', # a conditional call, with first argument as a condition
+    'CALL_ASSEMBLER/*d/*',  # call already compiled assembler
+    'CALL_MAY_FORCE/*d/*',
+    'CALL_LOOPINVARIANT/*d/*',
+    'CALL_RELEASE_GIL/*d/*',  # release the GIL and "close the stack" for asmgcc
+    'CALL_PURE/*d/*',             # removed before it's passed to the backend
+    'CALL_MALLOC_GC/*d/r',      # like CALL, but NULL => propagate MemoryError
+    'CALL_MALLOC_NURSERY/1/r',  # nursery malloc, const number of bytes, zeroed
+    'CALL_MALLOC_NURSERY_VARSIZE/3d/r',
+    'CALL_MALLOC_NURSERY_VARSIZE_FRAME/1/r',
     # nursery malloc, non-const number of bytes, zeroed
     # note that the number of bytes must be well known to be small enough
     # to fulfill allocating in the nursery rules (and no card markings)
@@ -540,9 +588,9 @@
     '_CANRAISE_LAST', # ----- end of can_raise operations -----
 
     '_OVF_FIRST', # ----- start of is_ovf operations -----
-    'INT_ADD_OVF/2',
-    'INT_SUB_OVF/2',
-    'INT_MUL_OVF/2',
+    'INT_ADD_OVF/2/i',
+    'INT_SUB_OVF/2/i',
+    'INT_MUL_OVF/2/i',
     '_OVF_LAST', # ----- end of is_ovf operations -----
     '_LAST',     # for the backend to add more internal operations
 ]
@@ -559,11 +607,10 @@
 
 
 def setup(debug_print=False):
-    for i, name in enumerate(_oplist):
-        if debug_print:
-            print '%30s = %d' % (name, i)
+    i = 0
+    for name in _oplist:
         if '/' in name:
-            name, arity = name.split('/')
+            name, arity, result = name.split('/')
             withdescr = 'd' in arity
             boolresult = 'b' in arity
             arity = arity.rstrip('db')
@@ -572,32 +619,40 @@
             else:
                 arity = int(arity)
         else:
-            arity, withdescr, boolresult = -1, True, False       # default
-        setattr(rop, name, i)
+            arity, withdescr, boolresult, result = -1, True, False, None       # default
+        if result == '*':
+            result = 'rfiN'
+        elif result == '+':
+            result = 'fiN'
+        elif result == '':
+            result = 'N'
         if not name.startswith('_'):
-            opname[i] = name
-            cls = create_class_for_op(name, i, arity, withdescr)
-            cls._cls_has_bool_result = boolresult
-        else:
-            cls = None
-        opclasses.append(cls)
-        oparity.append(arity)
-        opwithdescr.append(withdescr)
-    assert len(opclasses) == len(oparity) == len(opwithdescr) == len(_oplist)
+            for r in result:
+                cls_name = name + '_' + r
+                setattr(rop, cls_name, i)
+                opname[i] = cls_name
+                cls = create_class_for_op(cls_name, i, arity, withdescr, r)
+                cls._cls_has_bool_result = boolresult
+                opclasses.append(cls)
+                oparity.append(arity)
+                opwithdescr.append(withdescr)
+                if debug_print:
+                    print '%30s = %d' % (cls_name, i)
+                i += 1
 
-def get_base_class(mixin, base):
+def get_base_class(mixins, base):
     try:
-        return get_base_class.cache[(mixin, base)]
+        return get_base_class.cache[(base,) + mixins]
     except KeyError:
-        arity_name = mixin.__name__[:-2]  # remove the trailing "Op"
+        arity_name = mixins[0].__name__[:-2]  # remove the trailing "Op"
         name = arity_name + base.__name__ # something like BinaryPlainResOp
-        bases = (mixin, base)
+        bases = mixins + (base,)
         cls = type(name, bases, {})
-        get_base_class.cache[(mixin, base)] = cls
+        get_base_class.cache[(base,) + mixins] = cls
         return cls
 get_base_class.cache = {}
 
-def create_class_for_op(name, opnum, arity, withdescr):
+def create_class_for_op(name, opnum, arity, withdescr, result_type):
     arity2mixin = {
         0: NullaryOp,
         1: UnaryOp,
@@ -613,10 +668,18 @@
         baseclass = ResOpWithDescr
     else:
         baseclass = PlainResOp
-    mixin = arity2mixin.get(arity, N_aryOp)
+    mixins = [arity2mixin.get(arity, N_aryOp)]
+    if result_type == 'i':
+        mixins.append(IntOp)
+    elif result_type == 'f':
+        mixins.append(FloatOp)
+    elif result_type == 'r':
+        mixins.append(RefOp)
+    else:
+        assert result_type == 'N'
 
     cls_name = '%s_OP' % name
-    bases = (get_base_class(mixin, baseclass),)
+    bases = (get_base_class(tuple(mixins), baseclass),)
     dic = {'opnum': opnum}
     return type(cls_name, bases, dic)
 
@@ -624,51 +687,51 @@
 del _oplist
 
 opboolinvers = {
-    rop.INT_EQ: rop.INT_NE,
-    rop.INT_NE: rop.INT_EQ,
-    rop.INT_LT: rop.INT_GE,
-    rop.INT_GE: rop.INT_LT,
-    rop.INT_GT: rop.INT_LE,
-    rop.INT_LE: rop.INT_GT,
+    rop.INT_EQ_i: rop.INT_NE_i,
+    rop.INT_NE_i: rop.INT_EQ_i,
+    rop.INT_LT_i: rop.INT_GE_i,
+    rop.INT_GE_i: rop.INT_LT_i,
+    rop.INT_GT_i: rop.INT_LE_i,
+    rop.INT_LE_i: rop.INT_GT_i,
 
-    rop.UINT_LT: rop.UINT_GE,
-    rop.UINT_GE: rop.UINT_LT,
-    rop.UINT_GT: rop.UINT_LE,
-    rop.UINT_LE: rop.UINT_GT,
+    rop.UINT_LT_i: rop.UINT_GE_i,
+    rop.UINT_GE_i: rop.UINT_LT_i,
+    rop.UINT_GT_i: rop.UINT_LE_i,
+    rop.UINT_LE_i: rop.UINT_GT_i,
 
-    rop.FLOAT_EQ: rop.FLOAT_NE,
-    rop.FLOAT_NE: rop.FLOAT_EQ,
-    rop.FLOAT_LT: rop.FLOAT_GE,
-    rop.FLOAT_GE: rop.FLOAT_LT,
-    rop.FLOAT_GT: rop.FLOAT_LE,
-    rop.FLOAT_LE: rop.FLOAT_GT,
+    rop.FLOAT_EQ_i: rop.FLOAT_NE_i,
+    rop.FLOAT_NE_i: rop.FLOAT_EQ_i,
+    rop.FLOAT_LT_i: rop.FLOAT_GE_i,
+    rop.FLOAT_GE_i: rop.FLOAT_LT_i,
+    rop.FLOAT_GT_i: rop.FLOAT_LE_i,
+    rop.FLOAT_LE_i: rop.FLOAT_GT_i,
 
-    rop.PTR_EQ: rop.PTR_NE,
-    rop.PTR_NE: rop.PTR_EQ,
+    rop.PTR_EQ_i: rop.PTR_NE_i,
+    rop.PTR_NE_i: rop.PTR_EQ_i,
 }
 
 opboolreflex = {
-    rop.INT_EQ: rop.INT_EQ,
-    rop.INT_NE: rop.INT_NE,
-    rop.INT_LT: rop.INT_GT,
-    rop.INT_GE: rop.INT_LE,
-    rop.INT_GT: rop.INT_LT,
-    rop.INT_LE: rop.INT_GE,
+    rop.INT_EQ_i: rop.INT_EQ_i,
+    rop.INT_NE_i: rop.INT_NE_i,
+    rop.INT_LT_i: rop.INT_GT_i,
+    rop.INT_GE_i: rop.INT_LE_i,
+    rop.INT_GT_i: rop.INT_LT_i,
+    rop.INT_LE_i: rop.INT_GE_i,
 
-    rop.UINT_LT: rop.UINT_GT,
-    rop.UINT_GE: rop.UINT_LE,
-    rop.UINT_GT: rop.UINT_LT,
-    rop.UINT_LE: rop.UINT_GE,
+    rop.UINT_LT_i: rop.UINT_GT_i,
+    rop.UINT_GE_i: rop.UINT_LE_i,
+    rop.UINT_GT_i: rop.UINT_LT_i,
+    rop.UINT_LE_i: rop.UINT_GE_i,
 
-    rop.FLOAT_EQ: rop.FLOAT_EQ,
-    rop.FLOAT_NE: rop.FLOAT_NE,
-    rop.FLOAT_LT: rop.FLOAT_GT,
-    rop.FLOAT_GE: rop.FLOAT_LE,
-    rop.FLOAT_GT: rop.FLOAT_LT,
-    rop.FLOAT_LE: rop.FLOAT_GE,
+    rop.FLOAT_EQ_i: rop.FLOAT_EQ_i,
+    rop.FLOAT_NE_i: rop.FLOAT_NE_i,
+    rop.FLOAT_LT_i: rop.FLOAT_GT_i,
+    rop.FLOAT_GE_i: rop.FLOAT_LE_i,
+    rop.FLOAT_GT_i: rop.FLOAT_LT_i,
+    rop.FLOAT_LE_i: rop.FLOAT_GE_i,
 
-    rop.PTR_EQ: rop.PTR_EQ,
-    rop.PTR_NE: rop.PTR_NE,
+    rop.PTR_EQ_i: rop.PTR_EQ_i,
+    rop.PTR_NE_i: rop.PTR_NE_i,
 }
 
 

From noreply at buildbot.pypy.org  Thu Nov 13 14:23:38 2014
From: noreply at buildbot.pypy.org (fijal)
Date: Thu, 13 Nov 2014 14:23:38 +0100 (CET)
Subject: [pypy-commit] pypy optresult: hack enough to import optimizeopt
	tests
Message-ID: <20141113132338.404EA1C0548@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: optresult
Changeset: r74500:64caf45b4229
Date: 2014-11-12 13:54 +0200
http://bitbucket.org/pypy/pypy/changeset/64caf45b4229/

Log:	hack enough to import optimizeopt tests

diff --git a/rpython/jit/metainterp/executor.py b/rpython/jit/metainterp/executor.py
--- a/rpython/jit/metainterp/executor.py
+++ b/rpython/jit/metainterp/executor.py
@@ -75,32 +75,48 @@
         return None
     raise AssertionError("bad rettype")
 
-do_call_loopinvariant = do_call
-do_call_may_force = do_call
+do_call_r = do_call
+do_call_i = do_call
+do_call_f = do_call
+do_call_n = do_call
+do_call_loopinvariant_r = do_call
+do_call_loopinvariant_i = do_call
+do_call_loopinvariant_f = do_call
+do_call_loopinvariant_n = do_call
+do_call_may_force_r = do_call
+do_call_may_force_i = do_call
+do_call_may_force_f = do_call
+do_call_may_force_n = do_call
 
 def do_cond_call(cpu, metainterp, argboxes, descr):
     condbox = argboxes[0]
     if condbox.getint():
-        do_call(cpu, metainterp, argboxes[1:], descr)
+        do_call_n(cpu, metainterp, argboxes[1:], descr)
 
-def do_getarrayitem_gc(cpu, _, arraybox, indexbox, arraydescr):
+def do_getarrayitem_gc_i(cpu, _, arraybox, indexbox, arraydescr):
     array = arraybox.getref_base()
     index = indexbox.getint()
-    if arraydescr.is_array_of_pointers():
-        return BoxPtr(cpu.bh_getarrayitem_gc_r(array, index, arraydescr))
-    elif arraydescr.is_array_of_floats():
-        return BoxFloat(cpu.bh_getarrayitem_gc_f(array, index, arraydescr))
-    else:
-        return BoxInt(cpu.bh_getarrayitem_gc_i(array, index, arraydescr))
+    return cpu.bh_getarrayitem_gc_i(array, index, arraydescr)
 
-def do_getarrayitem_raw(cpu, _, arraybox, indexbox, arraydescr):
+def do_getarrayitem_gc_r(cpu, _, arraybox, indexbox, arraydescr):
+    array = arraybox.getref_base()
+    index = indexbox.getint()
+    return cpu.bh_getarrayitem_gc_r(array, index, arraydescr)
+
+def do_getarrayitem_gc_f(cpu, _, arraybox, indexbox, arraydescr):
+    array = arraybox.getref_base()
+    index = indexbox.getint()
+    return cpu.bh_getarrayitem_gc_f(array, index, arraydescr)
+
+def do_getarrayitem_raw_i(cpu, _, arraybox, indexbox, arraydescr):
     array = arraybox.getint()
     index = indexbox.getint()
-    assert not arraydescr.is_array_of_pointers()
-    if arraydescr.is_array_of_floats():
-        return BoxFloat(cpu.bh_getarrayitem_raw_f(array, index, arraydescr))
-    else:
-        return BoxInt(cpu.bh_getarrayitem_raw_i(array, index, arraydescr))
+    return cpu.bh_getarrayitem_raw_i(array, index, arraydescr)
+
+def do_getarrayitem_raw_f(cpu, _, arraybox, indexbox, arraydescr):
+    array = arraybox.getint()
+    index = indexbox.getint()
+    return cpu.bh_getarrayitem_raw_f(array, index, arraydescr)
 
 def do_setarrayitem_gc(cpu, _, arraybox, indexbox, itembox, arraydescr):
     array = arraybox.getref_base()
@@ -146,14 +162,17 @@
     else:
         cpu.bh_setinteriorfield_gc_i(array, index, valuebox.getint(), descr)
 
-def do_getfield_gc(cpu, _, structbox, fielddescr):
+def do_getfield_gc_i(cpu, _, structbox, fielddescr):
     struct = structbox.getref_base()
-    if fielddescr.is_pointer_field():
-        return BoxPtr(cpu.bh_getfield_gc_r(struct, fielddescr))
-    elif fielddescr.is_float_field():
-        return BoxFloat(cpu.bh_getfield_gc_f(struct, fielddescr))
-    else:
-        return BoxInt(cpu.bh_getfield_gc_i(struct, fielddescr))
+    return cpu.bh_getfield_gc_i(struct, fielddescr)
+
+def do_getfield_gc_r(cpu, _, structbox, fielddescr):
+    struct = structbox.getref_base()
+    return cpu.bh_getfield_gc_r(struct, fielddescr)
+
+def do_getfield_gc_f(cpu, _, structbox, fielddescr):
+    struct = structbox.getref_base()
+    return cpu.bh_getfield_gc_f(struct, fielddescr)
 
 def do_getfield_raw(cpu, _, structbox, fielddescr):
     check_descr(fielddescr)
@@ -246,8 +265,14 @@
         z = 0
     return BoxInt(z)
 
-def do_same_as(cpu, _, box):
-    return box.clonebox()
+def do_same_as_i(cpu, _, v):
+    return v
+
+def do_same_as_r(cpu, _, v):
+    return v
+
+def do_same_as_f(cpu, _, v):
+    return v
 
 def do_copystrcontent(cpu, _, srcbox, dstbox,
                       srcstartbox, dststartbox, lengthbox):
@@ -301,8 +326,8 @@
                 continue
             #
             # Maybe the same without the _PURE suffix?
-            if key.endswith('_PURE'):
-                key = key[:-5]
+            if key[-7:-2] == '_PURE':
+                key = key[:-7] + key[-2:]
                 name = 'do_' + key.lower()
                 if name in globals():
                     execute[value] = globals()[name]
@@ -321,7 +346,10 @@
                     execute[value] = func
                     continue
             if value in (rop.FORCE_TOKEN,
-                         rop.CALL_ASSEMBLER,
+                         rop.CALL_ASSEMBLER_R,
+                         rop.CALL_ASSEMBLER_F,
+                         rop.CALL_ASSEMBLER_I,
+                         rop.CALL_ASSEMBLER_N,
                          rop.INCREMENT_DEBUG_COUNTER,
                          rop.COND_CALL_GC_WB,
                          rop.COND_CALL_GC_WB_ARRAY,
@@ -331,7 +359,10 @@
                          rop.JIT_DEBUG,
                          rop.SETARRAYITEM_RAW,
                          rop.SETINTERIORFIELD_RAW,
-                         rop.CALL_RELEASE_GIL,
+                         rop.CALL_RELEASE_GIL_I,
+                         rop.CALL_RELEASE_GIL_R,
+                         rop.CALL_RELEASE_GIL_F,
+                         rop.CALL_RELEASE_GIL_N,
                          rop.QUASIIMMUT_FIELD,
                          rop.CALL_MALLOC_GC,
                          rop.CALL_MALLOC_NURSERY,
@@ -352,10 +383,7 @@
             return None
     if list(func.argtypes).count('d') > 1:
         return None
-    if func.resulttype not in ('i', 'r', 'f', None):
-        return None
     argtypes = unrolling_iterable(func.argtypes)
-    resulttype = func.resulttype
     #
     def do(cpu, _, *argboxes):
         newargs = ()
@@ -375,12 +403,7 @@
             newargs = newargs + (value,)
         assert not argboxes
         #
-        result = func(*newargs)
-        #
-        if resulttype == 'i': return BoxInt(result)
-        if resulttype == 'r': return BoxPtr(result)
-        if resulttype == 'f': return BoxFloat(result)
-        return None
+        return func(*newargs)
     #
     do.func_name = 'do_' + name
     return do
@@ -408,6 +431,7 @@
 
 
 def execute(cpu, metainterp, opnum, descr, *argboxes):
+    xxx
     # only for opnums with a fixed arity
     num_args = len(argboxes)
     withdescr = has_descr(opnum)
@@ -422,6 +446,7 @@
 execute._annspecialcase_ = 'specialize:arg(2)'
 
 def execute_varargs(cpu, metainterp, opnum, argboxes, descr):
+    xxxx
     # only for opnums with a variable arity (calls, typically)
     check_descr(descr)
     func = get_execute_function(opnum, -1, True)
@@ -430,6 +455,7 @@
 
 
 def execute_nonspec(cpu, metainterp, opnum, argboxes, descr=None):
+    xxxx
     arity = resoperation.oparity[opnum]
     assert arity == -1 or len(argboxes) == arity
     if resoperation.opwithdescr[opnum]:
diff --git a/rpython/jit/metainterp/heapcache.py b/rpython/jit/metainterp/heapcache.py
--- a/rpython/jit/metainterp/heapcache.py
+++ b/rpython/jit/metainterp/heapcache.py
@@ -76,7 +76,8 @@
                 self.dependencies.setdefault(box, []).append(valuebox)
             else:
                 self._escape(valuebox)
-        elif (opnum == rop.CALL and
+        elif ((opnum == rop.CALL_R or opnum == rop.CALL_I or
+               opnum == rop.CALL_N or opnum == rop.CALL_F) and
               descr.get_extra_info().oopspecindex == descr.get_extra_info().OS_ARRAYCOPY and
               isinstance(argboxes[3], ConstInt) and
               isinstance(argboxes[4], ConstInt) and
@@ -87,8 +88,12 @@
             pass
         # GETFIELD_GC, MARK_OPAQUE_PTR, PTR_EQ, and PTR_NE don't escape their
         # arguments
-        elif (opnum != rop.GETFIELD_GC and
-              opnum != rop.GETFIELD_GC_PURE and
+        elif (opnum != rop.GETFIELD_GC_R and
+              opnum != rop.GETFIELD_GC_I and
+              opnum != rop.GETFIELD_GC_F and
+              opnum != rop.GETFIELD_GC_PURE_R and
+              opnum != rop.GETFIELD_GC_PURE_I and
+              opnum != rop.GETFIELD_GC_PURE_F and
               opnum != rop.MARK_OPAQUE_PTR and
               opnum != rop.PTR_EQ and
               opnum != rop.PTR_NE and
diff --git a/rpython/jit/metainterp/optimizeopt/heap.py b/rpython/jit/metainterp/optimizeopt/heap.py
--- a/rpython/jit/metainterp/optimizeopt/heap.py
+++ b/rpython/jit/metainterp/optimizeopt/heap.py
@@ -294,7 +294,7 @@
         self.force_all_lazy_setfields_and_arrayitems()
         self.clean_caches()
 
-    def optimize_CALL(self, op):
+    def optimize_CALL_I(self, op):
         # dispatch based on 'oopspecindex' to a method that handles
         # specifically the given oopspec call.  For non-oopspec calls,
         # oopspecindex is just zero.
@@ -304,6 +304,9 @@
             if self._optimize_CALL_DICT_LOOKUP(op):
                 return
         self.emit_operation(op)
+    optimize_CALL_F = optimize_CALL_I
+    optimize_CALL_R = optimize_CALL_I
+    optimize_CALL_N = optimize_CALL_I
 
     def _optimize_CALL_DICT_LOOKUP(self, op):
         descrs = op.getdescr().get_extra_info().extradescrs
@@ -432,7 +435,7 @@
                 cf.force_lazy_setfield(self)
         return pendingfields
 
-    def optimize_GETFIELD_GC(self, op):
+    def optimize_GETFIELD_GC_I(self, op):
         structvalue = self.getvalue(op.getarg(0))
         cf = self.field_cache(op.getdescr())
         fieldvalue = cf.getfield_from_cache(self, structvalue)
@@ -445,8 +448,10 @@
         # then remember the result of reading the field
         fieldvalue = self.getvalue(op.result)
         cf.remember_field_value(structvalue, fieldvalue, op)
+    optimize_GETFIELD_GC_R = optimize_GETFIELD_GC_I
+    optimize_GETFIELD_GC_F = optimize_GETFIELD_GC_I
 
-    def optimize_GETFIELD_GC_PURE(self, op):
+    def optimize_GETFIELD_GC_PURE_I(self, op):
         structvalue = self.getvalue(op.getarg(0))
         cf = self.field_cache(op.getdescr())
         fieldvalue = cf.getfield_from_cache(self, structvalue)
@@ -456,6 +461,8 @@
         # default case: produce the operation
         structvalue.ensure_nonnull()
         self.emit_operation(op)
+    optimize_GETFIELD_GC_PURE_R = optimize_GETFIELD_GC_PURE_I
+    optimize_GETFIELD_GC_PURE_F = optimize_GETFIELD_GC_PURE_I
 
     def optimize_SETFIELD_GC(self, op):
         if self.has_pure_result(rop.GETFIELD_GC_PURE, [op.getarg(0)],
@@ -467,7 +474,7 @@
         cf = self.field_cache(op.getdescr())
         cf.do_setfield(self, op)
 
-    def optimize_GETARRAYITEM_GC(self, op):
+    def optimize_GETARRAYITEM_GC_I(self, op):
         arrayvalue = self.getvalue(op.getarg(0))
         indexvalue = self.getvalue(op.getarg(1))
         cf = None
@@ -489,8 +496,10 @@
         if cf is not None:
             fieldvalue = self.getvalue(op.result)
             cf.remember_field_value(arrayvalue, fieldvalue, op)
+    optimize_GETARRAYITEM_GC_R = optimize_GETARRAYITEM_GC_I
+    optimize_GETARRAYITEM_GC_F = optimize_GETARRAYITEM_GC_I
 
-    def optimize_GETARRAYITEM_GC_PURE(self, op):
+    def optimize_GETARRAYITEM_GC_PURE_I(self, op):
         arrayvalue = self.getvalue(op.getarg(0))
         indexvalue = self.getvalue(op.getarg(1))
         cf = None
@@ -509,6 +518,9 @@
         arrayvalue.ensure_nonnull()
         self.emit_operation(op)
 
+    optimize_GETARRAYITEM_GC_PURE_R = optimize_GETARRAYITEM_GC_PURE_I
+    optimize_GETARRAYITEM_GC_PURE_F = optimize_GETARRAYITEM_GC_PURE_I
+
     def optimize_SETARRAYITEM_GC(self, op):
         if self.has_pure_result(rop.GETARRAYITEM_GC_PURE, [op.getarg(0),
                                                            op.getarg(1)],
diff --git a/rpython/jit/metainterp/optimizeopt/intbounds.py b/rpython/jit/metainterp/optimizeopt/intbounds.py
--- a/rpython/jit/metainterp/optimizeopt/intbounds.py
+++ b/rpython/jit/metainterp/optimizeopt/intbounds.py
@@ -372,7 +372,7 @@
         v1.intbound.make_ge(IntLowerBound(0))
         v1.intbound.make_lt(IntUpperBound(256))
 
-    def optimize_GETFIELD_RAW(self, op):
+    def optimize_GETFIELD_RAW_I(self, op):
         self.emit_operation(op)
         descr = op.getdescr()
         if descr.is_integer_bounded():
@@ -380,11 +380,16 @@
             v1.intbound.make_ge(IntLowerBound(descr.get_integer_min()))
             v1.intbound.make_le(IntUpperBound(descr.get_integer_max()))
 
-    optimize_GETFIELD_GC = optimize_GETFIELD_RAW
+    optimize_GETFIELD_RAW_F = optimize_GETFIELD_RAW_I
+    optimize_GETFIELD_GC_I = optimize_GETFIELD_RAW_I
+    optimize_GETFIELD_GC_R = optimize_GETFIELD_RAW_I
+    optimize_GETFIELD_GC_F = optimize_GETFIELD_RAW_I
 
-    optimize_GETINTERIORFIELD_GC = optimize_GETFIELD_RAW
+    optimize_GETINTERIORFIELD_GC_I = optimize_GETFIELD_RAW_I
+    optimize_GETINTERIORFIELD_GC_R = optimize_GETFIELD_RAW_I
+    optimize_GETINTERIORFIELD_GC_F = optimize_GETFIELD_RAW_I
 
-    def optimize_GETARRAYITEM_RAW(self, op):
+    def optimize_GETARRAYITEM_RAW_I(self, op):
         self.emit_operation(op)
         descr = op.getdescr()
         if descr and descr.is_item_integer_bounded():
@@ -392,7 +397,10 @@
             v1.intbound.make_ge(IntLowerBound(descr.get_item_integer_min()))
             v1.intbound.make_le(IntUpperBound(descr.get_item_integer_max()))
 
-    optimize_GETARRAYITEM_GC = optimize_GETARRAYITEM_RAW
+    optimize_GETARRAYITEM_RAW_F = optimize_GETARRAYITEM_RAW_I
+    optimize_GETARRAYITEM_GC_I = optimize_GETARRAYITEM_RAW_I
+    optimize_GETARRAYITEM_GC_F = optimize_GETARRAYITEM_RAW_I
+    optimize_GETARRAYITEM_GC_R = optimize_GETARRAYITEM_RAW_I
 
     def optimize_UNICODEGETITEM(self, op):
         self.emit_operation(op)
diff --git a/rpython/jit/metainterp/optimizeopt/optimizer.py b/rpython/jit/metainterp/optimizeopt/optimizer.py
--- a/rpython/jit/metainterp/optimizeopt/optimizer.py
+++ b/rpython/jit/metainterp/optimizeopt/optimizer.py
@@ -677,8 +677,10 @@
     # These are typically removed already by OptRewrite, but it can be
     # dissabled and unrolling emits some SAME_AS ops to setup the
     # optimizier state. These needs to always be optimized out.
-    def optimize_SAME_AS(self, op):
+    def optimize_SAME_AS_I(self, op):
         self.make_equal_to(op.result, self.getvalue(op.getarg(0)))
+    optimize_SAME_AS_R = optimize_SAME_AS_I
+    optimize_SAME_AS_F = optimize_SAME_AS_I
 
     def optimize_MARK_OPAQUE_PTR(self, op):
         value = self.getvalue(op.getarg(0))
diff --git a/rpython/jit/metainterp/optimizeopt/pure.py b/rpython/jit/metainterp/optimizeopt/pure.py
--- a/rpython/jit/metainterp/optimizeopt/pure.py
+++ b/rpython/jit/metainterp/optimizeopt/pure.py
@@ -56,7 +56,7 @@
         if nextop:
             self.emit_operation(nextop)
 
-    def optimize_CALL_PURE(self, op):
+    def optimize_CALL_PURE_I(self, op):
         # Step 1: check if all arguments are constant
         result = self._can_optimize_call_pure(op)
         if result is not None:
@@ -84,6 +84,9 @@
         args = op.getarglist()
         self.emit_operation(ResOperation(rop.CALL, args, op.result,
                                          op.getdescr()))
+    optimize_CALL_PURE_R = optimize_CALL_PURE_I
+    optimize_CALL_PURE_F = optimize_CALL_PURE_I
+    optimize_CALL_PURE_N = optimize_CALL_PURE_I
 
     def optimize_GUARD_NO_EXCEPTION(self, op):
         if self.last_emitted_operation is REMOVED:
diff --git a/rpython/jit/metainterp/optimizeopt/rewrite.py b/rpython/jit/metainterp/optimizeopt/rewrite.py
--- a/rpython/jit/metainterp/optimizeopt/rewrite.py
+++ b/rpython/jit/metainterp/optimizeopt/rewrite.py
@@ -389,7 +389,7 @@
                               'always fail' % r)
         self.optimize_GUARD_CLASS(op)
 
-    def optimize_CALL_LOOPINVARIANT(self, op):
+    def optimize_CALL_LOOPINVARIANT_I(self, op):
         arg = op.getarg(0)
         # 'arg' must be a Const, because residual_call in codewriter
         # expects a compile-time constant
@@ -408,6 +408,9 @@
         self.emit_operation(op)
         resvalue = self.getvalue(op.result)
         self.loop_invariant_results[key] = resvalue
+    optimize_CALL_LOOPINVARIANT_R = optimize_CALL_LOOPINVARIANT_I
+    optimize_CALL_LOOPINVARIANT_F = optimize_CALL_LOOPINVARIANT_I
+    optimize_CALL_LOOPINVARIANT_N = optimize_CALL_LOOPINVARIANT_I
 
     def optimize_COND_CALL(self, op):
         arg = op.getarg(0)
@@ -478,7 +481,7 @@
     def optimize_INSTANCE_PTR_NE(self, op):
         self._optimize_oois_ooisnot(op, True, True)
 
-    def optimize_CALL(self, op):
+    def optimize_CALL_N(self, op):
         # dispatch based on 'oopspecindex' to a method that handles
         # specifically the given oopspec call.  For non-oopspec calls,
         # oopspecindex is just zero.
@@ -538,7 +541,7 @@
             return True # 0-length arraycopy
         return False
 
-    def optimize_CALL_PURE(self, op):
+    def optimize_CALL_PURE_I(self, op):
         # this removes a CALL_PURE with all constant arguments.
         # Note that it's also done in pure.py.  For now we need both...
         result = self._can_optimize_call_pure(op)
@@ -547,6 +550,9 @@
             self.last_emitted_operation = REMOVED
             return
         self.emit_operation(op)
+    optimize_CALL_PURE_R = optimize_CALL_PURE_I
+    optimize_CALL_PURE_F = optimize_CALL_PURE_I
+    optimize_CALL_PURE_N = optimize_CALL_PURE_I
 
     def optimize_GUARD_NO_EXCEPTION(self, op):
         if self.last_emitted_operation is REMOVED:
@@ -583,8 +589,10 @@
         self.pure(rop.CAST_PTR_TO_INT, [op.result], op.getarg(0))
         self.emit_operation(op)
 
-    def optimize_SAME_AS(self, op):
+    def optimize_SAME_AS_i(self, op):
         self.make_equal_to(op.result, self.getvalue(op.getarg(0)))
+    optimize_SAME_AS_r = optimize_SAME_AS_i
+    optimize_SAME_AS_f = optimize_SAME_AS_i
 
 dispatch_opt = make_dispatcher_method(OptRewrite, 'optimize_',
         default=OptRewrite.emit_operation)
diff --git a/rpython/jit/metainterp/optimizeopt/simplify.py b/rpython/jit/metainterp/optimizeopt/simplify.py
--- a/rpython/jit/metainterp/optimizeopt/simplify.py
+++ b/rpython/jit/metainterp/optimizeopt/simplify.py
@@ -14,14 +14,20 @@
                 self.optimizer.pendingfields = []
         Optimization.emit_operation(self, op)
 
-    def optimize_CALL_PURE(self, op):
+    def optimize_CALL_PURE_I(self, op):
         args = op.getarglist()
         self.emit_operation(ResOperation(rop.CALL, args, op.result,
                                          op.getdescr()))
+    optimize_CALL_PURE_R = optimize_CALL_PURE_I
+    optimize_CALL_PURE_F = optimize_CALL_PURE_I
+    optimize_CALL_PURE_N = optimize_CALL_PURE_I
 
-    def optimize_CALL_LOOPINVARIANT(self, op):
+    def optimize_CALL_LOOPINVARIANT_I(self, op):
         op = op.copy_and_change(rop.CALL)
         self.emit_operation(op)
+    optimize_CALL_LOOPINVARIANT_R = optimize_CALL_LOOPINVARIANT_I
+    optimize_CALL_LOOPINVARIANT_F = optimize_CALL_LOOPINVARIANT_I
+    optimize_CALL_LOOPINVARIANT_N = optimize_CALL_LOOPINVARIANT_I
 
     def optimize_VIRTUAL_REF_FINISH(self, op):
         pass
diff --git a/rpython/jit/metainterp/optimizeopt/virtualize.py b/rpython/jit/metainterp/optimizeopt/virtualize.py
--- a/rpython/jit/metainterp/optimizeopt/virtualize.py
+++ b/rpython/jit/metainterp/optimizeopt/virtualize.py
@@ -571,13 +571,16 @@
         else:
             self.emit_operation(op)
 
-    def optimize_CALL_MAY_FORCE(self, op):
+    def optimize_CALL_MAY_FORCE_I(self, op):
         effectinfo = op.getdescr().get_extra_info()
         oopspecindex = effectinfo.oopspecindex
         if oopspecindex == EffectInfo.OS_JIT_FORCE_VIRTUAL:
             if self._optimize_JIT_FORCE_VIRTUAL(op):
                 return
         self.emit_operation(op)
+    optimize_CALL_MAY_FORCE_R = optimize_CALL_MAY_FORCE_I
+    optimize_CALL_MAY_FORCE_F = optimize_CALL_MAY_FORCE_I
+    optimize_CALL_MAY_FORCE_N = optimize_CALL_MAY_FORCE_I
 
     def optimize_COND_CALL(self, op):
         effectinfo = op.getdescr().get_extra_info()
@@ -653,7 +656,7 @@
                     return True
         return False
 
-    def optimize_GETFIELD_GC(self, op):
+    def optimize_GETFIELD_GC_I(self, op):
         value = self.getvalue(op.getarg(0))
         # If this is an immutable field (as indicated by op.is_always_pure())
         # then it's safe to reuse the virtual's field, even if it has been
@@ -672,10 +675,14 @@
         else:
             value.ensure_nonnull()
             self.emit_operation(op)
+    optimize_GETFIELD_GC_R = optimize_GETFIELD_GC_I
+    optimize_GETFIELD_GC_F = optimize_GETFIELD_GC_I
 
     # note: the following line does not mean that the two operations are
     # completely equivalent, because GETFIELD_GC_PURE is_always_pure().
-    optimize_GETFIELD_GC_PURE = optimize_GETFIELD_GC
+    optimize_GETFIELD_GC_PURE_I = optimize_GETFIELD_GC_I
+    optimize_GETFIELD_GC_PURE_R = optimize_GETFIELD_GC_I
+    optimize_GETFIELD_GC_PURE_F = optimize_GETFIELD_GC_I
 
     def optimize_SETFIELD_GC(self, op):
         value = self.getvalue(op.getarg(0))
@@ -707,7 +714,7 @@
         else:
             self.emit_operation(op)        
 
-    def optimize_CALL(self, op):
+    def optimize_CALL_N(self, op):
         effectinfo = op.getdescr().get_extra_info()
         if effectinfo.oopspecindex == EffectInfo.OS_RAW_MALLOC_VARSIZE_CHAR:
             self.do_RAW_MALLOC_VARSIZE_CHAR(op)
@@ -720,6 +727,7 @@
                 return
         else:
             self.emit_operation(op)
+    optimize_CALL_R = optimize_CALL_N
 
     def do_RAW_MALLOC_VARSIZE_CHAR(self, op):
         sizebox = self.get_constant_box(op.getarg(1))
@@ -761,7 +769,7 @@
             value.ensure_nonnull()
             self.emit_operation(op)
 
-    def optimize_GETARRAYITEM_GC(self, op):
+    def optimize_GETARRAYITEM_GC_I(self, op):
         value = self.getvalue(op.getarg(0))
         if value.is_virtual():
             assert isinstance(value, VArrayValue)
@@ -774,10 +782,14 @@
                 return
         value.ensure_nonnull()
         self.emit_operation(op)
+    optimize_GETARRAYITEM_GC_R = optimize_GETARRAYITEM_GC_I
+    optimize_GETARRAYITEM_GC_F = optimize_GETARRAYITEM_GC_I
 
     # note: the following line does not mean that the two operations are
     # completely equivalent, because GETARRAYITEM_GC_PURE is_always_pure().
-    optimize_GETARRAYITEM_GC_PURE = optimize_GETARRAYITEM_GC
+    optimize_GETARRAYITEM_GC_PURE_I = optimize_GETARRAYITEM_GC_I
+    optimize_GETARRAYITEM_GC_PURE_R = optimize_GETARRAYITEM_GC_I
+    optimize_GETARRAYITEM_GC_PURE_F = optimize_GETARRAYITEM_GC_I
 
     def optimize_SETARRAYITEM_GC(self, op):
         value = self.getvalue(op.getarg(0))
@@ -797,7 +809,7 @@
         offset = basesize + (itemsize*index)
         return offset, itemsize, descr
 
-    def optimize_GETARRAYITEM_RAW(self, op):
+    def optimize_GETARRAYITEM_RAW_I(self, op):
         value = self.getvalue(op.getarg(0))
         if value.is_virtual():
             indexbox = self.get_constant_box(op.getarg(1))
@@ -814,6 +826,7 @@
                 return
         value.ensure_nonnull()
         self.emit_operation(op)
+    optimize_GETARRAYITEM_RAW_F = optimize_GETARRAYITEM_RAW_I
 
     def optimize_SETARRAYITEM_RAW(self, op):
         value = self.getvalue(op.getarg(0))
@@ -839,7 +852,7 @@
         itemsize = cpu.unpack_arraydescr_size(descr)[1]
         return offset, itemsize, descr
 
-    def optimize_RAW_LOAD(self, op):
+    def optimize_RAW_LOAD_I(self, op):
         value = self.getvalue(op.getarg(0))
         if value.is_virtual():
             offsetbox = self.get_constant_box(op.getarg(1))
@@ -856,6 +869,7 @@
                 return
         value.ensure_nonnull()
         self.emit_operation(op)
+    optimize_RAW_LOAD_F = optimize_RAW_LOAD_I
 
     def optimize_RAW_STORE(self, op):
         value = self.getvalue(op.getarg(0))
@@ -874,7 +888,7 @@
         value.ensure_nonnull()
         self.emit_operation(op)
 
-    def optimize_GETINTERIORFIELD_GC(self, op):
+    def optimize_GETINTERIORFIELD_GC_I(self, op):
         value = self.getvalue(op.getarg(0))
         if value.is_virtual():
             indexbox = self.get_constant_box(op.getarg(1))
@@ -889,6 +903,8 @@
                 return
         value.ensure_nonnull()
         self.emit_operation(op)
+    optimize_GETINTERIORFIELD_GC_R = optimize_GETINTERIORFIELD_GC_I
+    optimize_GETINTERIORFIELD_GC_F = optimize_GETINTERIORFIELD_GC_I
 
     def optimize_SETINTERIORFIELD_GC(self, op):
         value = self.getvalue(op.getarg(0))
diff --git a/rpython/jit/metainterp/optimizeopt/vstring.py b/rpython/jit/metainterp/optimizeopt/vstring.py
--- a/rpython/jit/metainterp/optimizeopt/vstring.py
+++ b/rpython/jit/metainterp/optimizeopt/vstring.py
@@ -541,7 +541,7 @@
                 mode, need_next_offset=False
             )
 
-    def optimize_CALL(self, op):
+    def optimize_CALL_I(self, op):
         # dispatch based on 'oopspecindex' to a method that handles
         # specifically the given oopspec call.  For non-oopspec calls,
         # oopspecindex is just zero.
@@ -565,8 +565,13 @@
                 if self.opt_call_SHRINK_ARRAY(op):
                     return
         self.emit_operation(op)
-
-    optimize_CALL_PURE = optimize_CALL
+    optimize_CALL_R = optimize_CALL_I
+    optimize_CALL_F = optimize_CALL_I
+    optimize_CALL_N = optimize_CALL_I
+    optimize_CALL_PURE_I = optimize_CALL_I
+    optimize_CALL_PURE_R = optimize_CALL_I
+    optimize_CALL_PURE_F = optimize_CALL_I
+    optimize_CALL_PURE_N = optimize_CALL_I
 
     def optimize_GUARD_NO_EXCEPTION(self, op):
         if self.last_emitted_operation is REMOVED:
diff --git a/rpython/jit/metainterp/pyjitpl.py b/rpython/jit/metainterp/pyjitpl.py
--- a/rpython/jit/metainterp/pyjitpl.py
+++ b/rpython/jit/metainterp/pyjitpl.py
@@ -12,7 +12,8 @@
 from rpython.jit.metainterp.jitprof import EmptyProfiler
 from rpython.jit.metainterp.logger import Logger
 from rpython.jit.metainterp.optimizeopt.util import args_dict
-from rpython.jit.metainterp.resoperation import rop
+from rpython.jit.metainterp.resoperation import rop, InputArgInt,\
+     InputArgFloat, InputArgRef
 from rpython.rlib import nonconst, rstack
 from rpython.rlib.debug import debug_start, debug_stop, debug_print
 from rpython.rlib.debug import have_debug_prints, make_sure_not_resized
diff --git a/rpython/jit/metainterp/quasiimmut.py b/rpython/jit/metainterp/quasiimmut.py
--- a/rpython/jit/metainterp/quasiimmut.py
+++ b/rpython/jit/metainterp/quasiimmut.py
@@ -1,7 +1,8 @@
 from rpython.rtyper.lltypesystem import lltype
 from rpython.rtyper import rclass
 from rpython.rtyper.annlowlevel import cast_base_ptr_to_instance
-from rpython.jit.metainterp.history import AbstractDescr
+from rpython.jit.metainterp.history import AbstractDescr, ConstPtr, ConstInt,\
+     ConstFloat
 from rpython.rlib.objectmodel import we_are_translated
 
 
@@ -114,9 +115,18 @@
     def get_current_constant_fieldvalue(self):
         from rpython.jit.metainterp import executor
         from rpython.jit.metainterp.resoperation import rop
-        fieldbox = executor.execute(self.cpu, None, rop.GETFIELD_GC,
-                                    self.fielddescr, self.structbox)
-        return fieldbox.constbox()
+        if self.fielddescr.is_pointer_field():
+            return ConstPtr(executor.do_getfield_gc_r(self.cpu, None, rop.GETFIELD_GC_R,
+                                             self.fielddescr, self.structbox))
+        elif self.fielddescr.is_float_field():
+            return ConstFloat(executor.execute(self.cpu, None,
+                                               rop.GETFIELD_GC_F,
+                                               self.fielddescr, self.structbox))
+            
+        else:
+            return ConstInt(executor.do_getfield_gc_i(self.cpu, None,
+                                                      self.structbox,
+                                                      self.fielddescr))
 
     def is_still_valid_for(self, structconst):
         assert self.structbox is not None
diff --git a/rpython/jit/metainterp/resoperation.py b/rpython/jit/metainterp/resoperation.py
--- a/rpython/jit/metainterp/resoperation.py
+++ b/rpython/jit/metainterp/resoperation.py
@@ -1,5 +1,7 @@
 from rpython.rlib.objectmodel import we_are_translated, specialize
 
+class AbstractValue(object):
+    pass
 
 @specialize.argtype(2)
 def ResOperation(opnum, args, result, descr=None):
@@ -18,11 +20,13 @@
     elif isinstance(result, float):
         op._resfloat = result
     else:
+        from rpython.rtyper.lltypesystem import lltype, llmemory
+        assert lltype.typeOf(result) == llmemory.GCREF
         op._resref = result
     return op
 
 
-class AbstractResOp(object):
+class AbstractResOp(AbstractValue):
     """The central ResOperation class, representing one operation."""
 
     # debug
@@ -254,21 +258,30 @@
 class FloatOp(object):
     _mixin_ = True
 
-    def getfloat(self):
+    def getfloatstorage(self):
         return self._resfloat
 
-    def setfloat(self, floatval):
+    def setfloatstorage(self, floatval):
         self._resfloat = floatval
 
 class RefOp(object):
     _mixin_ = True
 
-    def getref(self):
+    def getref_base(self):
         return self._resref
 
-    def setref(self, refval):
+    def setref_base(self, refval):
         self._resref = refval
 
+class InputArgInt(IntOp, AbstractValue):
+    pass
+
+class InputArgFloat(FloatOp, AbstractValue):
+    pass
+
+class InputArgRef(FloatOp, AbstractValue):
+    pass
+
 # ============
 # arity mixins
 # ============
@@ -415,39 +428,39 @@
 
 """ All the operations are desribed like this:
 
-NAME/no-of-args-or-*[b][d]/type-of-result-or-none
+NAME/no-of-args-or-*[b][d]/types-of-result
 
 if b is present it means the operation produces a boolean
 if d is present it means there is a descr
-type of result can be one of r i f, * for anything, + for i or f or nothing
+type of result can be one or more of r i f n
 """
 
 _oplist = [
     '_FINAL_FIRST',
-    'JUMP/*d/',
-    'FINISH/*d/',
+    'JUMP/*d/n',
+    'FINISH/*d/n',
     '_FINAL_LAST',
 
-    'LABEL/*d/',
+    'LABEL/*d/n',
 
     '_GUARD_FIRST',
     '_GUARD_FOLDABLE_FIRST',
-    'GUARD_TRUE/1d/',
-    'GUARD_FALSE/1d/',
-    'GUARD_VALUE/2d/',
-    'GUARD_CLASS/2d/',
-    'GUARD_NONNULL/1d/',
-    'GUARD_ISNULL/1d/',
-    'GUARD_NONNULL_CLASS/2d/',
+    'GUARD_TRUE/1d/n',
+    'GUARD_FALSE/1d/n',
+    'GUARD_VALUE/2d/n',
+    'GUARD_CLASS/2d/n',
+    'GUARD_NONNULL/1d/n',
+    'GUARD_ISNULL/1d/n',
+    'GUARD_NONNULL_CLASS/2d/n',
     '_GUARD_FOLDABLE_LAST',
-    'GUARD_NO_EXCEPTION/0d/',   # may be called with an exception currently set
+    'GUARD_NO_EXCEPTION/0d/n',   # may be called with an exception currently set
     'GUARD_EXCEPTION/1d/r',     # may be called with an exception currently set
-    'GUARD_NO_OVERFLOW/0d/',
-    'GUARD_OVERFLOW/0d/',
-    'GUARD_NOT_FORCED/0d/',      # may be called with an exception currently set
-    'GUARD_NOT_FORCED_2/0d/',    # same as GUARD_NOT_FORCED, but for finish()
-    'GUARD_NOT_INVALIDATED/0d/',
-    'GUARD_FUTURE_CONDITION/0d/',
+    'GUARD_NO_OVERFLOW/0d/n',
+    'GUARD_OVERFLOW/0d/n',
+    'GUARD_NOT_FORCED/0d/n',      # may be called with an exception currently set
+    'GUARD_NOT_FORCED_2/0d/n',    # same as GUARD_NOT_FORCED, but for finish()
+    'GUARD_NOT_INVALIDATED/0d/n',
+    'GUARD_FUTURE_CONDITION/0d/n',
     # is removable, may be patched by an optimization
     '_GUARD_LAST', # ----- end of guard operations -----
 
@@ -501,7 +514,7 @@
     'INT_INVERT/1/i',
     'INT_FORCE_GE_ZERO/1/i',
     #
-    'SAME_AS/1/*',      # gets a Const or a Box, turns it into another Box
+    'SAME_AS/1/rfi',      # gets a Const or a Box, turns it into another Box
     'CAST_PTR_TO_INT/1/i',
     'CAST_INT_TO_PTR/1/r',
     #
@@ -513,21 +526,21 @@
     'ARRAYLEN_GC/1d/i',
     'STRLEN/1/i',
     'STRGETITEM/2/i',
-    'GETFIELD_GC_PURE/1d/*',
-    'GETFIELD_RAW_PURE/1d/*',
-    'GETARRAYITEM_GC_PURE/2d/*',
-    'GETARRAYITEM_RAW_PURE/2d/*',
+    'GETFIELD_GC_PURE/1d/rfi',
+    'GETFIELD_RAW_PURE/1d/fi',
+    'GETARRAYITEM_GC_PURE/2d/rfi',
+    'GETARRAYITEM_RAW_PURE/2d/fi',
     'UNICODELEN/1/i',
     'UNICODEGETITEM/2/i',
     #
     '_ALWAYS_PURE_LAST',  # ----- end of always_pure operations -----
 
-    'GETARRAYITEM_GC/2d/*',
-    'GETARRAYITEM_RAW/2d/+',
-    'GETINTERIORFIELD_GC/2d/*',
-    'RAW_LOAD/2d/+',
-    'GETFIELD_GC/1d/*',
-    'GETFIELD_RAW/1d/+',
+    'GETARRAYITEM_GC/2d/rfi',
+    'GETARRAYITEM_RAW/2d/fi',
+    'GETINTERIORFIELD_GC/2d/rfi',
+    'RAW_LOAD/2d/fi',
+    'GETFIELD_GC/1d/rfi',
+    'GETFIELD_RAW/1d/fi',
     '_MALLOC_FIRST',
     'NEW/0d/r',           #-> GcStruct, gcptrs inside are zeroed (not the rest)
     'NEW_WITH_VTABLE/1/r',#-> GcStruct with vtable, gcptrs inside are zeroed
@@ -538,45 +551,47 @@
     '_MALLOC_LAST',
     'FORCE_TOKEN/0/i',
     'VIRTUAL_REF/2/r',    # removed before it's passed to the backend
-    'MARK_OPAQUE_PTR/1b/',
+    'MARK_OPAQUE_PTR/1b/n',
     # this one has no *visible* side effect, since the virtualizable
     # must be forced, however we need to execute it anyway
     '_NOSIDEEFFECT_LAST', # ----- end of no_side_effect operations -----
 
-    'INCREMENT_DEBUG_COUNTER/1/',
-    'SETARRAYITEM_GC/3d/',
-    'SETARRAYITEM_RAW/3d/',
-    'SETINTERIORFIELD_GC/3d/',
-    'SETINTERIORFIELD_RAW/3d/',    # right now, only used by tests
-    'RAW_STORE/3d/',
-    'SETFIELD_GC/2d/',
-    'ZERO_PTR_FIELD/2/', # only emitted by the rewrite, clears a pointer field
+    'INCREMENT_DEBUG_COUNTER/1/n',
+    'SETARRAYITEM_GC/3d/n',
+    'SETARRAYITEM_RAW/3d/n',
+    'SETINTERIORFIELD_GC/3d/n',
+    'SETINTERIORFIELD_RAW/3d/n',    # right now, only used by tests
+    'RAW_STORE/3d/n',
+    'SETFIELD_GC/2d/n',
+    'ZERO_PTR_FIELD/2/n', # only emitted by the rewrite, clears a pointer field
                         # at a given constant offset, no descr
-    'ZERO_ARRAY/3d/',   # only emitted by the rewrite, clears (part of) an array
+    'ZERO_ARRAY/3d/n',  # only emitted by the rewrite, clears (part of) an array
                         # [arraygcptr, firstindex, length], descr=ArrayDescr
-    'SETFIELD_RAW/2d/',
-    'STRSETITEM/3/',
-    'UNICODESETITEM/3/',
-    'COND_CALL_GC_WB/1d/',       # [objptr] (for the write barrier)
-    'COND_CALL_GC_WB_ARRAY/2d/', # [objptr, arrayindex] (write barr. for array)
-    'DEBUG_MERGE_POINT/*/',      # debugging only
-    'JIT_DEBUG/*/',              # debugging only
-    'VIRTUAL_REF_FINISH/2/',   # removed before it's passed to the backend
-    'COPYSTRCONTENT/5/',       # src, dst, srcstart, dststart, length
-    'COPYUNICODECONTENT/5/',
-    'QUASIIMMUT_FIELD/1d/',    # [objptr], descr=SlowMutateDescr
-    'RECORD_KNOWN_CLASS/2/',   # [objptr, clsptr]
-    'KEEPALIVE/1/',
+    'SETFIELD_RAW/2d/n',
+    'STRSETITEM/3/n',
+    'UNICODESETITEM/3/n',
+    'COND_CALL_GC_WB/1d/n',       # [objptr] (for the write barrier)
+    'COND_CALL_GC_WB_ARRAY/2d/n', # [objptr, arrayindex] (write barr. for array)
+    'DEBUG_MERGE_POINT/*/n',      # debugging only
+    'JIT_DEBUG/*/n',              # debugging only
+    'VIRTUAL_REF_FINISH/2/n',   # removed before it's passed to the backend
+    'COPYSTRCONTENT/5/n',       # src, dst, srcstart, dststart, length
+    'COPYUNICODECONTENT/5/n',
+    'QUASIIMMUT_FIELD/1d/n',    # [objptr], descr=SlowMutateDescr
+    'RECORD_KNOWN_CLASS/2/n',   # [objptr, clsptr]
+    'KEEPALIVE/1/n',
 
     '_CANRAISE_FIRST', # ----- start of can_raise operations -----
     '_CALL_FIRST',
-    'CALL/*d/*',
-    'COND_CALL/*d/*', # a conditional call, with first argument as a condition
-    'CALL_ASSEMBLER/*d/*',  # call already compiled assembler
-    'CALL_MAY_FORCE/*d/*',
-    'CALL_LOOPINVARIANT/*d/*',
-    'CALL_RELEASE_GIL/*d/*',  # release the GIL and "close the stack" for asmgcc
-    'CALL_PURE/*d/*',             # removed before it's passed to the backend
+    'CALL/*d/rfin',
+    'COND_CALL/*d/n',
+    # a conditional call, with first argument as a condition
+    'CALL_ASSEMBLER/*d/rfin',  # call already compiled assembler
+    'CALL_MAY_FORCE/*d/rfin',
+    'CALL_LOOPINVARIANT/*d/rfin',
+    'CALL_RELEASE_GIL/*d/rfin',
+    # release the GIL and "close the stack" for asmgcc
+    'CALL_PURE/*d/rfin',             # removed before it's passed to the backend
     'CALL_MALLOC_GC/*d/r',      # like CALL, but NULL => propagate MemoryError
     'CALL_MALLOC_NURSERY/1/r',  # nursery malloc, const number of bytes, zeroed
     'CALL_MALLOC_NURSERY_VARSIZE/3d/r',
@@ -620,15 +635,12 @@
                 arity = int(arity)
         else:
             arity, withdescr, boolresult, result = -1, True, False, None       # default
-        if result == '*':
-            result = 'rfiN'
-        elif result == '+':
-            result = 'fiN'
-        elif result == '':
-            result = 'N'
         if not name.startswith('_'):
             for r in result:
-                cls_name = name + '_' + r
+                if len(result) == 1:
+                    cls_name = name
+                else:
+                    cls_name = name + '_' + r.upper()
                 setattr(rop, cls_name, i)
                 opname[i] = cls_name
                 cls = create_class_for_op(cls_name, i, arity, withdescr, r)
@@ -639,6 +651,14 @@
                 if debug_print:
                     print '%30s = %d' % (cls_name, i)
                 i += 1
+        else:
+            setattr(rop, name, i)
+            opclasses.append(None)
+            oparity.append(-1)
+            opwithdescr.append(False)
+            if debug_print:
+                print '%30s = %d' % (name, i)
+            i += 1
 
 def get_base_class(mixins, base):
     try:
@@ -676,7 +696,7 @@
     elif result_type == 'r':
         mixins.append(RefOp)
     else:
-        assert result_type == 'N'
+        assert result_type == 'n'
 
     cls_name = '%s_OP' % name
     bases = (get_base_class(tuple(mixins), baseclass),)
@@ -687,51 +707,51 @@
 del _oplist
 
 opboolinvers = {
-    rop.INT_EQ_i: rop.INT_NE_i,
-    rop.INT_NE_i: rop.INT_EQ_i,
-    rop.INT_LT_i: rop.INT_GE_i,
-    rop.INT_GE_i: rop.INT_LT_i,
-    rop.INT_GT_i: rop.INT_LE_i,
-    rop.INT_LE_i: rop.INT_GT_i,
+    rop.INT_EQ: rop.INT_NE,
+    rop.INT_NE: rop.INT_EQ,
+    rop.INT_LT: rop.INT_GE,
+    rop.INT_GE: rop.INT_LT,
+    rop.INT_GT: rop.INT_LE,
+    rop.INT_LE: rop.INT_GT,
 
-    rop.UINT_LT_i: rop.UINT_GE_i,
-    rop.UINT_GE_i: rop.UINT_LT_i,
-    rop.UINT_GT_i: rop.UINT_LE_i,
-    rop.UINT_LE_i: rop.UINT_GT_i,
+    rop.UINT_LT: rop.UINT_GE,
+    rop.UINT_GE: rop.UINT_LT,
+    rop.UINT_GT: rop.UINT_LE,
+    rop.UINT_LE: rop.UINT_GT,
 
-    rop.FLOAT_EQ_i: rop.FLOAT_NE_i,
-    rop.FLOAT_NE_i: rop.FLOAT_EQ_i,
-    rop.FLOAT_LT_i: rop.FLOAT_GE_i,
-    rop.FLOAT_GE_i: rop.FLOAT_LT_i,
-    rop.FLOAT_GT_i: rop.FLOAT_LE_i,
-    rop.FLOAT_LE_i: rop.FLOAT_GT_i,
+    rop.FLOAT_EQ: rop.FLOAT_NE,
+    rop.FLOAT_NE: rop.FLOAT_EQ,
+    rop.FLOAT_LT: rop.FLOAT_GE,
+    rop.FLOAT_GE: rop.FLOAT_LT,
+    rop.FLOAT_GT: rop.FLOAT_LE,
+    rop.FLOAT_LE: rop.FLOAT_GT,
 
-    rop.PTR_EQ_i: rop.PTR_NE_i,
-    rop.PTR_NE_i: rop.PTR_EQ_i,
+    rop.PTR_EQ: rop.PTR_NE,
+    rop.PTR_NE: rop.PTR_EQ,
 }
 
 opboolreflex = {
-    rop.INT_EQ_i: rop.INT_EQ_i,
-    rop.INT_NE_i: rop.INT_NE_i,
-    rop.INT_LT_i: rop.INT_GT_i,
-    rop.INT_GE_i: rop.INT_LE_i,
-    rop.INT_GT_i: rop.INT_LT_i,
-    rop.INT_LE_i: rop.INT_GE_i,
+    rop.INT_EQ: rop.INT_EQ,
+    rop.INT_NE: rop.INT_NE,
+    rop.INT_LT: rop.INT_GT,
+    rop.INT_GE: rop.INT_LE,
+    rop.INT_GT: rop.INT_LT,
+    rop.INT_LE: rop.INT_GE,
 
-    rop.UINT_LT_i: rop.UINT_GT_i,
-    rop.UINT_GE_i: rop.UINT_LE_i,
-    rop.UINT_GT_i: rop.UINT_LT_i,
-    rop.UINT_LE_i: rop.UINT_GE_i,
+    rop.UINT_LT: rop.UINT_GT,
+    rop.UINT_GE: rop.UINT_LE,
+    rop.UINT_GT: rop.UINT_LT,
+    rop.UINT_LE: rop.UINT_GE,
 
-    rop.FLOAT_EQ_i: rop.FLOAT_EQ_i,
-    rop.FLOAT_NE_i: rop.FLOAT_NE_i,
-    rop.FLOAT_LT_i: rop.FLOAT_GT_i,
-    rop.FLOAT_GE_i: rop.FLOAT_LE_i,
-    rop.FLOAT_GT_i: rop.FLOAT_LT_i,
-    rop.FLOAT_LE_i: rop.FLOAT_GE_i,
+    rop.FLOAT_EQ: rop.FLOAT_EQ,
+    rop.FLOAT_NE: rop.FLOAT_NE,
+    rop.FLOAT_LT: rop.FLOAT_GT,
+    rop.FLOAT_GE: rop.FLOAT_LE,
+    rop.FLOAT_GT: rop.FLOAT_LT,
+    rop.FLOAT_LE: rop.FLOAT_GE,
 
-    rop.PTR_EQ_i: rop.PTR_EQ_i,
-    rop.PTR_NE_i: rop.PTR_NE_i,
+    rop.PTR_EQ: rop.PTR_EQ,
+    rop.PTR_NE: rop.PTR_NE,
 }
 
 

From noreply at buildbot.pypy.org  Thu Nov 13 14:23:39 2014
From: noreply at buildbot.pypy.org (fijal)
Date: Thu, 13 Nov 2014 14:23:39 +0100 (CET)
Subject: [pypy-commit] pypy optresult: hack oparser to pass it's own tests.
 Don't care about the obscure features for now
Message-ID: <20141113132339.710C51C0548@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: optresult
Changeset: r74501:82186084fb67
Date: 2014-11-12 14:24 +0200
http://bitbucket.org/pypy/pypy/changeset/82186084fb67/

Log:	hack oparser to pass it's own tests. Don't care about the obscure
	features for now

diff --git a/rpython/jit/metainterp/resoperation.py b/rpython/jit/metainterp/resoperation.py
--- a/rpython/jit/metainterp/resoperation.py
+++ b/rpython/jit/metainterp/resoperation.py
@@ -19,6 +19,8 @@
         op._resint = result
     elif isinstance(result, float):
         op._resfloat = result
+    elif result is None:
+        pass
     else:
         from rpython.rtyper.lltypesystem import lltype, llmemory
         assert lltype.typeOf(result) == llmemory.GCREF
@@ -274,13 +276,16 @@
         self._resref = refval
 
 class InputArgInt(IntOp, AbstractValue):
-    pass
+    def __init__(self, intval):
+        self.setint(intval)
 
 class InputArgFloat(FloatOp, AbstractValue):
-    pass
+    def __init__(self, f):
+        self.setfloatstorage(f)
 
-class InputArgRef(FloatOp, AbstractValue):
-    pass
+class InputArgRef(RefOp, AbstractValue):
+    def __init__(self, r):
+        self.setref_base(r)
 
 # ============
 # arity mixins
@@ -619,7 +624,7 @@
 opname = {}      # mapping numbers to the original names, for debugging
 oparity = []     # mapping numbers to the arity of the operation or -1
 opwithdescr = [] # mapping numbers to a flag "takes a descr"
-
+optypes = []     # mapping numbers to type of return
 
 def setup(debug_print=False):
     i = 0
@@ -648,6 +653,7 @@
                 opclasses.append(cls)
                 oparity.append(arity)
                 opwithdescr.append(withdescr)
+                optypes.append(r)
                 if debug_print:
                     print '%30s = %d' % (cls_name, i)
                 i += 1
@@ -656,6 +662,7 @@
             opclasses.append(None)
             oparity.append(-1)
             opwithdescr.append(False)
+            optypes.append(' ')
             if debug_print:
                 print '%30s = %d' % (name, i)
             i += 1
diff --git a/rpython/jit/tool/oparser.py b/rpython/jit/tool/oparser.py
--- a/rpython/jit/tool/oparser.py
+++ b/rpython/jit/tool/oparser.py
@@ -6,8 +6,8 @@
 from rpython.jit.tool.oparser_model import get_model
 
 from rpython.jit.metainterp.resoperation import rop, ResOperation, \
-                                            ResOpWithDescr, N_aryOp, \
-                                            UnaryOp, PlainResOp
+     InputArgInt, InputArgRef, InputArgFloat, ResOpWithDescr, N_aryOp, \
+     UnaryOp, PlainResOp, optypes
 
 class ParseError(Exception):
     pass
@@ -103,6 +103,7 @@
                 raise
 
     def box_for_var(self, elem):
+        xxx
         try:
             return self._cache[self.type_system, elem]
         except KeyError:
@@ -135,9 +136,21 @@
         vars = []
         for elem in elements:
             elem = elem.strip()
-            vars.append(self.newvar(elem))
+            vars.append(self.newinputarg(elem))
         return vars
 
+    def newinputarg(self, elem):
+        if elem.startswith('i'):
+            v = InputArgInt(0)
+        elif elem.startswith('f'):
+            v = InputArgFloat(0.0)
+        else:
+            from rpython.rtyper.lltypesystem import lltype, llmemory
+            assert elem.startswith('p')
+            v = InputArgRef(lltype.nullptr(llmemory.GCREF.TO))
+        self.vars[elem] = v
+        return v
+
     def newvar(self, elem):
         box = self.box_for_var(elem)
         self.vars[elem] = box
@@ -250,18 +263,29 @@
 
         return opnum, args, descr, fail_args
 
-    def create_op(self, opnum, args, result, descr):
+    def create_op(self, opnum, args, descr):
         if opnum == ESCAPE_OP.OPNUM:
-            op = ESCAPE_OP(result)
+            op = ESCAPE_OP()
             op.initarglist(args)
             assert descr is None
             return op
         if opnum == FORCE_SPILL.OPNUM:
-            op = FORCE_SPILL(result)
+            op = FORCE_SPILL()
             op.initarglist(args)
             assert descr is None
             return op
         else:
+            tp = optypes[opnum]
+            if tp == 'i':
+                result = 0
+            elif tp == 'r':
+                from rpython.rtyper.lltypesystem import lltype, llmemory
+                result = lltype.nullptr(llmemory.GCREF.TO)
+            elif tp == 'f':
+                result = 0.0
+            else:
+                assert tp == 'n'
+                result = None
             return ResOperation(opnum, args, result, descr)
 
     def parse_result_op(self, line):
@@ -271,16 +295,15 @@
         opnum, args, descr, fail_args = self.parse_op(op)
         if res in self.vars:
             raise ParseError("Double assign to var %s in line: %s" % (res, line))
-        rvar = self.box_for_var(res)
-        self.vars[res] = rvar
-        res = self.create_op(opnum, args, rvar, descr)
+        resop = self.create_op(opnum, args, descr)
         if fail_args is not None:
-            res.setfailargs(fail_args)
-        return res
+            resop.setfailargs(fail_args)
+        self.vars[res] = resop
+        return resop
 
     def parse_op_no_result(self, line):
         opnum, args, descr, fail_args = self.parse_op(line)
-        res = self.create_op(opnum, args, None, descr)
+        res = self.create_op(opnum, args, descr)
         if fail_args is not None:
             res.setfailargs(fail_args)
         return res
diff --git a/rpython/jit/tool/test/test_oparser.py b/rpython/jit/tool/test/test_oparser.py
--- a/rpython/jit/tool/test/test_oparser.py
+++ b/rpython/jit/tool/test/test_oparser.py
@@ -48,7 +48,7 @@
 
         x = """
         [p0]
-        i1 = getfield_gc(p0, descr=stuff)
+        i1 = getfield_gc_i(p0, descr=stuff)
         """
         stuff = Xyz()
         loop = self.parse(x, None, locals())
@@ -75,39 +75,10 @@
         loop = self.parse(x, None, locals())
         assert loop.operations[0].getdescr() is stuff
 
-    def test_boxname(self):
-        x = """
-        [i42]
-        i50 = int_add(i42, 1)
-        """
-        loop = self.parse(x, None, {})
-        assert str(loop.inputargs[0]) == 'i42'
-        assert str(loop.operations[0].result) == 'i50'
-
-    def test_getboxes(self):
-        x = """
-        [i0]
-        i1 = int_add(i0, 10)
-        """
-        loop = self.parse(x, None, {})
-        boxes = loop.getboxes()
-        assert boxes.i0 is loop.inputargs[0]
-        assert boxes.i1 is loop.operations[0].result
-
-    def test_setvalues(self):
-        x = """
-        [i0]
-        i1 = int_add(i0, 10)
-        """
-        loop = self.parse(x, None, {})
-        loop.setvalues(i0=32, i1=42)
-        assert loop.inputargs[0].value == 32
-        assert loop.operations[0].result.value == 42
-
     def test_getvar_const_ptr(self):
         x = '''
         []
-        call(ConstPtr(func_ptr))
+        call_n(ConstPtr(func_ptr))
         '''
         TP = lltype.GcArray(lltype.Signed)
         NULL = lltype.cast_opaque_ptr(llmemory.GCREF, lltype.nullptr(TP))
@@ -141,7 +112,7 @@
         box = loop.operations[0].getarg(0)
         # we cannot use isinstance, because in case of mock the class will be
         # constructed on the fly
-        assert box.__class__.__name__ == 'BoxFloat'
+        assert box.__class__.__name__ == 'InputArgFloat'
 
     def test_debug_merge_point(self):
         x = '''
@@ -203,14 +174,6 @@
         loop = self.parse(x, nonstrict=True)
         assert loop.operations[0].getfailargs() == []
 
-    def test_no_inputargs(self):
-        x = '''
-        i2 = int_add(i0, i1)
-        '''
-        loop = self.parse(x, nonstrict=True)
-        assert loop.inputargs == []
-        assert loop.operations[0].getopname() == 'int_add'
-
     def test_offsets(self):
         x = """
         [i0, i1]
@@ -238,14 +201,6 @@
 
     OpParser = OpParser
 
-    def test_boxkind(self):
-        x = """
-        [sum0]
-        """
-        loop = self.parse(x, None, {}, boxkinds={'sum': BoxInt})
-        b = loop.getboxes()
-        assert isinstance(b.sum0, BoxInt)
-
     def test_label(self):
         x = """
         [i0]

From noreply at buildbot.pypy.org  Thu Nov 13 14:23:40 2014
From: noreply at buildbot.pypy.org (fijal)
Date: Thu, 13 Nov 2014 14:23:40 +0100 (CET)
Subject: [pypy-commit] pypy default: try a slightly more efficient logic to
 avoid some dict lookups for every operation (likely doomed to be lost in
 the noise though)
Message-ID: <20141113132340.AF1551C0548@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: 
Changeset: r74502:bd3a657b79af
Date: 2014-11-13 15:22 +0200
http://bitbucket.org/pypy/pypy/changeset/bd3a657b79af/

Log:	try a slightly more efficient logic to avoid some dict lookups for
	every operation (likely doomed to be lost in the noise though)

diff --git a/rpython/jit/metainterp/optimizeopt/rewrite.py b/rpython/jit/metainterp/optimizeopt/rewrite.py
--- a/rpython/jit/metainterp/optimizeopt/rewrite.py
+++ b/rpython/jit/metainterp/optimizeopt/rewrite.py
@@ -8,8 +8,7 @@
 from rpython.jit.metainterp.optimizeopt.optimizer import (Optimization, REMOVED,
     CONST_0, CONST_1)
 from rpython.jit.metainterp.optimizeopt.util import _findall, make_dispatcher_method
-from rpython.jit.metainterp.resoperation import (opboolinvers, opboolreflex, rop,
-    ResOperation)
+from rpython.jit.metainterp.resoperation import rop, ResOperation, opclasses
 from rpython.rlib.rarithmetic import highest_bit
 import math
 
@@ -26,9 +25,10 @@
             sb.add_potential(op)
 
     def propagate_forward(self, op):
-        args = self.optimizer.make_args_key(op)
-        if self.find_rewritable_bool(op, args):
-            return
+        if op.boolinverse != -1 or op.boolreflex != -1:
+            args = self.optimizer.make_args_key(op)
+            if self.find_rewritable_bool(op, args):
+                return
 
         dispatch_opt(self, op)
 
@@ -48,21 +48,15 @@
 
 
     def find_rewritable_bool(self, op, args):
-        try:
-            oldopnum = opboolinvers[op.getopnum()]
-        except KeyError:
-            pass
-        else:
+        oldopnum = op.boolinverse
+        if oldopnum != -1:
             targs = self.optimizer.make_args_key(ResOperation(oldopnum, [args[0], args[1]],
                                                               None))
             if self.try_boolinvers(op, targs):
                 return True
 
-        try:
-            oldopnum = opboolreflex[op.getopnum()] # FIXME: add INT_ADD, INT_MUL
-        except KeyError:
-            pass
-        else:
+        oldopnum = op.boolreflex # FIXME: add INT_ADD, INT_MUL
+        if oldopnum != -1:
             targs = self.optimizer.make_args_key(ResOperation(oldopnum, [args[1], args[0]],
                                                               None))
             oldop = self.get_pure_result(targs)
@@ -70,13 +64,12 @@
                 self.make_equal_to(op.result, self.getvalue(oldop.result))
                 return True
 
-        try:
-            oldopnum = opboolinvers[opboolreflex[op.getopnum()]]
-        except KeyError:
-            pass
-        else:
-            targs = self.optimizer.make_args_key(ResOperation(oldopnum, [args[1], args[0]],
-                                                              None))
+        if op.boolreflex == -1:
+            return False
+        oldopnum = opclasses[op.boolreflex].boolinverse
+        if oldopnum != -1:
+            targs = self.optimizer.make_args_key(
+                ResOperation(oldopnum, [args[1], args[0]], None))
             if self.try_boolinvers(op, targs):
                 return True
 
diff --git a/rpython/jit/metainterp/resoperation.py b/rpython/jit/metainterp/resoperation.py
--- a/rpython/jit/metainterp/resoperation.py
+++ b/rpython/jit/metainterp/resoperation.py
@@ -23,6 +23,8 @@
     pc = 0
     opnum = 0
     _cls_has_bool_result = False
+    boolreflex = -1
+    boolinverse = -1
 
     _attrs_ = ('result',)
 
@@ -623,7 +625,7 @@
 setup(__name__ == '__main__')   # print out the table when run directly
 del _oplist
 
-opboolinvers = {
+_opboolinverse = {
     rop.INT_EQ: rop.INT_NE,
     rop.INT_NE: rop.INT_EQ,
     rop.INT_LT: rop.INT_GE,
@@ -647,7 +649,7 @@
     rop.PTR_NE: rop.PTR_EQ,
 }
 
-opboolreflex = {
+_opboolreflex = {
     rop.INT_EQ: rop.INT_EQ,
     rop.INT_NE: rop.INT_NE,
     rop.INT_LT: rop.INT_GT,
@@ -671,6 +673,19 @@
     rop.PTR_NE: rop.PTR_NE,
 }
 
+def setup2():
+    for cls in opclasses:
+        if cls is None:
+            continue
+        opnum = cls.opnum
+        if opnum in _opboolreflex:
+            cls.boolreflex = _opboolreflex[opnum]
+        if opnum in _opboolinverse:
+            cls.boolinverse = _opboolinverse[opnum]
+
+setup2()
+del _opboolinverse
+del _opboolreflex
 
 def get_deep_immutable_oplist(operations):
     """

From noreply at buildbot.pypy.org  Thu Nov 13 14:23:41 2014
From: noreply at buildbot.pypy.org (fijal)
Date: Thu, 13 Nov 2014 14:23:41 +0100 (CET)
Subject: [pypy-commit] pypy optresult: small progres
Message-ID: <20141113132341.DC1091C0548@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: optresult
Changeset: r74503:4fe0a963f8a3
Date: 2014-11-13 15:22 +0200
http://bitbucket.org/pypy/pypy/changeset/4fe0a963f8a3/

Log:	small progres

diff --git a/rpython/jit/metainterp/history.py b/rpython/jit/metainterp/history.py
--- a/rpython/jit/metainterp/history.py
+++ b/rpython/jit/metainterp/history.py
@@ -6,9 +6,8 @@
 
 from rpython.conftest import option
 
-from rpython.jit.metainterp.resoperation import ResOperation, rop
+from rpython.jit.metainterp.resoperation import ResOperation, rop, AbstractValue
 from rpython.jit.codewriter import heaptracker, longlong
-from rpython.rlib.objectmodel import compute_identity_hash
 import weakref
 
 # ____________________________________________________________
@@ -72,7 +71,7 @@
                         compute_unique_id(box))
 
 
-class AbstractValue(object):
+class XxxAbstractValue(object):
     __slots__ = ()
 
     def getint(self):
@@ -95,9 +94,6 @@
         raise NotImplementedError
     getref._annspecialcase_ = 'specialize:arg(1)'
 
-    def _get_hash_(self):
-        return compute_identity_hash(self)
-
     def clonebox(self):
         raise NotImplementedError
 
diff --git a/rpython/jit/metainterp/optimizeopt/earlyforce.py b/rpython/jit/metainterp/optimizeopt/earlyforce.py
--- a/rpython/jit/metainterp/optimizeopt/earlyforce.py
+++ b/rpython/jit/metainterp/optimizeopt/earlyforce.py
@@ -4,7 +4,7 @@
 
 
 def is_raw_free(op, opnum):
-    if opnum != rop.CALL:
+    if not op.is_call():
         return False
     einfo = op.getdescr().get_extra_info()
     return einfo.oopspecindex == EffectInfo.OS_RAW_FREE
@@ -18,7 +18,9 @@
             opnum != rop.SETARRAYITEM_GC and
             opnum != rop.SETARRAYITEM_RAW and
             opnum != rop.QUASIIMMUT_FIELD and
-            opnum != rop.SAME_AS and
+            opnum != rop.SAME_AS_I and
+            opnum != rop.SAME_AS_R and
+            opnum != rop.SAME_AS_F and
             opnum != rop.MARK_OPAQUE_PTR and
             not is_raw_free(op, opnum)):
 
diff --git a/rpython/jit/metainterp/optimizeopt/heap.py b/rpython/jit/metainterp/optimizeopt/heap.py
--- a/rpython/jit/metainterp/optimizeopt/heap.py
+++ b/rpython/jit/metainterp/optimizeopt/heap.py
@@ -248,7 +248,7 @@
     def emit_operation(self, op):
         self.emitting_operation(op)
         self.emit_postponed_op()
-        if (op.is_comparison() or op.getopnum() == rop.CALL_MAY_FORCE
+        if (op.is_comparison() or op.is_call_may_force()
             or op.is_ovf()):
             self.postponed_op = op
         else:
@@ -276,13 +276,8 @@
             opnum == rop.COPYSTRCONTENT or       # no effect on GC struct/array
             opnum == rop.COPYUNICODECONTENT):    # no effect on GC struct/array
             return
-        if (opnum == rop.CALL or
-            opnum == rop.CALL_PURE or
-            opnum == rop.COND_CALL or
-            opnum == rop.CALL_MAY_FORCE or
-            opnum == rop.CALL_RELEASE_GIL or
-            opnum == rop.CALL_ASSEMBLER):
-            if opnum == rop.CALL_ASSEMBLER:
+        if op.is_call():
+            if op.is_call_assembler():
                 self._seen_guard_not_invalidated = False
             else:
                 effectinfo = op.getdescr().get_extra_info()
diff --git a/rpython/jit/metainterp/optimizeopt/optimizer.py b/rpython/jit/metainterp/optimizeopt/optimizer.py
--- a/rpython/jit/metainterp/optimizeopt/optimizer.py
+++ b/rpython/jit/metainterp/optimizeopt/optimizer.py
@@ -551,7 +551,7 @@
 
     @specialize.argtype(0)
     def _emit_operation(self, op):
-        assert op.getopnum() != rop.CALL_PURE
+        assert not op.is_call_pure()
         for i in range(op.numargs()):
             arg = op.getarg(i)
             try:
@@ -598,7 +598,8 @@
         modifier = resume.ResumeDataVirtualAdder(descr, self.resumedata_memo)
         try:
             newboxes = modifier.finish(self, pendingfields)
-            if len(newboxes) > self.metainterp_sd.options.failargs_limit:
+            if (newboxes is not None and
+                len(newboxes) > self.metainterp_sd.options.failargs_limit):
                 raise resume.TagOverflow
         except resume.TagOverflow:
             raise compile.giveup()
@@ -625,11 +626,11 @@
                 descr.make_a_counter_per_value(op)
         return op
 
-    def make_args_key(self, op):
-        n = op.numargs()
+    def make_args_key(self, opnum, arglist, descr):
+        n = len(arglist)
         args = [None] * (n + 2)
         for i in range(n):
-            arg = op.getarg(i)
+            arg = arglist[i]
             try:
                 value = self.values[arg]
             except KeyError:
@@ -637,8 +638,8 @@
             else:
                 arg = value.get_key_box()
             args[i] = arg
-        args[n] = ConstInt(op.getopnum())
-        args[n + 1] = op.getdescr()
+        args[n] = ConstInt(opnum)
+        args[n + 1] = descr
         return args
 
     def optimize_default(self, op):
diff --git a/rpython/jit/metainterp/optimizeopt/pure.py b/rpython/jit/metainterp/optimizeopt/pure.py
--- a/rpython/jit/metainterp/optimizeopt/pure.py
+++ b/rpython/jit/metainterp/optimizeopt/pure.py
@@ -105,7 +105,7 @@
         op = ResOperation(opnum, args, result)
         key = self.optimizer.make_args_key(op)
         if key not in self.pure_operations:
-            self.pure_operations[key] = op
+            self.pure_operations[key] = result
 
     def has_pure_result(self, opnum, args, descr):
         op = ResOperation(opnum, args, None, descr)
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_multilabel.py b/rpython/jit/metainterp/optimizeopt/test/test_multilabel.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_multilabel.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_multilabel.py
@@ -1,6 +1,6 @@
 from __future__ import with_statement
 from rpython.jit.metainterp.optimizeopt.test.test_util import (
-    LLtypeMixin, BaseTest, Storage, _sortboxes,
+    LLtypeMixin, BaseTest, Storage,
     FakeMetaInterpStaticData)
 from rpython.jit.metainterp.history import TreeLoop, JitCellToken, TargetToken
 from rpython.jit.metainterp.resoperation import rop, opname, ResOperation
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_util.py b/rpython/jit/metainterp/optimizeopt/test/test_util.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_util.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_util.py
@@ -338,21 +338,11 @@
 
 class Storage(compile.ResumeGuardDescr):
     "for tests."
-    def __init__(self, metainterp_sd=None, original_greenkey=None):
-        self.metainterp_sd = metainterp_sd
-        self.original_greenkey = original_greenkey
-    def store_final_boxes(self, op, boxes, metainterp_sd):
-        op.setfailargs(boxes)
-    def __eq__(self, other):
-        return type(self) is type(other)      # xxx obscure
     def clone_if_mutable(self):
-        res = Storage(self.metainterp_sd, self.original_greenkey)
-        self.copy_all_attributes_into(res)
-        return res
+        return Storage()
 
-def _sortboxes(boxes):
-    _kind2count = {history.INT: 1, history.REF: 2, history.FLOAT: 3}
-    return sorted(boxes, key=lambda box: _kind2count[box.type])
+    def store_final_boxes(self, *args):
+        pass
 
 class BaseTest(object):
 
@@ -376,8 +366,6 @@
         if fail_args is None:
             return None
         descr = Storage()
-        descr.rd_frame_info_list = resume.FrameInfo(None, "code", 11)
-        descr.rd_snapshot = resume.Snapshot(None, _sortboxes(fail_args))
         return descr
 
     def assert_equal(self, optimized, expected, text_right=None):
diff --git a/rpython/jit/metainterp/resoperation.py b/rpython/jit/metainterp/resoperation.py
--- a/rpython/jit/metainterp/resoperation.py
+++ b/rpython/jit/metainterp/resoperation.py
@@ -1,7 +1,9 @@
 from rpython.rlib.objectmodel import we_are_translated, specialize
+from rpython.rlib.objectmodel import compute_identity_hash
 
 class AbstractValue(object):
-    pass
+    def _get_hash_(self):
+        return compute_identity_hash(self)
 
 @specialize.argtype(2)
 def ResOperation(opnum, args, result, descr=None):
@@ -36,6 +38,7 @@
     pc = 0
     opnum = 0
     _cls_has_bool_result = False
+    type = 'v'
 
     _attrs_ = ()
 
@@ -177,6 +180,27 @@
     def is_call(self):
         return rop._CALL_FIRST <= self.getopnum() <= rop._CALL_LAST
 
+    def is_call_assembler(self):
+        opnum = self.opnum
+        return (opnum == rop.CALL_ASSEMBLER_I or
+                opnum == rop.CALL_ASSEMBLER_R or
+                opnum == rop.CALL_ASSEMBLER_N or
+                opnum == rop.CALL_ASSEMBLER_F)
+
+    def is_call_may_force(self):
+        opnum = self.opnum
+        return (opnum == rop.CALL_MAY_FORCE_I or
+                opnum == rop.CALL_MAY_FORCE_R or
+                opnum == rop.CALL_MAY_FORCE_N or
+                opnum == rop.CALL_MAY_FORCE_F)
+
+    def is_call_pure(self):
+        opnum = self.opnum
+        return (opnum == rop.CALL_PURE_I or
+                opnum == rop.CALL_PURE_R or
+                opnum == rop.CALL_PURE_N or
+                opnum == rop.CALL_PURE_F)        
+
     def is_ovf(self):
         return rop._OVF_FIRST <= self.getopnum() <= rop._OVF_LAST
 
@@ -251,6 +275,8 @@
 class IntOp(object):
     _mixin_ = True
 
+    type = 'i'
+
     def getint(self):
         return self._resint
 
@@ -260,6 +286,8 @@
 class FloatOp(object):
     _mixin_ = True
 
+    type = 'f'
+
     def getfloatstorage(self):
         return self._resfloat
 
@@ -269,6 +297,8 @@
 class RefOp(object):
     _mixin_ = True
 
+    type = 'r'
+
     def getref_base(self):
         return self._resref
 
diff --git a/rpython/jit/metainterp/resume.py b/rpython/jit/metainterp/resume.py
--- a/rpython/jit/metainterp/resume.py
+++ b/rpython/jit/metainterp/resume.py
@@ -358,7 +358,9 @@
         # make sure that nobody attached resume data to this guard yet
         assert not storage.rd_numb
         snapshot = storage.rd_snapshot
-        assert snapshot is not None # is that true?
+        if snapshot is None:
+            assert not we_are_translated()
+            return # for tests in optimizeopt
         numb, liveboxes_from_env, v = self.memo.number(optimizer, snapshot)
         self.liveboxes_from_env = liveboxes_from_env
         self.liveboxes = {}
diff --git a/rpython/jit/tool/oparser.py b/rpython/jit/tool/oparser.py
--- a/rpython/jit/tool/oparser.py
+++ b/rpython/jit/tool/oparser.py
@@ -23,7 +23,7 @@
         return 'escape'
 
     def clone(self):
-        op = ESCAPE_OP(self.result)
+        op = ESCAPE_OP(0)
         op.initarglist(self.getarglist()[:])
         return op
 
@@ -38,7 +38,7 @@
         return 'force_spill'
 
     def clone(self):
-        op = FORCE_SPILL(self.result)
+        op = FORCE_SPILL(0)
         op.initarglist(self.getarglist()[:])
         return op
 
@@ -265,12 +265,12 @@
 
     def create_op(self, opnum, args, descr):
         if opnum == ESCAPE_OP.OPNUM:
-            op = ESCAPE_OP()
+            op = ESCAPE_OP(0)
             op.initarglist(args)
             assert descr is None
             return op
         if opnum == FORCE_SPILL.OPNUM:
-            op = FORCE_SPILL()
+            op = FORCE_SPILL(0)
             op.initarglist(args)
             assert descr is None
             return op

From noreply at buildbot.pypy.org  Thu Nov 13 14:23:44 2014
From: noreply at buildbot.pypy.org (fijal)
Date: Thu, 13 Nov 2014 14:23:44 +0100 (CET)
Subject: [pypy-commit] pypy optresult: merge default
Message-ID: <20141113132344.313BD1C0548@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: optresult
Changeset: r74504:25a2e8817653
Date: 2014-11-13 15:22 +0200
http://bitbucket.org/pypy/pypy/changeset/25a2e8817653/

Log:	merge default

diff --git a/pypy/module/test_lib_pypy/ctypes_tests/_ctypes_test.c b/pypy/module/test_lib_pypy/ctypes_tests/_ctypes_test.c
--- a/pypy/module/test_lib_pypy/ctypes_tests/_ctypes_test.c
+++ b/pypy/module/test_lib_pypy/ctypes_tests/_ctypes_test.c
@@ -316,8 +316,8 @@
 {
 }
 
-EXPORT(LONG_LONG) last_tf_arg_s;
-EXPORT(unsigned LONG_LONG) last_tf_arg_u;
+EXPORT(LONG_LONG) last_tf_arg_s = 0;
+EXPORT(unsigned LONG_LONG) last_tf_arg_u = 0;
 
 struct BITS {
 	int A: 1, B:2, C:3, D:4, E: 5, F: 6, G: 7, H: 8, I: 9;
diff --git a/pypy/objspace/std/strbufobject.py b/pypy/objspace/std/strbufobject.py
--- a/pypy/objspace/std/strbufobject.py
+++ b/pypy/objspace/std/strbufobject.py
@@ -5,6 +5,7 @@
 from pypy.objspace.std.bytesobject import (W_AbstractBytesObject,
     W_BytesObject, StringBuffer)
 from pypy.interpreter.gateway import interp2app, unwrap_spec
+from pypy.interpreter.error import OperationError
 from rpython.rlib.rstring import StringBuilder
 
 
@@ -46,15 +47,18 @@
         return space.wrap(self.length)
 
     def descr_add(self, space, w_other):
+        try:
+            other = W_BytesObject._op_val(space, w_other)
+        except OperationError as e:
+            if e.match(space, space.w_TypeError):
+                return space.w_NotImplemented
+            raise
         if self.builder.getlength() != self.length:
             builder = StringBuilder()
             builder.append(self.force())
         else:
             builder = self.builder
-        if isinstance(w_other, W_StringBufferObject):
-            builder.append(w_other.force())
-        else:
-            builder.append(w_other._value)
+        builder.append(other)
         return W_StringBufferObject(builder)
 
     def descr_str(self, space):
diff --git a/pypy/objspace/std/test/test_strbufobject.py b/pypy/objspace/std/test/test_strbufobject.py
--- a/pypy/objspace/std/test/test_strbufobject.py
+++ b/pypy/objspace/std/test/test_strbufobject.py
@@ -78,3 +78,8 @@
         c = '0'.__add__('1')
         x = c + a
         assert x == '01ab'
+
+    def test_add_non_string(self):
+        a = 'a'
+        a += 'b'
+        raises(TypeError, "a += 5")
diff --git a/rpython/__main__.py b/rpython/__main__.py
new file mode 100644
--- /dev/null
+++ b/rpython/__main__.py
@@ -0,0 +1,16 @@
+"""RPython translation usage:
+
+rpython  target 
+
+run with --help for more information
+"""
+
+import sys
+
+# no implicit targets
+if len(sys.argv) == 1:
+    print __doc__
+    sys.exit(1)
+
+from rpython.translator.goal.translate import main
+main()
diff --git a/rpython/jit/backend/llsupport/gc.py b/rpython/jit/backend/llsupport/gc.py
--- a/rpython/jit/backend/llsupport/gc.py
+++ b/rpython/jit/backend/llsupport/gc.py
@@ -55,7 +55,6 @@
 # ____________________________________________________________
 
 class GcLLDescription(GcCache):
-    malloc_zero_filled = True
 
     def __init__(self, gcdescr, translator=None, rtyper=None):
         GcCache.__init__(self, translator is not None, rtyper)
@@ -246,6 +245,7 @@
 
 class GcLLDescr_boehm(GcLLDescription):
     kind                  = 'boehm'
+    malloc_zero_filled    = True
     moving_gc             = False
     round_up              = False
     write_barrier_descr   = None
diff --git a/rpython/jit/backend/llsupport/jitframe.py b/rpython/jit/backend/llsupport/jitframe.py
--- a/rpython/jit/backend/llsupport/jitframe.py
+++ b/rpython/jit/backend/llsupport/jitframe.py
@@ -3,6 +3,7 @@
 from rpython.rlib.objectmodel import specialize
 from rpython.rlib.debug import ll_assert
 from rpython.rlib.objectmodel import enforceargs
+from rpython.rlib import rgc
 
 SIZEOFSIGNED = rffi.sizeof(lltype.Signed)
 IS_32BIT = (SIZEOFSIGNED == 4)
@@ -45,6 +46,7 @@
 # detailed explanation how it is on your architecture
 
 def jitframe_allocate(frame_info):
+    rgc.register_custom_trace_hook(JITFRAME, lambda_jitframe_trace)
     frame = lltype.malloc(JITFRAME, frame_info.jfi_frame_depth)
     frame.jf_frame_info = frame_info
     frame.jf_extra_stack_depth = 0
@@ -80,8 +82,6 @@
     ('jf_guard_exc', llmemory.GCREF),
     # in case the frame got reallocated, we have to forward it somewhere
     ('jf_forward', lltype.Ptr(JITFRAME)),
-    # absolutely useless field used to make up for tracing hooks inflexibilities
-    ('jf_gc_trace_state', lltype.Signed),
     # the actual frame
     ('jf_frame', lltype.Array(lltype.Signed)),
     # note that we keep length field, because it's crucial to have the data
@@ -105,75 +105,38 @@
 UNSIGN_SIZE = llmemory.sizeof(lltype.Unsigned)
 STACK_DEPTH_OFS = getofs('jf_extra_stack_depth')
 
-def jitframe_trace(obj_addr, prev):
-    if prev == llmemory.NULL:
-        (obj_addr + getofs('jf_gc_trace_state')).signed[0] = -1
-        return obj_addr + getofs('jf_descr')
-    fld = (obj_addr + getofs('jf_gc_trace_state')).signed[0]
-    if fld < 0:
-        if fld == -1:
-            (obj_addr + getofs('jf_gc_trace_state')).signed[0] = -2
-            return obj_addr + getofs('jf_force_descr')
-        elif fld == -2:
-            (obj_addr + getofs('jf_gc_trace_state')).signed[0] = -3
-            return obj_addr + getofs('jf_savedata')
-        elif fld == -3:
-            (obj_addr + getofs('jf_gc_trace_state')).signed[0] = -4
-            return obj_addr + getofs('jf_guard_exc')
-        elif fld == -4:
-            (obj_addr + getofs('jf_gc_trace_state')).signed[0] = -5
-            return obj_addr + getofs('jf_forward')
-        else:
-            if not (obj_addr + getofs('jf_gcmap')).address[0]:
-                return llmemory.NULL    # done
-            else:
-                fld = 0    # fall-through
-    # bit pattern
-    # decode the pattern
+def jitframe_trace(gc, obj_addr, callback, arg):
+    gc._trace_callback(callback, arg, obj_addr + getofs('jf_descr'))
+    gc._trace_callback(callback, arg, obj_addr + getofs('jf_force_descr'))
+    gc._trace_callback(callback, arg, obj_addr + getofs('jf_savedata'))
+    gc._trace_callback(callback, arg, obj_addr + getofs('jf_guard_exc'))
+    gc._trace_callback(callback, arg, obj_addr + getofs('jf_forward'))
+
     if IS_32BIT:
-        # 32 possible bits
-        state = fld & 0x1f
-        no = fld >> 5
         MAX = 32
     else:
-        # 64 possible bits
-        state = fld & 0x3f
-        no = fld >> 6
         MAX = 64
     gcmap = (obj_addr + getofs('jf_gcmap')).address[0]
+    if not gcmap:
+        return      # done
     gcmap_lgt = (gcmap + GCMAPLENGTHOFS).signed[0]
+    no = 0
     while no < gcmap_lgt:
         cur = (gcmap + GCMAPBASEOFS + UNSIGN_SIZE * no).unsigned[0]
-        while not (cur & (1 << state)):
-            state += 1
-            if state == MAX:
-                no += 1
-                state = 0
-                break      # next iteration of the outermost loop
-        else:
-            # found it
-            index = no * SIZEOFSIGNED * 8 + state
-            # save new state
-            state += 1
-            if state == MAX:
-                no += 1
-                state = 0
-            if IS_32BIT:
-                new_state = state | (no << 5)
-            else:
-                new_state = state | (no << 6)
-            (obj_addr + getofs('jf_gc_trace_state')).signed[0] = new_state
-            # sanity check
-            frame_lgt = (obj_addr + getofs('jf_frame') + LENGTHOFS).signed[0]
-            ll_assert(index < frame_lgt, "bogus frame field get")
-            return (obj_addr + getofs('jf_frame') + BASEITEMOFS + SIGN_SIZE *
-                    (index))
-    return llmemory.NULL
-
-CUSTOMTRACEFUNC = lltype.FuncType([llmemory.Address, llmemory.Address],
-                                  llmemory.Address)
-jitframe_trace_ptr = llhelper(lltype.Ptr(CUSTOMTRACEFUNC), jitframe_trace)
-
-lltype.attachRuntimeTypeInfo(JITFRAME, customtraceptr=jitframe_trace_ptr)
+        bitindex = 0
+        while bitindex < MAX:
+            if cur & (1 << bitindex):
+                # the 'bitindex' is set in 'cur'
+                index = no * SIZEOFSIGNED * 8 + bitindex
+                # sanity check
+                frame_lgt = (obj_addr + getofs('jf_frame') + LENGTHOFS) \
+                    .signed[0]
+                ll_assert(index < frame_lgt, "bogus frame field get")
+                gc._trace_callback(callback, arg,
+                                   obj_addr + getofs('jf_frame') +
+                                   BASEITEMOFS + SIGN_SIZE * index)
+            bitindex += 1
+        no += 1
+lambda_jitframe_trace = lambda: jitframe_trace
 
 JITFRAMEPTR = lltype.Ptr(JITFRAME)
diff --git a/rpython/jit/backend/llsupport/rewrite.py b/rpython/jit/backend/llsupport/rewrite.py
--- a/rpython/jit/backend/llsupport/rewrite.py
+++ b/rpython/jit/backend/llsupport/rewrite.py
@@ -263,7 +263,7 @@
     def gen_malloc_frame(self, frame_info, frame, size_box):
         descrs = self.gc_ll_descr.getframedescrs(self.cpu)
         if self.gc_ll_descr.kind == 'boehm':
-            op0 = ResOperation(rop.GETFIELD_GC, [history.ConstInt(frame_info)],
+            op0 = ResOperation(rop.GETFIELD_RAW, [history.ConstInt(frame_info)],
                                size_box,
                                descr=descrs.jfi_frame_depth)
             self.newops.append(op0)
@@ -272,7 +272,7 @@
             self.handle_new_array(descrs.arraydescr, op1)
         else:
             # we read size in bytes here, not the length
-            op0 = ResOperation(rop.GETFIELD_GC, [history.ConstInt(frame_info)],
+            op0 = ResOperation(rop.GETFIELD_RAW, [history.ConstInt(frame_info)],
                                size_box,
                                descr=descrs.jfi_frame_size)
             self.newops.append(op0)
@@ -282,7 +282,7 @@
             # we need to explicitely zero all the gc fields, because
             # of the unusal malloc pattern
             extra_ops = [
-                ResOperation(rop.GETFIELD_GC, [history.ConstInt(frame_info)],
+                ResOperation(rop.GETFIELD_RAW, [history.ConstInt(frame_info)],
                              length_box, descr=descrs.jfi_frame_depth),
                 ResOperation(rop.SETFIELD_GC, [frame, self.c_zero],
                              None, descr=descrs.jf_extra_stack_depth),
diff --git a/rpython/jit/backend/llsupport/test/test_gc.py b/rpython/jit/backend/llsupport/test/test_gc.py
--- a/rpython/jit/backend/llsupport/test/test_gc.py
+++ b/rpython/jit/backend/llsupport/test/test_gc.py
@@ -254,11 +254,15 @@
     frame.jf_gcmap[2] = r_uint(2 | 16 | 32 | 128)
     frame.jf_gcmap[3] = r_uint(0)
     frame_adr = llmemory.cast_ptr_to_adr(frame)
+    #
     all_addrs = []
-    next = jitframe.jitframe_trace(frame_adr, llmemory.NULL)
-    while next:
-        all_addrs.append(next)
-        next = jitframe.jitframe_trace(frame_adr, next)
+    class FakeGC:
+        def _trace_callback(self, callback, arg, addr):
+            assert callback == "hello"
+            assert arg == "world"
+            all_addrs.append(addr)
+    jitframe.jitframe_trace(FakeGC(), frame_adr, "hello", "world")
+    #
     counter = 0
     for name in jitframe.JITFRAME._names:
         TP = getattr(jitframe.JITFRAME, name)
@@ -297,12 +301,12 @@
     frame.jf_gcmap[0] = r_uint(18446744073441116160)
     frame.jf_gcmap[1] = r_uint(18446740775107559407)
     frame.jf_gcmap[2] = r_uint(3)
-    all_addrs = []
     frame_adr = llmemory.cast_ptr_to_adr(frame)
-    next = jitframe.jitframe_trace(frame_adr, llmemory.NULL)
-    while next:
-        all_addrs.append(next)
-        next = jitframe.jitframe_trace(frame_adr, next)
+    class FakeGC:
+        def _trace_callback(self, callback, arg, addr):
+            assert callback == "hello"
+            assert arg == "world"
+    jitframe.jitframe_trace(FakeGC(), frame_adr, "hello", "world")
     # assert did not hang
 
     lltype.free(frame_info, flavor='raw')
diff --git a/rpython/jit/backend/llsupport/test/test_rewrite.py b/rpython/jit/backend/llsupport/test/test_rewrite.py
--- a/rpython/jit/backend/llsupport/test/test_rewrite.py
+++ b/rpython/jit/backend/llsupport/test/test_rewrite.py
@@ -981,10 +981,10 @@
         i2 = call_assembler(i0, f0, descr=casmdescr)
         """, """
         [i0, f0]
-        i1 = getfield_gc(ConstClass(frame_info), descr=jfi_frame_size)
+        i1 = getfield_raw(ConstClass(frame_info), descr=jfi_frame_size)
         p1 = call_malloc_nursery_varsize_frame(i1)
         setfield_gc(p1, 0, descr=tiddescr)
-        i2 = getfield_gc(ConstClass(frame_info), descr=jfi_frame_depth)
+        i2 = getfield_raw(ConstClass(frame_info), descr=jfi_frame_depth)
         setfield_gc(p1, 0, descr=jf_extra_stack_depth)
         setfield_gc(p1, NULL, descr=jf_savedata)
         setfield_gc(p1, NULL, descr=jf_force_descr)
diff --git a/rpython/jit/backend/x86/regalloc.py b/rpython/jit/backend/x86/regalloc.py
--- a/rpython/jit/backend/x86/regalloc.py
+++ b/rpython/jit/backend/x86/regalloc.py
@@ -1398,7 +1398,7 @@
         startindex_loc = self.rm.make_sure_var_in_reg(args[1], args)
         if 0 <= constbytes <= 16 * 8 and (
                 valid_addressing_size(itemsize) or
--               isinstance(startindex_loc, ImmedLoc)):
+                isinstance(startindex_loc, ImmedLoc)):
             if IS_X86_64:
                 null_loc = X86_64_XMM_SCRATCH_REG
             else:
diff --git a/rpython/jit/metainterp/logger.py b/rpython/jit/metainterp/logger.py
--- a/rpython/jit/metainterp/logger.py
+++ b/rpython/jit/metainterp/logger.py
@@ -137,6 +137,14 @@
             s = jd_sd.warmstate.get_location_str(op.getarglist()[3:])
             s = s.replace(',', '.') # we use comma for argument splitting
             return "debug_merge_point(%d, %d, '%s')" % (op.getarg(1).getint(), op.getarg(2).getint(), s)
+        if op.getopnum() == rop.JIT_DEBUG:
+            args = op.getarglist()
+            s = args[0]._get_str()
+            s = s.replace(',', '.') # we use comma for argument splitting
+            s2 = ''
+            for box in args[1:]:
+                s2 += ', %d' % box.getint()
+            return "jit_debug('%s'%s)" % (s, s2)
         if ops_offset is None:
             offset = -1
         else:
diff --git a/rpython/jit/metainterp/optimizeopt/heap.py b/rpython/jit/metainterp/optimizeopt/heap.py
--- a/rpython/jit/metainterp/optimizeopt/heap.py
+++ b/rpython/jit/metainterp/optimizeopt/heap.py
@@ -273,6 +273,7 @@
             opnum == rop.STRSETITEM or           # no effect on GC struct/array
             opnum == rop.UNICODESETITEM or       # no effect on GC struct/array
             opnum == rop.DEBUG_MERGE_POINT or    # no effect whatsoever
+            opnum == rop.JIT_DEBUG or            # no effect whatsoever
             opnum == rop.COPYSTRCONTENT or       # no effect on GC struct/array
             opnum == rop.COPYUNICODECONTENT):    # no effect on GC struct/array
             return
diff --git a/rpython/jit/metainterp/optimizeopt/optimizer.py b/rpython/jit/metainterp/optimizeopt/optimizer.py
--- a/rpython/jit/metainterp/optimizeopt/optimizer.py
+++ b/rpython/jit/metainterp/optimizeopt/optimizer.py
@@ -661,6 +661,9 @@
     def optimize_DEBUG_MERGE_POINT(self, op):
         self.emit_operation(op)
 
+    def optimize_JIT_DEBUG(self, op):
+        self.emit_operation(op)
+
     def optimize_STRGETITEM(self, op):
         indexvalue = self.getvalue(op.getarg(1))
         if indexvalue.is_constant():
diff --git a/rpython/jit/metainterp/optimizeopt/rewrite.py b/rpython/jit/metainterp/optimizeopt/rewrite.py
--- a/rpython/jit/metainterp/optimizeopt/rewrite.py
+++ b/rpython/jit/metainterp/optimizeopt/rewrite.py
@@ -8,8 +8,7 @@
 from rpython.jit.metainterp.optimizeopt.optimizer import (Optimization, REMOVED,
     CONST_0, CONST_1)
 from rpython.jit.metainterp.optimizeopt.util import _findall, make_dispatcher_method
-from rpython.jit.metainterp.resoperation import (opboolinvers, opboolreflex, rop,
-    ResOperation)
+from rpython.jit.metainterp.resoperation import rop, ResOperation, opclasses
 from rpython.rlib.rarithmetic import highest_bit
 import math
 
@@ -26,9 +25,10 @@
             sb.add_potential(op)
 
     def propagate_forward(self, op):
-        args = self.optimizer.make_args_key(op)
-        if self.find_rewritable_bool(op, args):
-            return
+        if op.boolinverse != -1 or op.boolreflex != -1:
+            args = self.optimizer.make_args_key(op)
+            if self.find_rewritable_bool(op, args):
+                return
 
         dispatch_opt(self, op)
 
@@ -48,21 +48,15 @@
 
 
     def find_rewritable_bool(self, op, args):
-        try:
-            oldopnum = opboolinvers[op.getopnum()]
-        except KeyError:
-            pass
-        else:
+        oldopnum = op.boolinverse
+        if oldopnum != -1:
             targs = self.optimizer.make_args_key(ResOperation(oldopnum, [args[0], args[1]],
                                                               None))
             if self.try_boolinvers(op, targs):
                 return True
 
-        try:
-            oldopnum = opboolreflex[op.getopnum()] # FIXME: add INT_ADD, INT_MUL
-        except KeyError:
-            pass
-        else:
+        oldopnum = op.boolreflex # FIXME: add INT_ADD, INT_MUL
+        if oldopnum != -1:
             targs = self.optimizer.make_args_key(ResOperation(oldopnum, [args[1], args[0]],
                                                               None))
             oldop = self.get_pure_result(targs)
@@ -70,13 +64,12 @@
                 self.make_equal_to(op.result, self.getvalue(oldop.result))
                 return True
 
-        try:
-            oldopnum = opboolinvers[opboolreflex[op.getopnum()]]
-        except KeyError:
-            pass
-        else:
-            targs = self.optimizer.make_args_key(ResOperation(oldopnum, [args[1], args[0]],
-                                                              None))
+        if op.boolreflex == -1:
+            return False
+        oldopnum = opclasses[op.boolreflex].boolinverse
+        if oldopnum != -1:
+            targs = self.optimizer.make_args_key(
+                ResOperation(oldopnum, [args[1], args[0]], None))
             if self.try_boolinvers(op, targs):
                 return True
 
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
@@ -2024,6 +2024,27 @@
         """
         self.optimize_loop(ops, expected)
 
+    def test_virtual_raw_buffer_forced_but_slice_not_forced(self):
+        ops = """
+        [f1]
+        i0 = call('malloc', 16, descr=raw_malloc_descr)
+        guard_no_exception() []
+        i1 = int_add(i0, 8)
+        escape(i0)
+        setarrayitem_raw(i1, 0, f1, descr=rawarraydescr_float)
+        jump(f1)
+        """
+        expected = """
+        [f1]
+        i0 = call('malloc', 16, descr=raw_malloc_descr)
+        #guard_no_exception() []  # XXX should appear
+        escape(i0)
+        i1 = int_add(i0, 8)
+        setarrayitem_raw(i1, 0, f1, descr=rawarraydescr_float)
+        jump(f1)
+        """
+        self.optimize_loop(ops, expected)
+
     def test_duplicate_getfield_1(self):
         ops = """
         [p1, p2]
diff --git a/rpython/jit/metainterp/optimizeopt/virtualize.py b/rpython/jit/metainterp/optimizeopt/virtualize.py
--- a/rpython/jit/metainterp/optimizeopt/virtualize.py
+++ b/rpython/jit/metainterp/optimizeopt/virtualize.py
@@ -443,9 +443,17 @@
         self.buffer.values[i] = newval
 
     def getitem_raw(self, offset, length, descr):
+        if not self.is_virtual():
+            raise InvalidRawOperation
+            # see 'test_virtual_raw_buffer_forced_but_slice_not_forced'
+            # for the test above: it's not enough to check is_virtual()
+            # on the original object, because it might be a VRawSliceValue
+            # instead.  If it is a virtual one, then we'll reach here anway.
         return self.buffer.read_value(offset, length, descr)
 
     def setitem_raw(self, offset, length, descr, value):
+        if not self.is_virtual():
+            raise InvalidRawOperation
         self.buffer.write_value(offset, length, descr, value)
 
     def _really_force(self, optforce):
@@ -818,12 +826,10 @@
                 try:
                     itemvalue = value.getitem_raw(offset, itemsize, descr)
                 except InvalidRawOperation:
-                    box = value.force_box(self)
-                    op.setarg(0, box)
-                    self.emit_operation(op)
+                    pass
                 else:
                     self.make_equal_to(op.result, itemvalue)
-                return
+                    return
         value.ensure_nonnull()
         self.emit_operation(op)
     optimize_GETARRAYITEM_RAW_F = optimize_GETARRAYITEM_RAW_I
@@ -837,11 +843,9 @@
                 itemvalue = self.getvalue(op.getarg(2))
                 try:
                     value.setitem_raw(offset, itemsize, descr, itemvalue)
+                    return
                 except InvalidRawOperation:
-                    box = value.force_box(self)
-                    op.setarg(0, box)
-                    self.emit_operation(op)
-                return
+                    pass
         value.ensure_nonnull()
         self.emit_operation(op)
 
@@ -861,12 +865,10 @@
                 try:
                     itemvalue = value.getitem_raw(offset, itemsize, descr)
                 except InvalidRawOperation:
-                    box = value.force_box(self)
-                    op.setarg(0, box)
-                    self.emit_operation(op)
+                    pass
                 else:
                     self.make_equal_to(op.result, itemvalue)
-                return
+                    return
         value.ensure_nonnull()
         self.emit_operation(op)
     optimize_RAW_LOAD_F = optimize_RAW_LOAD_I
@@ -880,11 +882,9 @@
                 itemvalue = self.getvalue(op.getarg(2))
                 try:
                     value.setitem_raw(offset, itemsize, descr, itemvalue)
+                    return
                 except InvalidRawOperation:
-                    box = value.force_box(self)
-                    op.setarg(0, box)
-                    self.emit_operation(op)
-                return
+                    pass
         value.ensure_nonnull()
         self.emit_operation(op)
 
diff --git a/rpython/jit/metainterp/pyjitpl.py b/rpython/jit/metainterp/pyjitpl.py
--- a/rpython/jit/metainterp/pyjitpl.py
+++ b/rpython/jit/metainterp/pyjitpl.py
@@ -1179,10 +1179,7 @@
 
     @arguments("box", "box", "box", "box", "box")
     def opimpl_jit_debug(self, stringbox, arg1box, arg2box, arg3box, arg4box):
-        from rpython.rtyper.lltypesystem import rstr
-        from rpython.rtyper.annlowlevel import hlstr
-        msg = stringbox.getref(lltype.Ptr(rstr.STR))
-        debug_print('jit_debug:', hlstr(msg),
+        debug_print('jit_debug:', stringbox._get_str(),
                     arg1box.getint(), arg2box.getint(),
                     arg3box.getint(), arg4box.getint())
         args = [stringbox, arg1box, arg2box, arg3box, arg4box]
diff --git a/rpython/jit/metainterp/resoperation.py b/rpython/jit/metainterp/resoperation.py
--- a/rpython/jit/metainterp/resoperation.py
+++ b/rpython/jit/metainterp/resoperation.py
@@ -39,6 +39,8 @@
     opnum = 0
     _cls_has_bool_result = False
     type = 'v'
+    boolreflex = -1
+    boolinverse = -1
 
     _attrs_ = ()
 
@@ -743,7 +745,7 @@
 setup(__name__ == '__main__')   # print out the table when run directly
 del _oplist
 
-opboolinvers = {
+_opboolinverse = {
     rop.INT_EQ: rop.INT_NE,
     rop.INT_NE: rop.INT_EQ,
     rop.INT_LT: rop.INT_GE,
@@ -767,7 +769,7 @@
     rop.PTR_NE: rop.PTR_EQ,
 }
 
-opboolreflex = {
+_opboolreflex = {
     rop.INT_EQ: rop.INT_EQ,
     rop.INT_NE: rop.INT_NE,
     rop.INT_LT: rop.INT_GT,
@@ -791,6 +793,19 @@
     rop.PTR_NE: rop.PTR_NE,
 }
 
+def setup2():
+    for cls in opclasses:
+        if cls is None:
+            continue
+        opnum = cls.opnum
+        if opnum in _opboolreflex:
+            cls.boolreflex = _opboolreflex[opnum]
+        if opnum in _opboolinverse:
+            cls.boolinverse = _opboolinverse[opnum]
+
+setup2()
+del _opboolinverse
+del _opboolreflex
 
 def get_deep_immutable_oplist(operations):
     """
diff --git a/rpython/jit/metainterp/test/test_logger.py b/rpython/jit/metainterp/test/test_logger.py
--- a/rpython/jit/metainterp/test/test_logger.py
+++ b/rpython/jit/metainterp/test/test_logger.py
@@ -137,6 +137,17 @@
         assert loop.operations[0].getarg(2).getint() == 0
         assert oloop.operations[0].getarg(2)._get_str() == "dupa"
 
+    def test_jit_debug(self):
+        inp = '''
+        []
+        jit_debug('foobar', -1, 5)
+        '''
+        _, loop, oloop = self.reparse(inp)
+        assert loop.operations[0].getarg(0)._get_str() == "foobar"
+        assert loop.operations[0].getarg(1).getint() == -1
+        assert oloop.operations[0].getarg(0)._get_str() == "foobar"
+        assert oloop.operations[0].getarg(1).getint() == -1
+
     def test_floats(self):
         inp = '''
         [f0]
diff --git a/rpython/jit/tool/oparser.py b/rpython/jit/tool/oparser.py
--- a/rpython/jit/tool/oparser.py
+++ b/rpython/jit/tool/oparser.py
@@ -321,8 +321,9 @@
         first_comment = None
         for line in lines:
             # for simplicity comments are not allowed on
-            # debug_merge_point lines
-            if '#' in line and 'debug_merge_point(' not in line:
+            # debug_merge_point or jit_debug lines
+            if '#' in line and ('debug_merge_point(' not in line and
+                                'jit_debug(' not in line):
                 if line.lstrip()[0] == '#': # comment only
                     if first_comment is None:
                         first_comment = line
diff --git a/rpython/memory/gc/base.py b/rpython/memory/gc/base.py
--- a/rpython/memory/gc/base.py
+++ b/rpython/memory/gc/base.py
@@ -71,7 +71,6 @@
                             member_index,
                             is_rpython_class,
                             has_custom_trace,
-                            get_custom_trace,
                             fast_path_tracing,
                             has_gcptr,
                             cannot_pin):
@@ -90,7 +89,6 @@
         self.member_index = member_index
         self.is_rpython_class = is_rpython_class
         self.has_custom_trace = has_custom_trace
-        self.get_custom_trace = get_custom_trace
         self.fast_path_tracing = fast_path_tracing
         self.has_gcptr = has_gcptr
         self.cannot_pin = cannot_pin
@@ -235,16 +233,14 @@
                 item += itemlength
                 length -= 1
         if self.has_custom_trace(typeid):
-            generator = self.get_custom_trace(typeid)
-            item = llmemory.NULL
-            while True:
-                item = generator(obj, item)
-                if not item:
-                    break
-                if self.points_to_valid_gc_object(item):
-                    callback(item, arg)
+            self.custom_trace_dispatcher(obj, typeid, callback, arg)
     _trace_slow_path._annspecialcase_ = 'specialize:arg(2)'
 
+    def _trace_callback(self, callback, arg, addr):
+        if self.is_valid_gc_object(addr.address[0]):
+            callback(addr, arg)
+    _trace_callback._annspecialcase_ = 'specialize:arg(1)'
+
     def trace_partial(self, obj, start, stop, callback, arg):
         """Like trace(), but only walk the array part, for indices in
         range(start, stop).  Must only be called if has_gcptr_in_varsize().
diff --git a/rpython/memory/gctransform/framework.py b/rpython/memory/gctransform/framework.py
--- a/rpython/memory/gctransform/framework.py
+++ b/rpython/memory/gctransform/framework.py
@@ -1,9 +1,11 @@
 from rpython.annotator import model as annmodel
 from rpython.rtyper.llannotation import SomeAddress, SomePtr
 from rpython.rlib import rgc
+from rpython.rlib.objectmodel import specialize
+from rpython.rlib.unroll import unrolling_iterable
 from rpython.rtyper import rmodel, annlowlevel
 from rpython.rtyper.lltypesystem import lltype, llmemory, rffi, llgroup
-from rpython.rtyper.lltypesystem.lloperation import LL_OPERATIONS
+from rpython.rtyper.lltypesystem.lloperation import LL_OPERATIONS, llop
 from rpython.memory import gctypelayout
 from rpython.memory.gctransform.log import log
 from rpython.memory.gctransform.support import get_rtti, ll_call_destructor
@@ -239,6 +241,7 @@
             root_walker.need_stacklet_support(self, getfn)
 
         self.layoutbuilder.encode_type_shapes_now()
+        self.create_custom_trace_funcs(gcdata.gc, translator.rtyper)
 
         annhelper.finish()   # at this point, annotate all mix-level helpers
         annhelper.backend_optimize()
@@ -502,6 +505,29 @@
                                                    [SomeAddress()],
                                                    annmodel.s_None)
 
+    def create_custom_trace_funcs(self, gc, rtyper):
+        custom_trace_funcs = tuple(rtyper.custom_trace_funcs)
+        rtyper.custom_trace_funcs = custom_trace_funcs
+        # too late to register new custom trace functions afterwards
+
+        custom_trace_funcs_unrolled = unrolling_iterable(
+            [(self.get_type_id(TP), func) for TP, func in custom_trace_funcs])
+
+        @specialize.arg(2)
+        def custom_trace_dispatcher(obj, typeid, callback, arg):
+            for type_id_exp, func in custom_trace_funcs_unrolled:
+                if (llop.combine_ushort(lltype.Signed, typeid, 0) ==
+                    llop.combine_ushort(lltype.Signed, type_id_exp, 0)):
+                    func(gc, obj, callback, arg)
+                    return
+            else:
+                assert False
+
+        gc.custom_trace_dispatcher = custom_trace_dispatcher
+
+        for TP, func in custom_trace_funcs:
+            self.gcdata._has_got_custom_trace(self.get_type_id(TP))
+            specialize.arg(2)(func)
 
     def consider_constant(self, TYPE, value):
         self.layoutbuilder.consider_constant(TYPE, value, self.gcdata.gc)
diff --git a/rpython/memory/gctransform/shadowstack.py b/rpython/memory/gctransform/shadowstack.py
--- a/rpython/memory/gctransform/shadowstack.py
+++ b/rpython/memory/gctransform/shadowstack.py
@@ -73,16 +73,13 @@
             return top
         self.decr_stack = decr_stack
 
-        root_iterator = get_root_iterator(gctransformer)
         def walk_stack_root(callback, start, end):
-            root_iterator.setcontext(NonConstant(llmemory.NULL))
             gc = self.gc
             addr = end
-            while True:
-                addr = root_iterator.nextleft(gc, start, addr)
-                if addr == llmemory.NULL:
-                    return
-                callback(gc, addr)
+            while addr != start:
+                addr -= sizeofaddr
+                if gc.points_to_valid_gc_object(addr):
+                    callback(gc, addr)
         self.rootstackhook = walk_stack_root
 
         self.shadow_stack_pool = ShadowStackPool(gcdata)
@@ -349,25 +346,6 @@
                 raise MemoryError
 
 
-def get_root_iterator(gctransformer):
-    if hasattr(gctransformer, '_root_iterator'):
-        return gctransformer._root_iterator     # if already built
-    class RootIterator(object):
-        def _freeze_(self):
-            return True
-        def setcontext(self, context):
-            pass
-        def nextleft(self, gc, start, addr):
-            while addr != start:
-                addr -= sizeofaddr
-                if gc.points_to_valid_gc_object(addr):
-                    return addr
-            return llmemory.NULL
-    result = RootIterator()
-    gctransformer._root_iterator = result
-    return result
-
-
 def get_shadowstackref(root_walker, gctransformer):
     if hasattr(gctransformer, '_SHADOWSTACKREF'):
         return gctransformer._SHADOWSTACKREF
@@ -381,19 +359,19 @@
                                      rtti=True)
     SHADOWSTACKREFPTR.TO.become(SHADOWSTACKREF)
 
+    def customtrace(gc, obj, callback, arg):
+        obj = llmemory.cast_adr_to_ptr(obj, SHADOWSTACKREFPTR)
+        addr = obj.top
+        start = obj.base
+        while addr != start:
+            addr -= sizeofaddr
+            gc._trace_callback(callback, arg, addr)
+
     gc = gctransformer.gcdata.gc
-    root_iterator = get_root_iterator(gctransformer)
-
-    def customtrace(obj, prev):
-        obj = llmemory.cast_adr_to_ptr(obj, SHADOWSTACKREFPTR)
-        if not prev:
-            root_iterator.setcontext(obj.context)
-            prev = obj.top
-        return root_iterator.nextleft(gc, obj.base, prev)
-
-    CUSTOMTRACEFUNC = lltype.FuncType([llmemory.Address, llmemory.Address],
-                                      llmemory.Address)
-    customtraceptr = llhelper(lltype.Ptr(CUSTOMTRACEFUNC), customtrace)
+    assert not hasattr(gc, 'custom_trace_dispatcher')
+    # ^^^ create_custom_trace_funcs() must not run before this
+    gctransformer.translator.rtyper.custom_trace_funcs.append(
+        (SHADOWSTACKREF, customtrace))
 
     def shadowstack_destructor(shadowstackref):
         if root_walker.stacklet_support:
@@ -414,8 +392,7 @@
     destrptr = gctransformer.annotate_helper(shadowstack_destructor,
                                              [SHADOWSTACKREFPTR], lltype.Void)
 
-    lltype.attachRuntimeTypeInfo(SHADOWSTACKREF, customtraceptr=customtraceptr,
-                                 destrptr=destrptr)
+    lltype.attachRuntimeTypeInfo(SHADOWSTACKREF, destrptr=destrptr)
 
     gctransformer._SHADOWSTACKREF = SHADOWSTACKREF
     return SHADOWSTACKREF
diff --git a/rpython/memory/gctypelayout.py b/rpython/memory/gctypelayout.py
--- a/rpython/memory/gctypelayout.py
+++ b/rpython/memory/gctypelayout.py
@@ -21,18 +21,12 @@
     # It is called with the object as first argument, and the previous
     # returned address (or NULL the first time) as the second argument.
     FINALIZER_FUNC = lltype.FuncType([llmemory.Address], lltype.Void)
-    CUSTOMTRACER_FUNC = lltype.FuncType([llmemory.Address, llmemory.Address],
-                                        llmemory.Address)
     FINALIZER = lltype.Ptr(FINALIZER_FUNC)
-    CUSTOMTRACER = lltype.Ptr(CUSTOMTRACER_FUNC)
-    EXTRA = lltype.Struct("type_info_extra",
-                          ('finalizer', FINALIZER),
-                          ('customtracer', CUSTOMTRACER))
 
     # structure describing the layout of a typeid
     TYPE_INFO = lltype.Struct("type_info",
         ("infobits",       lltype.Signed),    # combination of the T_xxx consts
-        ("extra",          lltype.Ptr(EXTRA)),
+        ("finalizer",      FINALIZER),
         ("fixedsize",      lltype.Signed),
         ("ofstoptrs",      lltype.Ptr(OFFSETS_TO_GC_PTR)),
         hints={'immutable': True},
@@ -84,26 +78,18 @@
         return (infobits & T_IS_GCARRAY_OF_GCPTR) != 0
 
     def q_cannot_pin(self, typeid):
-        infobits = self.get(typeid).infobits
-        ANY = (T_HAS_GCPTR |
-               T_IS_WEAKREF |
-               T_HAS_FINALIZER |
-               T_HAS_LIGHTWEIGHT_FINALIZER)
-        return (infobits & ANY) != 0
+        typeinfo = self.get(typeid)
+        ANY = (T_HAS_GCPTR | T_IS_WEAKREF)
+        return (typeinfo.infobits & ANY) != 0 or bool(typeinfo.finalizer)
 
     def q_finalizer(self, typeid):
-        typeinfo = self.get(typeid)
-        if typeinfo.infobits & T_HAS_FINALIZER:
-            return typeinfo.extra.finalizer
-        else:
-            return lltype.nullptr(GCData.FINALIZER_FUNC)
+        return self.get(typeid).finalizer
 
     def q_light_finalizer(self, typeid):
         typeinfo = self.get(typeid)
         if typeinfo.infobits & T_HAS_LIGHTWEIGHT_FINALIZER:
-            return typeinfo.extra.finalizer
-        else:
-            return lltype.nullptr(GCData.FINALIZER_FUNC)
+            return typeinfo.finalizer
+        return lltype.nullptr(GCData.FINALIZER_FUNC)
 
     def q_offsets_to_gc_pointers(self, typeid):
         return self.get(typeid).ofstoptrs
@@ -141,12 +127,6 @@
         infobits = self.get(typeid).infobits
         return infobits & T_HAS_CUSTOM_TRACE != 0
 
-    def q_get_custom_trace(self, typeid):
-        ll_assert(self.q_has_custom_trace(typeid),
-                  "T_HAS_CUSTOM_TRACE missing")
-        typeinfo = self.get(typeid)
-        return typeinfo.extra.customtracer
-
     def q_fast_path_tracing(self, typeid):
         # return True if none of the flags T_HAS_GCPTR_IN_VARSIZE,
         # T_IS_GCARRAY_OF_GCPTR or T_HAS_CUSTOM_TRACE is set
@@ -173,11 +153,14 @@
             self.q_member_index,
             self.q_is_rpython_class,
             self.q_has_custom_trace,
-            self.q_get_custom_trace,
             self.q_fast_path_tracing,
             self.q_has_gcptr,
             self.q_cannot_pin)
 
+    def _has_got_custom_trace(self, typeid):
+        type_info = self.get(typeid)
+        type_info.infobits |= (T_HAS_CUSTOM_TRACE | T_HAS_GCPTR)
+
 
 # the lowest 16bits are used to store group member index
 T_MEMBER_INDEX              =   0xffff
@@ -186,9 +169,8 @@
 T_IS_GCARRAY_OF_GCPTR       = 0x040000
 T_IS_WEAKREF                = 0x080000
 T_IS_RPYTHON_INSTANCE       = 0x100000 # the type is a subclass of OBJECT
-T_HAS_FINALIZER             = 0x200000
-T_HAS_CUSTOM_TRACE          = 0x400000
-T_HAS_LIGHTWEIGHT_FINALIZER = 0x800000
+T_HAS_CUSTOM_TRACE          = 0x200000
+T_HAS_LIGHTWEIGHT_FINALIZER = 0x400000
 T_HAS_GCPTR                 = 0x1000000
 T_KEY_MASK                  = intmask(0xFE000000) # bug detection only
 T_KEY_VALUE                 = intmask(0x5A000000) # bug detection only
@@ -217,18 +199,11 @@
     #
     fptrs = builder.special_funcptr_for_type(TYPE)
     if fptrs:
-        extra = lltype.malloc(GCData.EXTRA, zero=True, immortal=True,
-                              flavor='raw')
         if "finalizer" in fptrs:
-            extra.finalizer = fptrs["finalizer"]
-            infobits |= T_HAS_FINALIZER
+            info.finalizer = fptrs["finalizer"]
         if "light_finalizer" in fptrs:
-            extra.finalizer = fptrs["light_finalizer"]
-            infobits |= T_HAS_FINALIZER | T_HAS_LIGHTWEIGHT_FINALIZER
-        if "custom_trace" in fptrs:
-            extra.customtracer = fptrs["custom_trace"]
-            infobits |= T_HAS_CUSTOM_TRACE | T_HAS_GCPTR
-        info.extra = extra
+            info.finalizer = fptrs["light_finalizer"]
+            infobits |= T_HAS_LIGHTWEIGHT_FINALIZER
     #
     if not TYPE._is_varsize():
         info.fixedsize = llarena.round_up_for_allocation(
@@ -420,7 +395,9 @@
         return None
 
     def initialize_gc_query_function(self, gc):
-        return GCData(self.type_info_group).set_query_functions(gc)
+        gcdata = GCData(self.type_info_group)
+        gcdata.set_query_functions(gc)
+        return gcdata
 
     def consider_constant(self, TYPE, value, gc):
         if value is not lltype.top_container(value):
diff --git a/rpython/memory/gcwrapper.py b/rpython/memory/gcwrapper.py
--- a/rpython/memory/gcwrapper.py
+++ b/rpython/memory/gcwrapper.py
@@ -29,7 +29,7 @@
                                                lltype2vtable,
                                                self.llinterp)
         self.get_type_id = layoutbuilder.get_type_id
-        layoutbuilder.initialize_gc_query_function(self.gc)
+        gcdata = layoutbuilder.initialize_gc_query_function(self.gc)
 
         constants = collect_constants(flowgraphs)
         for obj in constants:
@@ -38,8 +38,25 @@
 
         self.constantroots = layoutbuilder.addresses_of_static_ptrs
         self.constantrootsnongc = layoutbuilder.addresses_of_static_ptrs_in_nongc
+        self.prepare_custom_trace_funcs(gcdata)
         self._all_prebuilt_gc = layoutbuilder.all_prebuilt_gc
 
+    def prepare_custom_trace_funcs(self, gcdata):
+        custom_trace_funcs = self.llinterp.typer.custom_trace_funcs
+
+        def custom_trace(obj, typeid, callback, arg):
+            for TP, func in custom_trace_funcs:
+                if typeid == self.get_type_id(TP):
+                    func(self.gc, obj, callback, arg)
+                    return
+            else:
+                assert False
+        
+        for TP, func in custom_trace_funcs:
+            gcdata._has_got_custom_trace(self.get_type_id(TP))
+
+        self.gc.custom_trace_dispatcher = custom_trace
+
     # ____________________________________________________________
     #
     # Interface for the llinterp
diff --git a/rpython/memory/test/gc_test_base.py b/rpython/memory/test/gc_test_base.py
--- a/rpython/memory/test/gc_test_base.py
+++ b/rpython/memory/test/gc_test_base.py
@@ -6,7 +6,7 @@
 from rpython.rtyper.test.test_llinterp import get_interpreter
 from rpython.rtyper.lltypesystem import lltype
 from rpython.rtyper.lltypesystem.lloperation import llop
-from rpython.rlib.objectmodel import we_are_translated
+from rpython.rlib.objectmodel import we_are_translated, keepalive_until_here
 from rpython.rlib.objectmodel import compute_unique_id
 from rpython.rlib import rgc
 from rpython.rlib.rstring import StringBuilder
@@ -237,26 +237,20 @@
         assert 160 <= res <= 165
 
     def test_custom_trace(self):
-        from rpython.rtyper.annlowlevel import llhelper
         from rpython.rtyper.lltypesystem import llmemory
         from rpython.rtyper.lltypesystem.llarena import ArenaError
         #
         S = lltype.GcStruct('S', ('x', llmemory.Address),
-                                 ('y', llmemory.Address), rtti=True)
+                                 ('y', llmemory.Address))
         T = lltype.GcStruct('T', ('z', lltype.Signed))
         offset_of_x = llmemory.offsetof(S, 'x')
-        def customtrace(obj, prev):
-            if not prev:
-                return obj + offset_of_x
-            else:
-                return llmemory.NULL
-        CUSTOMTRACEFUNC = lltype.FuncType([llmemory.Address, llmemory.Address],
-                                          llmemory.Address)
-        customtraceptr = llhelper(lltype.Ptr(CUSTOMTRACEFUNC), customtrace)
-        lltype.attachRuntimeTypeInfo(S, customtraceptr=customtraceptr)
+        def customtrace(gc, obj, callback, arg):
+            gc._trace_callback(callback, arg, obj + offset_of_x)
+        lambda_customtrace = lambda: customtrace
         #
         for attrname in ['x', 'y']:
             def setup():
+                rgc.register_custom_trace_hook(S, lambda_customtrace)
                 s1 = lltype.malloc(S)
                 tx = lltype.malloc(T)
                 tx.z = 42
@@ -762,6 +756,23 @@
             assert rgc.get_gcflag_extra(a1) == False
             assert rgc.get_gcflag_extra(a2) == False
         self.interpret(fn, [])
+    
+    def test_register_custom_trace_hook(self):
+        S = lltype.GcStruct('S', ('x', lltype.Signed))
+        called = []
+
+        def trace_hook(gc, obj, callback, arg):
+            called.append("called")
+        lambda_trace_hook = lambda: trace_hook
+
+        def f():
+            rgc.register_custom_trace_hook(S, lambda_trace_hook)
+            s = lltype.malloc(S)
+            rgc.collect()
+            keepalive_until_here(s)
+
+        self.interpret(f, [])
+        assert called # not empty, can contain more than one item
 
     def test_pinning(self):
         def fn(n):
diff --git a/rpython/memory/test/test_transformed_gc.py b/rpython/memory/test/test_transformed_gc.py
--- a/rpython/memory/test/test_transformed_gc.py
+++ b/rpython/memory/test/test_transformed_gc.py
@@ -14,7 +14,7 @@
 from rpython.conftest import option
 from rpython.rlib.rstring import StringBuilder
 from rpython.rlib.rarithmetic import LONG_BIT
-import pdb
+
 
 WORD = LONG_BIT // 8
 
@@ -385,26 +385,20 @@
         assert 160 <= res <= 165
 
     def define_custom_trace(cls):
-        from rpython.rtyper.annlowlevel import llhelper
-        from rpython.rtyper.lltypesystem import llmemory
         #
-        S = lltype.GcStruct('S', ('x', llmemory.Address), rtti=True)
+        S = lltype.GcStruct('S', ('x', llmemory.Address))
         T = lltype.GcStruct('T', ('z', lltype.Signed))
         offset_of_x = llmemory.offsetof(S, 'x')
-        def customtrace(obj, prev):
-            if not prev:
-                return obj + offset_of_x
-            else:
-                return llmemory.NULL
-        CUSTOMTRACEFUNC = lltype.FuncType([llmemory.Address, llmemory.Address],
-                                          llmemory.Address)
-        customtraceptr = llhelper(lltype.Ptr(CUSTOMTRACEFUNC), customtrace)
-        lltype.attachRuntimeTypeInfo(S, customtraceptr=customtraceptr)
+        def customtrace(gc, obj, callback, arg):
+            gc._trace_callback(callback, arg, obj + offset_of_x)
+        lambda_customtrace = lambda: customtrace
+
         #
         def setup():
-            s1 = lltype.malloc(S)
+            rgc.register_custom_trace_hook(S, lambda_customtrace)
             tx = lltype.malloc(T)
             tx.z = 4243
+            s1 = lltype.malloc(S)
             s1.x = llmemory.cast_ptr_to_adr(tx)
             return s1
         def f():
diff --git a/rpython/rlib/_stacklet_asmgcc.py b/rpython/rlib/_stacklet_asmgcc.py
--- a/rpython/rlib/_stacklet_asmgcc.py
+++ b/rpython/rlib/_stacklet_asmgcc.py
@@ -1,4 +1,6 @@
 from rpython.rlib.debug import ll_assert
+from rpython.rlib import rgc
+from rpython.rlib.objectmodel import specialize
 from rpython.rtyper.lltypesystem import lltype, llmemory, rffi
 from rpython.rtyper.lltypesystem.lloperation import llop
 from rpython.rtyper.annlowlevel import llhelper, MixLevelHelperAnnotator
@@ -11,6 +13,10 @@
 _stackletrootwalker = None
 
 def get_stackletrootwalker():
+    # XXX this is too complicated now; we don't need a StackletRootWalker
+    # instance to store global state.  We could rewrite it all in one big
+    # function.  We don't care enough for now.
+
     # lazily called, to make the following imports lazy
     global _stackletrootwalker
     if _stackletrootwalker is not None:
@@ -25,8 +31,6 @@
     class StackletRootWalker(object):
         _alloc_flavor_ = "raw"
 
-        enumerating = False
-
         def setup(self, obj):
             # initialization: read the SUSPSTACK object
             p = llmemory.cast_adr_to_ptr(obj, lltype.Ptr(SUSPSTACK))
@@ -66,7 +70,8 @@
                 self.fill_initial_frame(self.curframe, anchor)
                 return True
 
-        def next(self, obj, prev):
+        @specialize.arg(3)
+        def customtrace(self, gc, obj, callback, arg):
             #
             # Pointers to the stack can be "translated" or not:
             #
@@ -79,29 +84,20 @@
             # Note that 'curframe' contains non-translated pointers, and
             # of course the stack itself is full of non-translated pointers.
             #
+            if not self.setup(obj):
+                return
+
             while True:
-                if not self.enumerating:
-                    if not prev:
-                        if not self.setup(obj):      # one-time initialization
-                            return llmemory.NULL
-                        prev = obj   # random value, but non-NULL
-                    callee = self.curframe
-                    retaddraddr = self.translateptr(callee.frame_address)
-                    retaddr = retaddraddr.address[0]
-                    ebp_in_caller = callee.regs_stored_at[INDEX_OF_EBP]
-                    ebp_in_caller = self.translateptr(ebp_in_caller)
-                    ebp_in_caller = ebp_in_caller.address[0]
-                    basewalker.locate_caller_based_on_retaddr(retaddr,
-                                                              ebp_in_caller)
-                    self.enumerating = True
-                else:
-                    callee = self.curframe
-                    ebp_in_caller = callee.regs_stored_at[INDEX_OF_EBP]
-                    ebp_in_caller = self.translateptr(ebp_in_caller)
-                    ebp_in_caller = ebp_in_caller.address[0]
-                #
-                # not really a loop, but kept this way for similarity
-                # with asmgcroot:
+                callee = self.curframe
+                retaddraddr = self.translateptr(callee.frame_address)
+                retaddr = retaddraddr.address[0]
+                ebp_in_caller = callee.regs_stored_at[INDEX_OF_EBP]
+                ebp_in_caller = self.translateptr(ebp_in_caller)
+                ebp_in_caller = ebp_in_caller.address[0]
+                basewalker.locate_caller_based_on_retaddr(retaddr,
+                                                          ebp_in_caller)
+
+                # see asmgcroot for similarity:
                 while True:
                     location = basewalker._shape_decompressor.next()
                     if location == 0:
@@ -109,9 +105,9 @@
                     addr = basewalker.getlocation(callee, ebp_in_caller,
                                                   location)
                     # yield the translated addr of the next GCREF in the stack
-                    return self.translateptr(addr)
-                #
-                self.enumerating = False
+                    addr = self.translateptr(addr)
+                    gc._trace_callback(callback, arg, addr)
+
                 caller = self.otherframe
                 reg = CALLEE_SAVED_REGS - 1
                 while reg >= 0:
@@ -129,7 +125,7 @@
                 if caller.frame_address == llmemory.NULL:
                     # completely done with this piece of stack
                     if not self.fetch_next_stack_piece():
-                        return llmemory.NULL
+                        return
                     continue
                 #
                 self.otherframe = callee
@@ -154,9 +150,10 @@
     lltype.attachRuntimeTypeInfo(SUSPSTACK, destrptr=destrptr)
 
 
-def customtrace(obj, prev):
+def customtrace(gc, obj, callback, arg):
     stackletrootwalker = get_stackletrootwalker()
-    return stackletrootwalker.next(obj, prev)
+    stackletrootwalker.customtrace(gc, obj, callback, arg)
+lambda_customtrace = lambda: customtrace
 
 def suspstack_destructor(suspstack):
     h = suspstack.handle
@@ -170,10 +167,6 @@
                             ('callback_pieces', llmemory.Address),
                             rtti=True)
 NULL_SUSPSTACK = lltype.nullptr(SUSPSTACK)
-CUSTOMTRACEFUNC = lltype.FuncType([llmemory.Address, llmemory.Address],
-                                  llmemory.Address)
-customtraceptr = llhelper(lltype.Ptr(CUSTOMTRACEFUNC), customtrace)
-lltype.attachRuntimeTypeInfo(SUSPSTACK, customtraceptr=customtraceptr)
 
 ASM_FRAMEDATA_HEAD_PTR = lltype.Ptr(lltype.ForwardReference())
 ASM_FRAMEDATA_HEAD_PTR.TO.become(lltype.Struct('ASM_FRAMEDATA_HEAD',
@@ -263,6 +256,7 @@
         self.runfn = callback
         self.arg = arg
         # make a fresh new clean SUSPSTACK
+        rgc.register_custom_trace_hook(SUSPSTACK, lambda_customtrace)
         newsuspstack = lltype.malloc(SUSPSTACK)
         newsuspstack.handle = _c.null_handle
         self.suspstack = newsuspstack
diff --git a/rpython/rlib/rgc.py b/rpython/rlib/rgc.py
--- a/rpython/rlib/rgc.py
+++ b/rpython/rlib/rgc.py
@@ -643,3 +643,22 @@
 
 def lltype_is_gc(TP):
     return getattr(getattr(TP, "TO", None), "_gckind", "?") == 'gc'
+
+def register_custom_trace_hook(TP, lambda_func):
+    """ This function does not do anything, but called from any annotated
+    place, will tell that "func" is used to trace GC roots inside any instance
+    of the type TP.  The func must be specified as "lambda: func" in this
+    call, for internal reasons.
+    """
+
+class RegisterGcTraceEntry(ExtRegistryEntry):
+    _about_ = register_custom_trace_hook
+
+    def compute_result_annotation(self, *args_s):
+        pass
+
+    def specialize_call(self, hop):
+        TP = hop.args_s[0].const
+        lambda_func = hop.args_s[1].const
+        hop.exception_cannot_occur()
+        hop.rtyper.custom_trace_funcs.append((TP, lambda_func()))
diff --git a/rpython/rlib/test/test_libffi.py b/rpython/rlib/test/test_libffi.py
--- a/rpython/rlib/test/test_libffi.py
+++ b/rpython/rlib/test/test_libffi.py
@@ -576,7 +576,9 @@
             }
             """
             libfoo = self.get_libfoo()
-            func = (libfoo, 'std_diff_xy', [types.sint, types.signed], types.sint)
+            # __stdcall without a DEF file decorates the name with the number of bytes
+            # that the callee will remove from the call stack
+            func = (libfoo, '_std_diff_xy at 8', [types.sint, types.signed], types.sint)
             try:
                 self.call(func, [50, 8], lltype.Signed)
             except ValueError, e:
@@ -613,7 +615,9 @@
             """
             from rpython.rlib.libffi import WinDLL
             dll = WinDLL(self.libfoo_name)
-            f_by_name = dll.getpointer('BBB_second_ordinal_function' ,[],
+            # __stdcall without a DEF file decorates the name with the number of bytes
+            # that the callee will remove from the call stack
+            f_by_name = dll.getpointer('_BBB_second_ordinal_function at 0' ,[],
                                           types.uint)
             f_by_ordinal = dll.getpointer_by_ordinal(2 ,[], types.uint)
             print dir(f_by_name)
diff --git a/rpython/rlib/test/test_rgc.py b/rpython/rlib/test/test_rgc.py
--- a/rpython/rlib/test/test_rgc.py
+++ b/rpython/rlib/test/test_rgc.py
@@ -228,3 +228,17 @@
     x1 = X()
     n = rgc.get_rpy_memory_usage(rgc.cast_instance_to_gcref(x1))
     assert n >= 8 and n <= 64
+
+def test_register_custom_trace_hook():
+    TP = lltype.GcStruct('X')
+
+    def trace_func():
+        xxx # should not be annotated here
+    lambda_trace_func = lambda: trace_func
+    
+    def f():
+        rgc.register_custom_trace_hook(TP, lambda_trace_func)
+    
+    t, typer, graph = gengraph(f, [])
+
+    assert typer.custom_trace_funcs == [(TP, trace_func)]
diff --git a/rpython/rtyper/annlowlevel.py b/rpython/rtyper/annlowlevel.py
--- a/rpython/rtyper/annlowlevel.py
+++ b/rpython/rtyper/annlowlevel.py
@@ -513,6 +513,13 @@
                                   % (ptr, Class))
     return ptr
 
+ at specialize.arg(0)
+def cast_gcref_to_instance(Class, ptr):
+    """Reverse the hacking done in cast_instance_to_gcref()."""
+    from rpython.rtyper.rclass import OBJECTPTR
+    ptr = lltype.cast_opaque_ptr(OBJECTPTR, ptr)
+    return cast_base_ptr_to_instance(Class, ptr)
+
 class CastBasePtrToInstanceEntry(extregistry.ExtRegistryEntry):
     _about_ = cast_base_ptr_to_instance
 
diff --git a/rpython/rtyper/lltypesystem/lltype.py b/rpython/rtyper/lltypesystem/lltype.py
--- a/rpython/rtyper/lltypesystem/lltype.py
+++ b/rpython/rtyper/lltypesystem/lltype.py
@@ -383,8 +383,7 @@
                                                 about=self)._obj
         Struct._install_extras(self, **kwds)
 
-    def _attach_runtime_type_info_funcptr(self, funcptr, destrptr,
-                                          customtraceptr):
+    def _attach_runtime_type_info_funcptr(self, funcptr, destrptr):
         if self._runtime_type_info is None:
             raise TypeError("attachRuntimeTypeInfo: %r must have been built "
                             "with the rtti=True argument" % (self,))
@@ -408,18 +407,6 @@
                 raise TypeError("expected a destructor function "
                                 "implementation, got: %s" % destrptr)
             self._runtime_type_info.destructor_funcptr = destrptr
-        if customtraceptr is not None:
-            from rpython.rtyper.lltypesystem import llmemory
-            T = typeOf(customtraceptr)
-            if (not isinstance(T, Ptr) or
-                not isinstance(T.TO, FuncType) or
-                len(T.TO.ARGS) != 2 or
-                T.TO.RESULT != llmemory.Address or
-                T.TO.ARGS[0] != llmemory.Address or
-                T.TO.ARGS[1] != llmemory.Address):
-                raise TypeError("expected a custom trace function "
-                                "implementation, got: %s" % customtraceptr)
-            self._runtime_type_info.custom_trace_funcptr = customtraceptr
 
 class GcStruct(RttiStruct):
     _gckind = 'gc'
@@ -2288,12 +2275,10 @@
     return SomePtr(ll_ptrtype=PtrT.const)
 
 
-def attachRuntimeTypeInfo(GCSTRUCT, funcptr=None, destrptr=None,
-                          customtraceptr=None):
+def attachRuntimeTypeInfo(GCSTRUCT, funcptr=None, destrptr=None):
     if not isinstance(GCSTRUCT, RttiStruct):
         raise TypeError("expected a RttiStruct: %s" % GCSTRUCT)
-    GCSTRUCT._attach_runtime_type_info_funcptr(funcptr, destrptr,
-                                               customtraceptr)
+    GCSTRUCT._attach_runtime_type_info_funcptr(funcptr, destrptr)
     return _ptr(Ptr(RuntimeTypeInfo), GCSTRUCT._runtime_type_info)
 
 def getRuntimeTypeInfo(GCSTRUCT):
diff --git a/rpython/rtyper/lltypesystem/opimpl.py b/rpython/rtyper/lltypesystem/opimpl.py
--- a/rpython/rtyper/lltypesystem/opimpl.py
+++ b/rpython/rtyper/lltypesystem/opimpl.py
@@ -82,13 +82,11 @@
         else:
             def op_function(x, y):
                 if not isinstance(x, argtype):
-                    if not (isinstance(x, AddressAsInt) and argtype is int):
-                        raise TypeError("%r arg 1 must be %s, got %r instead"% (
-                            fullopname, typname, type(x).__name__))
+                    raise TypeError("%r arg 1 must be %s, got %r instead"% (
+                        fullopname, typname, type(x).__name__))
                 if not isinstance(y, argtype):
-                    if not (isinstance(y, AddressAsInt) and argtype is int):
-                        raise TypeError("%r arg 2 must be %s, got %r instead"% (
-                            fullopname, typname, type(y).__name__))
+                    raise TypeError("%r arg 2 must be %s, got %r instead"% (
+                        fullopname, typname, type(y).__name__))
                 return adjust_result(func(x, y))
 
     return func_with_new_name(op_function, 'op_' + fullopname)
@@ -104,6 +102,19 @@
             lltype.typeOf(adr),))
 
 
+def op_int_eq(x, y):
+    if not isinstance(x, (int, long)):
+        from rpython.rtyper.lltypesystem import llgroup
+        assert isinstance(x, llgroup.CombinedSymbolic), (
+            "'int_eq' arg 1 must be int-like, got %r instead" % (
+                type(x).__name__,))
+    if not isinstance(y, (int, long)):
+        from rpython.rtyper.lltypesystem import llgroup
+        assert isinstance(y, llgroup.CombinedSymbolic), (
+            "'int_eq' arg 2 must be int-like, got %r instead" % (
+                type(y).__name__,))
+    return x == y
+
 def op_ptr_eq(ptr1, ptr2):
     checkptr(ptr1)
     checkptr(ptr2)
diff --git a/rpython/rtyper/rtyper.py b/rpython/rtyper/rtyper.py
--- a/rpython/rtyper/rtyper.py
+++ b/rpython/rtyper/rtyper.py
@@ -60,6 +60,7 @@
         # make the primitive_to_repr constant mapping
         self.primitive_to_repr = {}
         self.exceptiondata = ExceptionData(self)
+        self.custom_trace_funcs = []
 
         try:
             self.seed = int(os.getenv('RTYPERSEED'))
@@ -645,7 +646,7 @@
             raise TyperError("runtime type info function %r returns %r, "
                              "excepted Ptr(RuntimeTypeInfo)" % (func, s))
         funcptr = self.getcallable(graph)
-        attachRuntimeTypeInfo(GCSTRUCT, funcptr, destrptr, None)
+        attachRuntimeTypeInfo(GCSTRUCT, funcptr, destrptr)
 
 # register operations from annotation model
 RPythonTyper._registeroperations(unaryop.UNARY_OPERATIONS, binaryop.BINARY_OPERATIONS)
diff --git a/rpython/rtyper/test/test_annlowlevel.py b/rpython/rtyper/test/test_annlowlevel.py
--- a/rpython/rtyper/test/test_annlowlevel.py
+++ b/rpython/rtyper/test/test_annlowlevel.py
@@ -4,7 +4,7 @@
 
 from rpython.rtyper.test.tool import BaseRtypingTest
 from rpython.rtyper.lltypesystem.rstr import mallocstr, mallocunicode
-from rpython.rtyper.lltypesystem import lltype
+from rpython.rtyper.lltypesystem import lltype, llmemory
 from rpython.rtyper.annlowlevel import hlstr, llstr
 from rpython.rtyper.annlowlevel import hlunicode, llunicode
 from rpython.rtyper import annlowlevel
@@ -73,6 +73,15 @@
         y = annlowlevel.cast_base_ptr_to_instance(X, ptr)
         assert y is x
 
+    def test_cast_instance_to_gcref(self):
+        class X(object):
+            pass
+        x = X()
+        ptr = annlowlevel.cast_instance_to_gcref(x)
+        assert lltype.typeOf(ptr) == llmemory.GCREF
+        y = annlowlevel.cast_gcref_to_instance(X, ptr)
+        assert y is x
+
     def test_delayedptr(self):
         FUNCTYPE = lltype.FuncType([], lltype.Signed)
         name = "delayed!myfunc"
diff --git a/rpython/translator/c/test/test_newgc.py b/rpython/translator/c/test/test_newgc.py
--- a/rpython/translator/c/test/test_newgc.py
+++ b/rpython/translator/c/test/test_newgc.py
@@ -443,19 +443,14 @@
     def define_custom_trace(cls):
         from rpython.rtyper.annlowlevel import llhelper
         #
-        S = lltype.GcStruct('S', ('x', llmemory.Address), rtti=True)
+        S = lltype.GcStruct('S', ('x', llmemory.Address))
         offset_of_x = llmemory.offsetof(S, 'x')
-        def customtrace(obj, prev):
-            if not prev:
-                return obj + offset_of_x
-            else:
-                return llmemory.NULL
-        CUSTOMTRACEFUNC = lltype.FuncType([llmemory.Address, llmemory.Address],
-                                          llmemory.Address)
-        customtraceptr = llhelper(lltype.Ptr(CUSTOMTRACEFUNC), customtrace)
-        lltype.attachRuntimeTypeInfo(S, customtraceptr=customtraceptr)
+        def customtrace(gc, obj, callback, arg):
+            gc._trace_callback(callback, arg, obj + offset_of_x)
+        lambda_customtrace = lambda: customtrace
         #
         def setup():
+            rgc.register_custom_trace_hook(S, lambda_customtrace)
             s = lltype.nullptr(S)
             for i in range(10000):
                 t = lltype.malloc(S)

From noreply at buildbot.pypy.org  Thu Nov 13 15:49:18 2014
From: noreply at buildbot.pypy.org (anton_gulenko)
Date: Thu, 13 Nov 2014 15:49:18 +0100 (CET)
Subject: [pypy-commit] lang-smalltalk rstrategies: More info in README
Message-ID: <20141113144918.A7AEF1C0548@cobra.cs.uni-duesseldorf.de>

Author: Anton Gulenko 
Branch: rstrategies
Changeset: r1061:e2085c645395
Date: 2014-11-13 15:11 +0100
http://bitbucket.org/pypy/lang-smalltalk/changeset/e2085c645395/

Log:	More info in README

diff --git a/README.md b/README.md
--- a/README.md
+++ b/README.md
@@ -1,69 +1,77 @@
-Spy
+RSqueak
 =========
 
-A Squeak VM written in RPython, called "SPy VM".
+A Squeak VM written in RPython.
 
 Setup
 ----
+
 ### Required Projects
 You need three repositories: 
-* This one
-* pypy/pypy
-* pypy/rsdl
+* [This one](https://bitbucket.org/pypy/lang-smalltalk)
+* [pypy/pypy](https://bitbucket.org/pypy/pypy)
+* [pypy/rsdl](https://bitbucket.org/pypy/rsdl)
+    * Alternatively download RSDL package from [PYPI](https://pypi.python.org/pypi/rsdl)
+    * Then unpack and install it using ```python setup.py install``` or ```pypy setup.py install```
 
 ### Required packages
-You need the following packages on your OS. Install with your favorite package
-manager:
-* pypy (For faster translation of the SPY VM)
+You need the following packages on your OS. Install with your favorite package manager:
+* pypy
+    * For faster translation of the SPY VM. Alternatively use default Python.
 * libsdl-dev
+* libffi-dev
 
 ### Adjusting the PYTHONPATH
-In order to allow the RPython toolchain to find the rsdl module you have to add
-the rsdl folder to the PYTHONPATH. Note that you have to add the rsdl subfolder
-of the rsdl repository to the PYTHONPATH.
+In order to allow the RPython toolchain to find the rsdl and pypy packages you have to add the two folders to the PYTHONPATH.
+
+If you are using the rsdl *repository*, you have to add the rsdl subfolder of the rsdl repository to the PYTHONPATH.
 
 ```
-export PYTHONPATH=${PYTHONPATH}:[path to rsdl repository]/rsdl
+export PYTHONPATH=${PYTHONPATH}:.:[path/to/pypy]:[path/to/rsdl]
 ```
 
 ### Setting the SDL Driver
-For testing the basic functionality of the VM it is currently best to disable
-the UI. You can do so by setting the SDL_VIDEODRIVER environment variable to
-dummy.
+For testing the basic functionality of the VM you might want to disable the UI. You can do so by setting the SDL_VIDEODRIVER environment variable to dummy.
+
 ```
 export SDL_VIDEODRIVER=dummy
 ```
 
-### Building
-To build the VM enter the following:
+### Building & Tests
+Execute the following commands inside the main directory of this repository.
+
+To build the VM:
 
 ```
-[path to pypy repository]/rpython/bin/rpython [path to lang-smalltalk
-repository]/targetimageloadingsmalltalk.py
+[path to pypy repository]/rpython/bin/rpython targetimageloadingsmalltalk.py
 ```
 
 To build the VM with enabled just-in-time compiler:
+
 ```
-[path to pypy repository]/rpython/bin/rpython -O jit [path to lang-smalltalk
-repository]/targetimageloadingsmalltalk.py
+[path to pypy repository]/rpython/bin/rpython -Ojit targetimageloadingsmalltalk.py
+```
+
+To run the tests (most do not require building):
+
+```
+[path to pypy repository]/pytest.py [--slow|--quick] spyvm/test
 ```
 
 ### Starting an image
-The build process will produce an executable e.g. called
-targetimageloadingsmalltalk-c. Start it with the following:
+The build process will produce an executable e.g. called targetimageloadingsmalltalk-c. Start it with the following:
+
 ```
 ./targetimageloadingsmalltalk-c images/Squeak4.5-*.image
 ```
 
 Setup for stm-enabled SPY
 ---
-There are two branches integrating the RPython STM into SPY: stm-c4,
-storage-stm-c4. You have to change two things of the setup to build those
-branches.
+
+There are two branches integrating the RPython STM into SPY: stm-c4, storage-stm-c4. You have to change two things of the setup to build those branches.
 
 1. Change your local pypy repository to the stm-c4 branch.
-2. Build using the following command:
+2. Add the ```--gc=stmgc``` when building:
 ```
-[path to pypy repository]/rpython/bin/rpython --gc=stmgc [path to lang-smalltalk
-repository]/targetimageloadingsmalltalk.py
+[path to pypy repository]/rpython/bin/rpython --gc=stmgc targetimageloadingsmalltalk.py
 ```

From noreply at buildbot.pypy.org  Thu Nov 13 15:49:19 2014
From: noreply at buildbot.pypy.org (anton_gulenko)
Date: Thu, 13 Nov 2014 15:49:19 +0100 (CET)
Subject: [pypy-commit] lang-smalltalk rstrategies: Changed comment
Message-ID: <20141113144919.B45581C0548@cobra.cs.uni-duesseldorf.de>

Author: Anton Gulenko 
Branch: rstrategies
Changeset: r1062:f1e3b1f904d8
Date: 2014-11-13 15:12 +0100
http://bitbucket.org/pypy/lang-smalltalk/changeset/f1e3b1f904d8/

Log:	Changed comment

diff --git a/rstrategies.py b/rstrategies.py
--- a/rstrategies.py
+++ b/rstrategies.py
@@ -348,7 +348,7 @@
     def check_can_handle(self, wrapped_value):
         return True
     
-# ============== Mixins for index checking operations ==============
+# ============== Mixins for StrategyWithStorage ==============
 
 class SafeIndexingMixin(object):
     def check_index_store(self, index0):

From noreply at buildbot.pypy.org  Thu Nov 13 15:49:20 2014
From: noreply at buildbot.pypy.org (anton_gulenko)
Date: Thu, 13 Nov 2014 15:49:20 +0100 (CET)
Subject: [pypy-commit] lang-smalltalk rstrategies: Merged default.
Message-ID: <20141113144920.AC3E71C0548@cobra.cs.uni-duesseldorf.de>

Author: Anton Gulenko 
Branch: rstrategies
Changeset: r1063:8f01f05e7dc3
Date: 2014-11-13 15:22 +0100
http://bitbucket.org/pypy/lang-smalltalk/changeset/8f01f05e7dc3/

Log:	Merged default.

diff --git a/README.md b/README.md
--- a/README.md
+++ b/README.md
@@ -17,7 +17,7 @@
 ### Required packages
 You need the following packages on your OS. Install with your favorite package manager:
 * pypy
-    * For faster translation of the SPY VM. Alternatively use default Python.
+    * For faster translation of the RSqueak VM. Alternatively use default Python.
 * libsdl-dev
 * libffi-dev
 
@@ -59,19 +59,31 @@
 ```
 
 ### Starting an image
-The build process will produce an executable e.g. called targetimageloadingsmalltalk-c. Start it with the following:
+The build process will produce an executable called rsqueak.
+The ```image/``` directory contains two images you can open with the following.
+Use ```--help``` to see command line switches.
 
 ```
-./targetimageloadingsmalltalk-c images/Squeak4.5-*.image
+./rsqueak images/mini.image
+./rsqueak images/Squeak4.5-noBitBlt.image
 ```
 
-Setup for stm-enabled SPY
+
+
+
+STM-enabled Rsqueak
+===
+This is a branch of RSqueak which incorporates the RPython STM transformation. Most of the initial code base comes from the results of a project seminar (https://bitbucket.org/amintos/lang-smalltalk). The stmgc-c7 branch is based on this version and the 64bit branch.
+
+Setup for stm-enabled RSqueak
 ---
+You can see the current state of the integration of the RPython STM in our stmgc-c7 branch.
+Beware that you can only build this branch if you have a 64-bit linux. To build this branch you have to setup several things:
 
-There are two branches integrating the RPython STM into SPY: stm-c4, storage-stm-c4. You have to change two things of the setup to build those branches.
+1. Change your local pypy repository to the stm-gc7 branch, commit dd3c06b
+2. Get a clang which has the patches from ([Clang patches](https://bitbucket.org/pypy/stmgc/src/d164a5bcad5e7615b4362b6a1a49d51e2e06de0c/c7/llvmfix/?at=default)). If you have a Debian-based OS you can use the following package: [llvm-pypystm](https://launchpad.net/~malte.swart/+archive/ubuntu/llvm-pypystm).
 
-1. Change your local pypy repository to the stm-c4 branch.
-2. Add the ```--gc=stmgc``` when building:
+To build, use the following command:
 ```
 [path to pypy repository]/rpython/bin/rpython --gc=stmgc targetimageloadingsmalltalk.py
 ```

From noreply at buildbot.pypy.org  Thu Nov 13 15:49:21 2014
From: noreply at buildbot.pypy.org (anton_gulenko)
Date: Thu, 13 Nov 2014 15:49:21 +0100 (CET)
Subject: [pypy-commit] lang-smalltalk rstrategies: Changed --help output.
Message-ID: <20141113144921.AD4FC1C0548@cobra.cs.uni-duesseldorf.de>

Author: Anton Gulenko 
Branch: rstrategies
Changeset: r1064:11d2244a8410
Date: 2014-11-13 15:48 +0100
http://bitbucket.org/pypy/lang-smalltalk/changeset/11d2244a8410/

Log:	Changed --help output.

diff --git a/targetimageloadingsmalltalk.py b/targetimageloadingsmalltalk.py
--- a/targetimageloadingsmalltalk.py
+++ b/targetimageloadingsmalltalk.py
@@ -11,19 +11,22 @@
 
           Execution mode:
             (no flags)             - Image will be normally opened.
-            -r|--run         - Code will be compiled and executed, result printed.
-            -m|--method  - Selector will be sent to a SmallInteger, result printed.
+            -r|--run         - Code will be compiled and executed in headless mode, result printed.
+            -m|--method  - Selector will be sent to a SmallInteger in headless mode, result printed.
             -h|--help              - Output this and exit.
 
           Execution parameters:
-            -n|--num  - Only with -m or -r, SmallInteger to be used as receiver (default: nil).
-            -a|--arg  - Only with -m, will be used as single String argument.
-            -P|--process   - Only with -m or -r, create a high-priority Process for the context.
-                             The images last active Process will be started first.
-                             By default, run in headless mode. This will ignore the active process
-                             in the image and execute the context directly. The image window will
-                             probably not open. Good for benchmarking.
-            -u             - Only with -m or -r, try to stop UI-process at startup. Can help benchmarking.
+            -n|--num  - Only with -m or -r. SmallInteger to be used as receiver (default: nil).
+            -a|--arg  - Only with -m. Will be used as single String argument.
+            -P|--process   - Only with -m or -r. Disable headless mode.
+                             A high-priority Process for the new context will be created.
+                             The last active Process in the image will be started,
+                             but then quickly switch to the new synthetic high-prio Process.
+                             By default, in headless mode, the active process in the image will be ignored,
+                             and the image window will probably not open (good for benchmarking).
+                             Headless mode also influences error reporting.
+                             Without -r or -m, headless mode is always disabled.
+            -u             - Only with -m or -r. Try to stop UI-process at startup. Can help benchmarking.
 
           Other parameters:
             -j|--jit  - jitargs will be passed to the jit configuration.

From noreply at buildbot.pypy.org  Thu Nov 13 15:50:42 2014
From: noreply at buildbot.pypy.org (anton_gulenko)
Date: Thu, 13 Nov 2014 15:50:42 +0100 (CET)
Subject: [pypy-commit] lang-smalltalk strategies-allocRemoval: 
Message-ID: <20141113145042.BA2C91C0548@cobra.cs.uni-duesseldorf.de>

Author: Anton Gulenko 
Branch: strategies-allocRemoval
Changeset: r1065:87fb3f5ce164
Date: 2014-07-28 10:26 +0200
http://bitbucket.org/pypy/lang-smalltalk/changeset/87fb3f5ce164/

Log:	


From noreply at buildbot.pypy.org  Thu Nov 13 15:50:43 2014
From: noreply at buildbot.pypy.org (anton_gulenko)
Date: Thu, 13 Nov 2014 15:50:43 +0100 (CET)
Subject: [pypy-commit] lang-smalltalk refactoring-virtualizable: 
Message-ID: <20141113145043.CE1F51C0548@cobra.cs.uni-duesseldorf.de>

Author: Anton Gulenko 
Branch: refactoring-virtualizable
Changeset: r1066:acc0c78ea21e
Date: 2014-07-28 10:29 +0200
http://bitbucket.org/pypy/lang-smalltalk/changeset/acc0c78ea21e/

Log:	


From noreply at buildbot.pypy.org  Thu Nov 13 15:51:05 2014
From: noreply at buildbot.pypy.org (anton_gulenko)
Date: Thu, 13 Nov 2014 15:51:05 +0100 (CET)
Subject: [pypy-commit] lang-smalltalk default: Merged rstrategies.
Message-ID: <20141113145105.0B1DF1C3140@cobra.cs.uni-duesseldorf.de>

Author: Anton Gulenko 
Branch: 
Changeset: r1067:e43179221228
Date: 2014-11-13 15:49 +0100
http://bitbucket.org/pypy/lang-smalltalk/changeset/e43179221228/

Log:	Merged rstrategies.

diff too long, truncating to 2000 out of 757036 lines

diff --git a/.hgignore b/.hgignore
--- a/.hgignore
+++ b/.hgignore
@@ -12,5 +12,6 @@
 versions
 coglinux
 *.orig
+SDL.dll
 spy-*.log
-SDL.dll
+st-*
diff --git a/.hgtags b/.hgtags
new file mode 100644
--- /dev/null
+++ b/.hgtags
@@ -0,0 +1,1 @@
+91b2b3229ec1403d37e8d34836249a897b9a1ead strategies-refactored
diff --git a/README.md b/README.md
--- a/README.md
+++ b/README.md
@@ -1,75 +1,89 @@
-Spy
-=========
-
-A Squeak VM written in RPython, called "SPy VM".
-
-Setup
-----
-### Required Projects
-You need three repositories: 
-
-* This one
-* pypy/pypy
-* pypy/rsdl
-
-### Required packages
-You need the following packages on your OS. Install with your favorite package
-manager:
-* pypy (For faster translation of the SPY VM)
-* libsdl-dev
-
-### Adjusting the PYTHONPATH
-In order to allow the RPython toolchain to find the rsdl module you have to add
-the rsdl folder to the PYTHONPATH. Note that you have to add the rsdl subfolder
-of the rsdl repository to the PYTHONPATH.
-
-```
-export PYTHONPATH=${PYTHONPATH}:[path to rsdl repository]/rsdl
-```
-
-### Setting the SDL Driver
-For testing the basic functionality of the VM it is currently best to disable
-the UI. You can do so by setting the SDL_VIDEODRIVER environment variable to
-dummy.
-```
-export SDL_VIDEODRIVER=dummy
-```
-
-### Building
-To build the VM enter the following:
-
-```
-[path to pypy repository]/rpython/bin/rpython [path to lang-smalltalk
-repository]/targetimageloadingsmalltalk.py
-```
-
-To build the VM with enabled just-in-time compiler:
-```
-[path to pypy repository]/rpython/bin/rpython -O jit [path to lang-smalltalk
-repository]/targetimageloadingsmalltalk.py
-```
-
-### Starting an image
-The build process will produce an executable e.g. called
-targetimageloadingsmalltalk-c. Start it with the following:
-```
-./targetimageloadingsmalltalk-c images/Squeak4.5-*.image
-```
-
-STM-enabled SPY
-===
-This is a branch of SPY which incorporates the RPython STM transformation. Most of the initial code base comes from the results of a project seminar (https://bitbucket.org/amintos/lang-smalltalk). The stmgc-c7 branch is based on this version and the 64bit branch.
-
-Setup for stm-enabled SPY
----
-You can see the current state of the integration of the RPython STM in our stmgc-c7 branch.
-Beware that you can only build this branch if you have a 64-bit linux. To build this branch you have to setup several things:
-
-1. Change your local pypy repository to the stm-gc7 branch, commit dd3c06b
-2. Get a clang which has the patches from ([Clang patches](https://bitbucket.org/pypy/stmgc/src/d164a5bcad5e7615b4362b6a1a49d51e2e06de0c/c7/llvmfix/?at=default)). If you have a Debian-based OS you can use the following package: https://launchpad.net/~malte.swart/+archive/ubuntu/llvm-pypystm
-
-To build, use the following command:
-```
-[path to pypy repository]/rpython/bin/rpython --gc=stmgc [path to lang-smalltalk
-repository]/targetimageloadingsmalltalk.py
-```
\ No newline at end of file
+RSqueak
+=========
+
+A Squeak VM written in RPython.
+
+Setup
+----
+
+### Required Projects
+You need three repositories: 
+* [This one](https://bitbucket.org/pypy/lang-smalltalk)
+* [pypy/pypy](https://bitbucket.org/pypy/pypy)
+* [pypy/rsdl](https://bitbucket.org/pypy/rsdl)
+    * Alternatively download RSDL package from [PYPI](https://pypi.python.org/pypi/rsdl)
+    * Then unpack and install it using ```python setup.py install``` or ```pypy setup.py install```
+
+### Required packages
+You need the following packages on your OS. Install with your favorite package manager:
+* pypy
+    * For faster translation of the RSqueak VM. Alternatively use default Python.
+* libsdl-dev
+* libffi-dev
+
+### Adjusting the PYTHONPATH
+In order to allow the RPython toolchain to find the rsdl and pypy packages you have to add the two folders to the PYTHONPATH.
+
+If you are using the rsdl *repository*, you have to add the rsdl subfolder of the rsdl repository to the PYTHONPATH.
+
+```
+export PYTHONPATH=${PYTHONPATH}:.:[path/to/pypy]:[path/to/rsdl]
+```
+
+### Setting the SDL Driver
+For testing the basic functionality of the VM you might want to disable the UI. You can do so by setting the SDL_VIDEODRIVER environment variable to dummy.
+
+```
+export SDL_VIDEODRIVER=dummy
+```
+
+### Building & Tests
+Execute the following commands inside the main directory of this repository.
+
+To build the VM:
+
+```
+[path to pypy repository]/rpython/bin/rpython targetimageloadingsmalltalk.py
+```
+
+To build the VM with enabled just-in-time compiler:
+
+```
+[path to pypy repository]/rpython/bin/rpython -Ojit targetimageloadingsmalltalk.py
+```
+
+To run the tests (most do not require building):
+
+```
+[path to pypy repository]/pytest.py [--slow|--quick] spyvm/test
+```
+
+### Starting an image
+The build process will produce an executable called rsqueak.
+The ```image/``` directory contains two images you can open with the following.
+Use ```--help``` to see command line switches.
+
+```
+./rsqueak images/mini.image
+./rsqueak images/Squeak4.5-noBitBlt.image
+```
+
+
+
+
+STM-enabled Rsqueak
+===
+This is a branch of RSqueak which incorporates the RPython STM transformation. Most of the initial code base comes from the results of a project seminar (https://bitbucket.org/amintos/lang-smalltalk). The stmgc-c7 branch is based on this version and the 64bit branch.
+
+Setup for stm-enabled RSqueak
+---
+You can see the current state of the integration of the RPython STM in our stmgc-c7 branch.
+Beware that you can only build this branch if you have a 64-bit linux. To build this branch you have to setup several things:
+
+1. Change your local pypy repository to the stm-gc7 branch, commit dd3c06b
+2. Get a clang which has the patches from ([Clang patches](https://bitbucket.org/pypy/stmgc/src/d164a5bcad5e7615b4362b6a1a49d51e2e06de0c/c7/llvmfix/?at=default)). If you have a Debian-based OS you can use the following package: [llvm-pypystm](https://launchpad.net/~malte.swart/+archive/ubuntu/llvm-pypystm).
+
+To build, use the following command:
+```
+[path to pypy repository]/rpython/bin/rpython --gc=stmgc targetimageloadingsmalltalk.py
+```
diff --git a/images/Squeak4.5-12568.changes b/images/Squeak4.5-12568.changes
deleted file mode 100644
--- a/images/Squeak4.5-12568.changes
+++ /dev/null
@@ -1,39 +0,0 @@
-'From Squeak4.1 of 17 April 2010 [latest update: #9957] on 17 April 2010 at 5:22:05 pm'!


----STARTUP----{17 April 2010 . 5:21:54 pm} as C:\Squeak\4.0\4.1-final\Squeak4.1.image!


Smalltalk appendChangesTo: 'SqueakV41.sources'.!

----QUIT----{17 April 2010 . 5:22:11 pm} Squeak4.1.image priorSource: 89!

----STARTUP----{24 May 2010 . 8:07:26 pm} as C:\Squeak\4.2\Squeak4.1.image!


----SNAPSHOT----{24 May 2010 . 8:08:14 pm} Squeak4.2.image priorSource: 229!
!HashedCollection commentStamp: 'ul 4/12/2010 22:37' prior: 0!
I am an abstract collection of objects that implement hash and equality in a consitent way. This means that whenever two objects are equal, their hashes have to be equal too. If two objects are equal then I can only store one of them. Hashes are expected to be integers (preferably SmallIntegers). I also expect that the objects contained by me do not change their hashes. If that happens, hash invariants have to be re-established, which can be done by #rehash.

Since I'm abstract, no instances of me should exist. My subclasses should implement #scanFor:, #fixCollisionsFrom: and #noCheckNoGrowFillFrom:.

Instance Variables
	array:		 (typically Array or WeakArray)
	tally:		 (non-negative)

array
	- An array whose size is a prime number, it's non-nil elements are the elements of the collection, and whose nil elements are empty slots. There is always at least one nil. In fact I try to keep my "load" at 75% or less so that hashing will work well.

tally
	- The number of elements in the collection. The array size is always greater than this.

Implementation details:
I implement a hash table which uses open addressing with linear probing as the method of collision resolution. Searching for an element or a free slot for an element is done by #scanFor: which should return the index of the slot in array corresponding to it's argument. When an element is removed #fixCollisionsFrom: should rehash all elements in array between the original index of the removed element, wrapping around after the last slot until reaching an empty slot. My maximum load factor (75%) is hardcoded in #atNewIndex:put:, so it can only be changed by overriding that method. When my load factor reaches this limit I replace my array with a larger one (see #grow) ensuring that my load factor will be less than or equal to 50%. The new array is filled by #noCheckNoGrowFillFrom: which should use #scanForEmptySlotFor: instead of #scanFor: for better performance. I do not shrink.
!
!WeakKeyDictionary methodsFor: 'private' stamp: 'ul 4/12/2010 22:59'!
compact
	"Reduce the size of array so that the load factor will be ~75%."
	
	| newCapacity |
	newCapacity := self class goodPrimeAtLeast: self slowSize * 4 // 3.
	self growTo: newCapacity! !
!Collection methodsFor: 'adding' stamp: 'ul 4/12/2010 22:33' prior: 18816249!
add: newObject withOccurrences: anInteger
	"Add newObject anInteger times to the receiver. Do nothing if anInteger is less than one. Answer newObject."

	anInteger timesRepeat: [self add: newObject].
	^ newObject! !
!HashedCollection class methodsFor: 'initialize-release' stamp: 'ul 4/12/2010 23:49'!
compactAll
	"HashedCollection compactAll"	
		
	self allSubclassesDo: #compactAllInstances! !
!HashedCollection class methodsFor: 'initialize-release' stamp: 'ul 4/12/2010 23:49'!
compactAllInstances
	"Do not use #allInstancesDo: because compact may create new instances."

	self allInstances do: #compact! !
!HashedCollection class methodsFor: 'sizing' stamp: 'ul 4/7/2010 00:17' prior: 55063414!
goodPrimes
	"Answer a sorted array of prime numbers less than one billion that make good
	hash table sizes. Should be expanded as needed.  See comments below code"
	
	^#(
		5 11 17 23 31 43 59 79 107 149 199 269 359 479 641 857 1151 1549 2069
		2237 2423 2617 2797 2999 3167 3359 3539 3727 3911
		4441 4787 5119 5471 5801 6143 6521 6827 7177 7517 7853
		8783 9601 10243 10867 11549 12239 12919 13679 14293 15013 15731
		17569 19051 20443 21767 23159 24611 25847 27397 28571 30047 31397
		35771 38201 40841 43973 46633 48989 51631 54371 57349 60139 62969
		70589 76091 80347 85843 90697 95791 101051 106261 111143 115777 120691 126311
		140863 150523 160969 170557 181243 190717 201653 211891 221251 232591 242873 251443
		282089 300869 321949 341227 362353 383681 401411 422927 443231 464951 482033 504011
		562621 605779 647659 681607 723623 763307 808261 844709 886163 926623 967229 1014617
		1121987 1201469 1268789 1345651 1429531 1492177 1577839 1651547 1722601 1800377 1878623 1942141 2028401
		2242727 2399581 2559173 2686813 2836357 3005579 3144971 3283993 3460133 3582923 3757093 3903769 4061261
		4455361 4783837 5068529 5418079 5680243 6000023 6292981 6611497 6884641 7211599 7514189 7798313 8077189
		9031853 9612721 10226107 10745291 11338417 11939203 12567671 13212697 13816333 14337529 14938571 15595673 16147291
		17851577 18993941 20180239 21228533 22375079 23450491 24635579 25683871 26850101 27921689 29090911 30153841 31292507 32467307
		35817611 37983761 40234253 42457253 44750177 46957969 49175831 51442639 53726417 55954637 58126987 60365939 62666977 64826669
		71582779 76039231 80534381 84995153 89500331 93956777 98470819 102879613 107400389 111856841 116365721 120819287 125246581 129732203
		143163379 152076289 161031319 169981667 179000669 187913573 196826447 205826729 214748357 223713691 232679021 241591901 250504801 259470131
		285162679 301939921 318717121 335494331 352271573 369148753 385926017 402603193 419480419 436157621 453034849 469712051 486589307 503366497 520043707 
		570475349 603929813 637584271 671138659 704693081 738247541 771801929 805356457 838910803 872365267 905919671 939574117 973128521 1006682977 1040137411 
		1073741833)

"The above primes past 2069 were chosen carefully so that they do not interact badly with 1664525 (used by hashMultiply), and so that gcd(p, (256^k) +/- a) = 1, for 0 cost ifTrue: [ cost := newCost ] ].
	cost ]."! !
!HashedCollection methodsFor: 'adding' stamp: 'ul 4/12/2010 22:38' prior: 53647096!
add: newObject withOccurrences: anInteger
	"Add newObject anInteger times to the receiver. Do nothing if anInteger is less than one. Answer newObject."
	
	anInteger < 1 ifTrue: [ ^newObject ].
	^self add: newObject "I can only store an object once."
	! !
!HashedCollection methodsFor: 'private' stamp: 'ul 4/12/2010 22:53'!
compact
	"Reduce the size of array so that the load factor will be ~75%."
	
	| newCapacity |
	newCapacity := self class goodPrimeAtLeast: tally * 4 // 3.
	self growTo: newCapacity! !
!WeakSet methodsFor: 'private' stamp: 'ul 4/12/2010 22:59'!
compact
	"Reduce the size of array so that the load factor will be ~75%."
	
	| newCapacity |
	newCapacity := self class goodPrimeAtLeast: self slowSize * 4 // 3.
	self growTo: newCapacity! !
!Symbol class methodsFor: 'class initialization' stamp: 'ul 4/13/2010 00:00' prior: 30357901!
compactSymbolTable
	"Reduce the size of the symbol table so that it holds all existing symbols with 25% free space."

	| oldSize |
	Smalltalk garbageCollect.
	oldSize := SymbolTable capacity.
	SymbolTable compact.
	^(oldSize - SymbolTable capacity) printString, ' slot(s) reclaimed'! !

KeyedIdentitySet class removeSelector: #goodPrimes!

WeakIdentityKeyDictionary class removeSelector: #goodPrimes!

IdentitySet class removeSelector: #goodPrimes!

IdentityDictionary class removeSelector: #goodPrimes!

"Collections"!
!HashedCollectionTest methodsFor: 'test - class - sizing' stamp: 'ul 4/7/2010 00:18' prior: 58761579!
testPrimes: primes

	| badPrimes |
	badPrimes := #(3 5 71 139 479 5861 277421). "These primes are less than the hashMultiply constant (1664525) and 1664525 \\ prime is close to 0 (mod prime). The following snippet reproduces these numbers: 
	| hashMultiplyConstant |
	hashMultiplyConstant := 1 hashMultiply.
	(Integer primesUpTo: hashMultiplyConstant) select: [ :each |
		| remainder |
		remainder := hashMultiplyConstant \\ each.
		remainder <= 1 or: [ remainder + 1 = each ] ]."
	self assert: primes isSorted.
	primes do: [ :each |
		self assert: each isPrime.
		self deny: (each > 2069 and: [ badPrimes includes: each ]) ].
	self assert: (
		primes select: [ :p |
			| result |
			result := false.
			p > 2069 ifTrue: [
			1 to: 8 do: [ :k |
				1 to: 32 do: [ :a |
					(p gcd: (256 raisedTo: k) + a) = 1 ifFalse: [
						result := true ].
					(p gcd: (256 raisedTo: k) - a) = 1 ifFalse: [
						result := true ] ] ] ].
			result ]) isEmpty.! !

HashedCollectionTest removeSelector: #testGoodPrimesForIdentityBasedHashedCollections!

"CollectionsTests"!
!MCMczReader methodsFor: 'as yet unclassified' stamp: 'bf 4/18/2010 18:38' prior: 22938947!
extractInfoFrom: dict
	^MCWorkingCopy infoFromDictionary: dict cache: self infoCache! !
!MCWorkingCopy class methodsFor: 'as yet unclassified' stamp: 'bf 4/19/2010 00:39' prior: 23215403!
infoFromDictionary: aDictionary cache: cache
	| id |
	id := (aDictionary at: #id) asString.
	^ cache at: id ifAbsentPut:
		[MCVersionInfo
			name: (aDictionary at: #name ifAbsent: [''])
			id: (UUID fromString: id)
			message: (aDictionary at: #message ifAbsent: [''])
			date: ([Date fromString: (aDictionary at: #date)] ifError: [nil])
			time: ([Time fromString: (aDictionary at: #time)] ifError: [nil])
			author: (aDictionary at: #author ifAbsent: [''])
			ancestors: (self ancestorsFromArray: (aDictionary at: #ancestors ifAbsent: []) cache: cache)
			stepChildren: (self ancestorsFromArray: (aDictionary at: #stepChildren ifAbsent: []) cache: cache)]! !
!MCVersionInfo methodsFor: 'converting' stamp: 'bf 4/18/2010 23:25' prior: 23175569!
asDictionary
	^ Dictionary new
		at: #name put: name;
		at: #id put: id asString;
		at: #message put: message;
		at: #date put: date;
		at: #time put: time;
		at: #author put: author;
		at: #ancestors put: (self ancestors collect: [:a | a asDictionary]);
		yourself! !

"Monticello"!
!BlockContextTest methodsFor: 'running' stamp: 'md 9/6/2005 19:56' prior: 50431957!
setUp
	super setUp.
	aBlockContext := [100 at 100 corner: 200 at 200].
	contextOfaBlockContext := thisContext.! !
!BehaviorTest methodsFor: 'tests' stamp: 'md 2/18/2006 16:42' prior: 17365994!
testBinding
	self assert: Object binding value = Object.
	self assert: Object binding key = #Object.
	
	self assert: Object class binding value = Object class.
	
	"returns nil for Metaclasses... like Encoder>>#associationFor:"
	
	self assert: Object class binding key = nil.! !
!CompledMethodTrailerTest methodsFor: 'testing' stamp: 'Igor.Stasenko 12/13/2009 21:13' prior: 53956757!
testEmbeddingSourceCode

	| trailer newTrailer code |
	
	trailer := CompiledMethodTrailer new.
	
	code := 'foo'.
	trailer sourceCode: code.
	newTrailer := trailer testEncoding.
	
	self assert: (trailer kind == #EmbeddedSourceQCompress ).
	self assert: (newTrailer sourceCode = code).

	"the last bytecode index must be at 0"
	self assert: (newTrailer endPC = 0).

	code := 'testEmbeddingSourceCode

	| trailer newTrailer code |
	
	trailer := CompiledMethodTrailer new.
	
	trailer sourceCode: code.
	newTrailer := trailer testEncoding.
	
	self assert: (newTrailer sourceCode = code).'.

	trailer sourceCode: code.
	self assert: (trailer kind == #EmbeddedSourceZip ).
	newTrailer := trailer testEncoding.
	
	self assert: (newTrailer sourceCode = code).
	"the last bytecode index must be at 0"
	self assert: (newTrailer endPC = 0).
! !
!CompledMethodTrailerTest methodsFor: 'testing' stamp: 'Igor.Stasenko 12/13/2009 21:13' prior: 53957691!
testEmbeddingTempNames

	| trailer newTrailer code |
	
	trailer := CompiledMethodTrailer new.
	
	code := 'foo'.
	trailer tempNames: code.
	newTrailer := trailer testEncoding.
	
	self assert: (trailer kind == #TempsNamesQCompress ).
	self assert: (newTrailer tempNames = code).
	"the last bytecode index must be at 0"
	self assert: (newTrailer endPC = 0).
	

	code := 'testEmbeddingSourceCode

	| trailer newTrailer code |
	
	trailer := CompiledMethodTrailer new.
	
	trailer sourceCode: code.
	newTrailer := trailer testEncoding.
	
	self assert: (newTrailer sourceCode = code).'.

	trailer tempNames: code.
	self assert: (trailer kind == #TempsNamesZip ).
	newTrailer := trailer testEncoding.
	
	self assert: (newTrailer tempNames = code).
	"the last bytecode index must be at 0"
	self assert: (newTrailer endPC = 0).
! !
!CompledMethodTrailerTest methodsFor: 'testing' stamp: 'Igor.Stasenko 12/13/2009 21:17' prior: 53958613!
testEncodingNoTrailer

	| trailer |
	
	trailer := CompiledMethodTrailer new.
	
	"by default it should be a no-trailer"	
	self assert: (trailer kind == #NoTrailer ).
	self assert: (trailer size = 1).
	
	trailer := trailer testEncoding.
	
	self assert: (trailer kind == #NoTrailer ).
	self assert: (trailer size = 1).
	"the last bytecode index must be at 0"
	self assert: (trailer endPC = 0).
! !
!CompledMethodTrailerTest methodsFor: 'testing' stamp: 'Igor.Stasenko 12/13/2009 21:14' prior: 53959109!
testEncodingSourcePointer

	| trailer |
	
	trailer := CompiledMethodTrailer new.
	
	CompiledMethod allInstancesDo: [:method | | ptr |
		trailer method: method.
		self assert: ( (ptr := method sourcePointer) == trailer sourcePointer).
		"the last bytecode index must be at 0"
		ptr ~= 0 ifTrue: [
			self assert: (method endPC = trailer endPC) ].
	 ].! !
!CompledMethodTrailerTest methodsFor: 'testing' stamp: 'Igor.Stasenko 12/13/2009 21:15' prior: 53959564!
testEncodingVarLengthSourcePointer

	| trailer newTrailer |
	
	trailer := CompiledMethodTrailer new.
	
	trailer sourcePointer: 1.
	newTrailer := trailer testEncoding.
	
	self assert: (newTrailer sourcePointer = 1).
	
	trailer sourcePointer: 16r100000000000000.
	newTrailer := trailer testEncoding.
	self assert: (newTrailer sourcePointer = 16r100000000000000).
	"the last bytecode index must be at 0"
	self assert: (newTrailer endPC = 0).
! !
!CompledMethodTrailerTest methodsFor: 'testing' stamp: 'Igor.Stasenko 12/13/2009 21:15' prior: 53960108!
testSourceByIdentifierEncoding

	| trailer id |
	
	trailer := CompiledMethodTrailer new.
	
	id := UUID new asString.
	trailer sourceIdentifier: id.
	
	self assert: (trailer kind == #SourceByStringIdentifier ).
	
	trailer := trailer testEncoding.
	
	self assert: (trailer kind == #SourceByStringIdentifier ).
	self assert: (trailer sourceIdentifier = id).
	"the last bytecode index must be at 0"
	self assert: (trailer endPC = 0).
! !
!CompledMethodTrailerTest methodsFor: 'testing' stamp: 'Igor.Stasenko 12/13/2009 21:49' prior: 53960643!
testSourceBySelectorEncoding

	| trailer |
	
	trailer := CompiledMethodTrailer new.
	
	trailer setSourceBySelector.
	
	self assert: (trailer kind == #SourceBySelector ).
	self assert: (trailer size = 1).
	
	trailer := trailer testEncoding.
	
	self assert: (trailer kind == #SourceBySelector ).
	self assert: (trailer size = 1).
	"the last bytecode index must be at 0"
	self assert: (trailer endPC = 0).
! !
!CategorizerTest methodsFor: 'running' stamp: 'mtf 9/10/2007 10:10' prior: 18074036!
setUp
	categorizer := Categorizer defaultList: #(a b c d e).
	categorizer classifyAll: #(a b c) under: 'abc'.
	categorizer addCategory: 'unreal'.! !
!CategorizerTest methodsFor: 'testing' stamp: 'mtf 9/10/2007 10:17' prior: 18074267!
testClassifyNewElementNewCategory
	categorizer classify: #f under: #nice.
	self assert: categorizer printString =
'(''as yet unclassified'' d e)
(''abc'' a b c)
(''unreal'')
(''nice'' f)
'! !
!CategorizerTest methodsFor: 'testing' stamp: 'mtf 9/10/2007 10:18' prior: 18074541!
testClassifyNewElementOldCategory
	categorizer classify: #f under: #unreal.
	self assert: categorizer printString =
'(''as yet unclassified'' d e)
(''abc'' a b c)
(''unreal'' f)
'! !
!CategorizerTest methodsFor: 'testing' stamp: 'mtf 9/10/2007 10:17' prior: 18074806!
testClassifyOldElementNewCategory
	categorizer classify: #e under: #nice.
	self assert: categorizer printString =
'(''as yet unclassified'' d)
(''abc'' a b c)
(''unreal'')
(''nice'' e)
'! !
!CategorizerTest methodsFor: 'testing' stamp: 'mtf 9/10/2007 12:54' prior: 18075078!
testClassifyOldElementOldCategory
	categorizer classify: #e under: #unreal.
	self assert: categorizer printString =
'(''as yet unclassified'' d)
(''abc'' a b c)
(''unreal'' e)
'! !
!CategorizerTest methodsFor: 'testing' stamp: 'mtf 9/10/2007 10:22' prior: 18075341!
testDefaultCategoryIsTransient
	"Test that category 'as yet unclassified' disapears when all it's elements are removed'"
	categorizer classifyAll: #(d e) under: #abc.
	self assert: categorizer printString =
'(''abc'' a b c d e)
(''unreal'')
'! !
!CategorizerTest methodsFor: 'testing' stamp: 'mtf 9/11/2007 15:15' prior: 18075669!
testNullCategory
	"Test that category 'as yet unclassified' disapears when all it's elements are removed'"
	| aCategorizer |
	aCategorizer := Categorizer defaultList: #().
	self assert: aCategorizer printString =
'(''as yet unclassified'')
'.
	self assert: aCategorizer categories = #('no messages').
	aCategorizer classify: #a under: #b.
	self assert: aCategorizer printString =
'(''b'' a)
'.
	self assert: aCategorizer categories = #(b).! !
!CategorizerTest methodsFor: 'testing' stamp: 'mtf 9/10/2007 12:57' prior: 18076194!
testRemoveEmptyCategory
	categorizer removeCategory: #unreal.
	self assert: categorizer printString =
'(''as yet unclassified'' d e)
(''abc'' a b c)
'! !
!CategorizerTest methodsFor: 'testing' stamp: 'mtf 9/10/2007 12:55' prior: 18076430!
testRemoveExistingElement
	categorizer removeElement: #a.
	self assert: categorizer printString =
'(''as yet unclassified'' d e)
(''abc'' b c)
(''unreal'')
'! !
!CategorizerTest methodsFor: 'testing' stamp: 'mtf 9/10/2007 12:59' prior: 18076673!
testRemoveNonEmptyCategory
	self should: [categorizer removeCategory: #abc] raise: Error.
	self assert: categorizer printString =
'(''as yet unclassified'' d e)
(''abc'' a b c)
(''unreal'')
'! !
!CategorizerTest methodsFor: 'testing' stamp: 'mtf 9/10/2007 12:59' prior: 18076950!
testRemoveNonExistingCategory
	categorizer removeCategory: #nice.
	self assert: categorizer printString =
'(''as yet unclassified'' d e)
(''abc'' a b c)
(''unreal'')
'! !
!CategorizerTest methodsFor: 'testing' stamp: 'mtf 9/10/2007 12:57' prior: 18077203!
testRemoveNonExistingElement
	categorizer removeElement: #f.
	self assert: categorizer printString =
'(''as yet unclassified'' d e)
(''abc'' a b c)
(''unreal'')
'! !
!CategorizerTest methodsFor: 'testing' stamp: 'mtf 9/11/2007 14:49' prior: 18077451!
testRemoveThenRename
	categorizer removeCategory: #unreal.
	categorizer renameCategory: #abc toBe: #unreal.
	self assert: categorizer printString =
'(''as yet unclassified'' d e)
(''unreal'' a b c)
'! !
!CategorizerTest methodsFor: 'testing' stamp: 'mtf 9/10/2007 10:14' prior: 18077736!
testUnchanged
	self assert: categorizer printString =
'(''as yet unclassified'' d e)
(''abc'' a b c)
(''unreal'')
'! !

"KernelTests"!
!SmalltalkImage methodsFor: 'accessing' stamp: 'ul 4/18/2010 22:22'!
at: key ifPresentAndInMemory: aBlock
	"Lookup the given key in the receiver. If it is present, answer the value of evaluating the given block with the value associated with the key. Otherwise, answer nil."

	^globals at: key ifPresentAndInMemory: aBlock! !
!SmalltalkImage methodsFor: 'image' stamp: 'dtl 4/11/2010 11:45'!
image
	"Answer the object to query about the current object memory and execution environment."
	
	^self! !
!SmalltalkImage methodsFor: 'image' stamp: 'dtl 4/11/2010 11:47'!
imageFormatVersion
	"Answer an integer identifying the type of image. The image version number may
	identify the format of the image (e.g. 32 or 64-bit word size) or specific requirements
	of the image (e.g. block closure support required). This invokes an optional primitive
	that may not be available on all virtual machines."

	"Smalltalk image imageFormatVersion"

	
	self notify: 'This virtual machine does not support the optional primitive #primitiveImageFormatVersion' translated.
	^''! !
!SmalltalkImage methodsFor: 'vm' stamp: 'dtl 4/11/2010 11:38'!
interpreterSourceVersion
	"Answer a string corresponding to the version of the interpreter source.
	This represents the version level of the Smalltalk source code (interpreter
	and various plugins) that is translated to C by a CCodeGenerator, as distinct
	from the external platform source code, typically written in C and managed
	separately for each platform. An optional primitive is invoked that may not
	be available on all virtual machines."

	"Smalltalk vm interpreterSourceVersion"

	
	self notify: 'This virtual machine does not support the optional primitive #primitiveInterpreterSourceVersion' translated.
	^''! !
!SmalltalkImage methodsFor: 'vm' stamp: 'dtl 4/11/2010 11:39'!
platformSourceVersion
	"Answer a string corresponding to the version of the external platform source
	code, typically written in C and managed separately for each platform. This
	invokes an optional primitive that may not be available on all virtual machines."

	"Smalltalk vm platformSourceVersion"

	
	self notify: 'This virtual machine does not support the optional primitive #primitivePlatformSourceVersion' translated.
	^''! !
!SmalltalkImage methodsFor: 'image' stamp: 'md 5/16/2006 12:34' prior: 58536670!
version
	"Answer the version of this release."

	^SystemVersion current version! !
!SmalltalkImage methodsFor: 'vm' stamp: 'dtl 4/11/2010 11:39'!
versionLabel
	"Answer a string corresponding to the version of virtual machine. This
	represents the version level of the Smalltalk source code (interpreter
	and various plugins) that is translated to C by a CCodeGenerator,  in
	addition to the external platform source code, typically written in C and
	managed separately for each platform.
	
	This invokes an optional primitive that may not be available on all virtual
	machines. See also vmVersion, which answers a string identifying the image
	from which virtual machine sources were generated."

	"Smalltalk vm versionLabel"

	
	self notify: 'This virtual machine does not support the optional primitive #primitiveVMVersion' translated.
	^''! !
!SmalltalkImage methodsFor: 'vm' stamp: 'dtl 4/11/2010 11:15'!
vm
	"Answer the object to query about virtual machine."
	
	^self! !
!SmalltalkImage methodsFor: 'image' stamp: 'dtl 1/4/2010 21:40' prior: 58537225!
wordSize
	"Answer the size in bytes of an object pointer or word in the object memory.
	The value does not change for a given image, but may be modified by a SystemTracer
	when converting the image to another format. The value is cached in WordSize to
	avoid the performance overhead of repeatedly consulting the VM."

	"Smalltalk wordSize"

	^ WordSize ifNil: [WordSize := [SmalltalkImage current vmParameterAt: 40] on: Error do: [4]]! !

"System"!
!SMLoaderPlus commentStamp: 'btr 12/1/2006 15:16' prior: 0!
A simple package loader that is currently the standard UI for SqueakMap (the model is an SMSqueakMap instance). It uses ToolBuilder to construct its window. You can open one with:

	SMLoaderPlus open

Instance Variables
	categoriesToFilterIds:		 The set of categories to filter the packages list.
	filters:				 The set of filters to apply to the packages list.
	map:				 The model SqueakMap.
	packagesList:		 The list of packages from the map.
	selectedCategory:	 The current category.
	selectedItem:			 The selected package or release.
	window:			 The window, held only so we can reOpen.!
!SMLoaderCategoricalPlus commentStamp: 'btr 12/4/2006 15:47' prior: 0!
A variant package loader that uses a more-or-less standard Smalltalk-80 browser perspective of selecting categories in one pane and then selecting items within in the next pane.
You can open one with:

	SMLoaderCategoricalPlus open!
!SMLoader commentStamp: 'btr 11/30/2006 18:00' prior: 27913009!
A simple package loader that is currently the standard UI for SqueakMap (the model is an SMSqueakMap instance).
You can open one with:

	SMLoader open!
!SMLoaderCategorical commentStamp: 'btr 12/1/2006 15:16' prior: 0!
A variant package loader that uses a more-or-less standard Smalltalk-80 browser perspective of selecting categories in one pane and then selecting items within in the next pane.
You can open one with:

	SMLoaderCategorical open!
!SMLoaderCategoricalPlus class methodsFor: 'menu registration' stamp: 'btr 12/1/2006 18:06'!
initialize
	Smalltalk at: #ToolBuilder ifPresent: [:tb |
		(TheWorldMenu respondsTo: #registerOpenCommand:)
			ifTrue: [TheWorldMenu registerOpenCommand: {self openMenuString. {self. #open}}]]! !
!SMLoaderCategoricalPlus class methodsFor: 'menu registration' stamp: 'btr 12/1/2006 17:34'!
openMenuString
	^ 'SqueakMap Categories'! !
!SMLoaderCategoricalPlus class methodsFor: 'menu registration' stamp: 'btr 12/1/2006 17:34'!
removeFromSystem
	(TheWorldMenu respondsTo: #registerOpenCommand:)
		ifTrue: [TheWorldMenu unregisterOpenCommand: self openMenuString].
	self removeFromSystem: true! !
!SMLoaderCategoricalPlus class methodsFor: 'menu registration' stamp: 'btr 12/1/2006 17:34'!
unload
	(TheWorldMenu respondsTo: #registerOpenCommand:) ifTrue: 
		[TheWorldMenu unregisterOpenCommand: self openMenuString].! !
!SMLoaderCategoricalPlus methodsFor: 'interface' stamp: 'btr 12/5/2006 06:50'!
buildFancyWith: aBuilder
	"Creates a variant of the window where the package pane is split between installed and uninstalled packages."
	| buttonBarHeight searchHeight vertDivide horizDivide |
	buttonBarHeight := 0.07.
	searchHeight := 0.07.
	vertDivide := 0.5.
	horizDivide := 0.6.
	builder := aBuilder.
	window := builder build: (builder pluggableWindowSpec new model: self;
				label: #label;
				children: (OrderedCollection new add:
				((self buildButtonBarWith: builder)
					frame: (0 @ 0 corner: 1 @ buttonBarHeight); yourself);
				add: ((self buildCategoriesListWith: builder)
					frame: (0 @ buttonBarHeight corner: vertDivide @ horizDivide); yourself);
				add: ((self buildSearchPaneWith: builder)
					frame: (vertDivide @ buttonBarHeight corner: 1 @ (buttonBarHeight + searchHeight)); yourself);
				add: ((self buildNotInstalledPackagesListWith: builder)
					frame: (vertDivide @ (buttonBarHeight + searchHeight) corner: 1 @ (horizDivide / 2)); yourself);
				add: ((self buildInstalledPackagesListWith: builder)
					frame: (vertDivide @ (horizDivide / 2) corner: 1 @ horizDivide); yourself);
				add: ((self buildPackagePaneWith: builder)
					frame: (0 @ horizDivide corner: 1 @ 1); yourself); yourself)).
	window on: #mouseEnter send: #paneTransition: to: window.
	window on: #mouseLeave send: #paneTransition: to: window.
	self setUpdatablePanesFrom: #(#installedPackageList #notInstalledPackageList ).
	currentPackageList := #notInstalled.
	window extent: self initialExtent.
	^ window! !
!SMLoaderCategoricalPlus methodsFor: 'interface' stamp: 'btr 12/1/2006 17:56'!
buildInstalledPackagesListWith: aBuilder
	^ aBuilder pluggableTreeSpec new model: self;
		 roots: #installedPackageList;
		 getSelectedPath: #selectedItemPath;
		 setSelected: #selectedItem:;
		 menu: #packagesMenu:;
		 label: #itemLabel:;
		 getChildren: #itemChildren:;
		 hasChildren: #itemHasChildren:;
		 autoDeselect: true;
		 wantsDrop: true;
		 yourself! !
!SMLoaderCategoricalPlus methodsFor: 'interface' stamp: 'btr 12/1/2006 17:52'!
buildNotInstalledPackagesListWith: aBuilder
	^ aBuilder pluggableTreeSpec new model: self;
		 roots: #notInstalledPackageList;
		 getSelectedPath: #selectedItemPath;
		 setSelected: #selectedItem:;
		 menu: #packagesMenu:;
		 label: #itemLabel:;
		 getChildren: #itemChildren:;
		 hasChildren: #itemHasChildren:;
		 autoDeselect: true;
		 wantsDrop: true;
		 yourself! !
!SMLoaderCategoricalPlus methodsFor: 'interface' stamp: 'btr 12/5/2006 06:55'!
buildWith: aBuilder
	| buttonBarHeight searchHeight vertDivide horizDivide |
	buttonBarHeight := 0.07.
	searchHeight := 0.07.
	vertDivide := 0.5.
	horizDivide := 0.6.
	builder := aBuilder.
	window := builder build: (builder pluggableWindowSpec new model: self;
				label: #label;
				children: (OrderedCollection new add:
				((self buildButtonBarWith: builder)
					frame: (0 @ 0 corner: 1 @ buttonBarHeight); yourself);
				add: ((self buildCategoriesListWith: builder)
					frame: (0 @ buttonBarHeight corner: vertDivide @ horizDivide); yourself);
				add: ((self buildSearchPaneWith: builder)
					frame: (vertDivide @ buttonBarHeight corner: 1 @ (buttonBarHeight + searchHeight)));
				add: ((self buildPackagesListWith: builder)
					frame: (vertDivide @ (buttonBarHeight + searchHeight) corner: 1 @ horizDivide));
				add: ((self buildPackagePaneWith: builder)
					frame: (0 @ horizDivide corner: 1 @ 1)); yourself)).
	window on: #mouseEnter send: #paneTransition: to: window.
	window on: #mouseLeave send: #paneTransition: to: window.
	window extent: self initialExtent.
	^ window! !
!SMLoaderCategoricalPlus methodsFor: 'accessing' stamp: 'btr 12/1/2006 17:34'!
currentPackageList
	^currentPackageList! !
!SMLoaderCategoricalPlus methodsFor: 'accessing' stamp: 'btr 12/1/2006 17:34'!
currentPackageList: aSymbol
	currentPackageList := aSymbol.
	self changed: #installButtonLabel.! !
!SMLoaderCategoricalPlus methodsFor: 'interface' stamp: 'btr 12/4/2006 15:55'!
defaultLabel
	^ 'Categorical ' , super defaultLabel! !
!SMLoaderCategoricalPlus methodsFor: 'interface' stamp: 'btr 12/4/2006 15:58'!
installButtonLabel
	^ self currentPackageList = #notInstalled
		ifTrue: ['Install the above package']
		ifFalse: ['Remove the above package']! !
!SMLoaderCategoricalPlus methodsFor: 'lists' stamp: 'btr 12/1/2006 17:52'!
installedPackageList
	^self packageList select: [:e | e isInstalled]! !
!SMLoaderCategoricalPlus methodsFor: 'accessing' stamp: 'btr 12/1/2006 18:02'!
installedPackagesListIndex
	^ self currentPackageList = #installed
		ifTrue: [self packagesListIndex]
		ifFalse: [0]! !
!SMLoaderCategoricalPlus methodsFor: 'accessing' stamp: 'btr 12/1/2006 17:34'!
installedPackagesListIndex: anObject 
	packagesListIndex := anObject.
	self currentPackageList ~= #installed
		ifTrue: [self currentPackageList: #installed.
			self changed: #currentPackageList].
	self noteChanged! !
!SMLoaderCategoricalPlus methodsFor: 'accessing' stamp: 'btr 12/1/2006 17:34'!
isOn
	^false! !
!SMLoaderCategoricalPlus methodsFor: 'lists' stamp: 'btr 12/1/2006 17:53'!
notInstalledPackageList
	^self packageList reject: [:e | e isInstalled]! !
!SMLoaderCategoricalPlus methodsFor: 'accessing' stamp: 'btr 12/1/2006 18:02'!
notInstalledPackagesListIndex
	^ self currentPackageList = #notInstalled
		ifTrue: [self packagesListIndex]
		ifFalse: [0]! !
!SMLoaderCategoricalPlus methodsFor: 'accessing' stamp: 'btr 12/1/2006 18:03'!
notInstalledPackagesListIndex: anObject 
	packagesListIndex := anObject.
	self currentPackageList ~= #notInstalled ifTrue:
		[self currentPackageList: #notInstalled.
		 self changed: #currentPackageList].
	self changed: #packagesListIndex.
	"update my selection"
	self noteChanged.
	self contentsChanged! !
!SMLoaderCategoricalPlus methodsFor: 'private' stamp: 'btr 12/1/2006 17:53'!
noteChanged
	self changed: #installedPackageList.
	self changed: #notInstalledPackageList.
	super noteChanged."
	self changed: #packageNameList.
	self changed: #packagesListIndex.
	self changed: #categoriesForPackage.
	self contentsChanged."! !
!SMLoaderCategoricalPlus methodsFor: 'lists' stamp: 'btr 12/1/2006 17:34'!
packageList
	^ self packages
		select: [:e | (e categories
					anySatisfy: [:cat | cat = self selectedCategory])
				and: [(filters ifNil: [#()])
						allSatisfy: [:currFilter | (self perform: currFilter)
								value: e]]]! !
!SMLoaderPlus class methodsFor: 'parts bin' stamp: 'btr 11/22/2006 15:02'!
descriptionForPartsBin
	^self partName: 'Package Loader'
		categories: #(Tools)
		documentation: 'SqueakMap UI'
! !
!SMLoaderPlus class methodsFor: 'class initialization' stamp: 'btr 12/1/2006 15:47'!
initialize
	"Hook us up in the world menu."
	
	"self initialize"

	Smalltalk at: #ToolBuilder ifPresent: [:tb |
		self registerInFlapsRegistry.
		(Preferences windowColorFor: #SMLoader) = Color white "not set"
			ifTrue: [ Preferences setWindowColorFor: #SMLoader to: (Color colorFrom: self windowColorSpecification brightColor) ].
		 (TheWorldMenu respondsTo: #registerOpenCommand:)
	         ifTrue: [| oldCmds |
				oldCmds := TheWorldMenu registry select: [:cmd | cmd first includesSubString: 'Package Loader'].
				oldCmds do: [:cmd | TheWorldMenu unregisterOpenCommand: cmd first].
			TheWorldMenu registerOpenCommand: {self openMenuString. {self. #open}}]].
	DefaultFilters := OrderedCollection new.
	DefaultCategoriesToFilterIds := OrderedCollection new! !
!SMLoaderPlus class methodsFor: 'new-morph participation' stamp: 'btr 11/22/2006 15:16'!
initializedInstance
	^ (ToolBuilder open: self new) extent: 400 at 400! !
!SMLoaderPlus class methodsFor: 'instance creation' stamp: 'btr 11/22/2006 15:02'!
new
	"Create a SqueakMap loader on the default map."

	^self newOn: SMSqueakMap default! !
!SMLoaderPlus class methodsFor: 'instance creation' stamp: 'btr 11/22/2006 15:02'!
newOn: aMap
	"Create a SqueakMap loader on given map."

	^super new on: aMap; yourself! !
!SMLoaderPlus class methodsFor: 'new-morph participation' stamp: 'btr 11/22/2006 15:16'!
newStandAlone
	^ ToolBuilder open: self new! !
!SMLoaderPlus class methodsFor: 'instance creation' stamp: 'btr 11/23/2006 11:13'!
open
	"Create and open a SqueakMap Loader."
	
	"SMLoaderPlus open"

	^ (Smalltalk at: #ToolBuilder) open: self new! !
!SMLoaderPlus class methodsFor: 'class initialization' stamp: 'btr 11/30/2006 21:50'!
openMenuString
	^ 'SqueakMap Catalog'! !
!SMLoaderPlus class methodsFor: 'instance creation' stamp: 'btr 11/23/2006 11:21'!
openOn: aSqueakMap
	"Create and open a SqueakMap Loader on a given map."

	"self openOn: SqueakMap default"

	^ (Smalltalk at: #ToolBuilder) open: (self newOn: aSqueakMap)! !
!SMLoaderPlus class methodsFor: 'new-morph participation' stamp: 'btr 11/22/2006 15:18'!
prototypicalToolWindow
	^ ToolBuilder open: self new; applyModelExtent; yourself! !
!SMLoaderPlus class methodsFor: 'new-morph participation' stamp: 'btr 11/22/2006 15:02'!
registerInFlapsRegistry
	"Register the receiver in the system's flaps registry."

	self environment
		at: #Flaps
		ifPresent: [:cl | (cl respondsTo: #registerQuad:forFlapNamed:)
				ifTrue: [cl registerQuad: #(#SMLoader #prototypicalToolWindow 'Package Loader' 'The SqueakMap Package Loader' ) forFlapNamed: 'Tools']]! !
!SMLoaderPlus class methodsFor: 'class initialization' stamp: 'btr 11/30/2006 21:50'!
unload
	(TheWorldMenu respondsTo: #registerOpenCommand:) ifTrue: 
		[TheWorldMenu unregisterOpenCommand: self openMenuString].
	self environment at: #Flaps ifPresent: [:cl |
	cl unregisterQuadsWithReceiver: self] ! !
!SMLoaderPlus class methodsFor: 'window color' stamp: 'btr 11/22/2006 15:02'!
windowColorSpecification
	"Answer a WindowColorSpec object that declares my preference."

	^WindowColorSpec
		classSymbol: self name
		wording: 'Package Loader'
		brightColor: Color yellow muchLighter duller
		pastelColor: Color yellow veryMuchLighter duller
		helpMessage: 'The SqueakMap Package Loader'! !
!SMLoaderPlus methodsFor: 'menus' stamp: 'btr 11/22/2006 15:02'!
addFiltersToMenu: aMenu
	| filterSymbol help |
	self filterSpecs do: [:filterArray | 
		filterSymbol := filterArray second.
		help := filterArray third.
		aMenu addUpdating: #showFilterString: target: self selector: #toggleFilterState: argumentList: (Array with: filterSymbol).
		aMenu balloonTextForLastItem: help].
	aMenu addLine;
		addList: #(('Clear all filters' uncheckFilters 'Unchecks all filters to list all packages'))
	! !
!SMLoaderPlus methodsFor: 'actions' stamp: 'btr 11/22/2006 15:02'!
addSelectedCategoryAsFilter
	"Add a new filter that filters on the currently selected category.
	Make it enabled as default."

	categoriesToFilterIds add: self selectedCategory id! !
!SMLoaderPlus methodsFor: 'actions' stamp: 'btr 11/22/2006 16:11'!
askToLoadUpdates
	"Check how old the map is and ask to update it
	if it is older than 10 days or if there is no map on disk."

	| available |
	available := map isCheckpointAvailable.
	(available not or: [
		(Date today subtractDate: (Date fromSeconds:
			(map directory directoryEntryFor: map lastCheckpointFilename)
				modificationTime)) > 3])
		ifTrue: [
			(self confirm: 
				(available ifTrue: ['The map on disk is more than 10 days old,
update it from the Internet?'] ifFalse: ['There is no map on disk,
fetch it from the Internet?']))
				ifTrue: [self loadUpdates]]! !
!SMLoaderPlus methodsFor: 'interface' stamp: 'btr 12/1/2006 01:43'!
browseCacheDirectory
	"Open a FileList2 on the directory for the package or release."
	| item dir win |
	item := self selectedPackageOrRelease
				ifNil: [^ nil].
	dir := item isPackage
				ifTrue: [map cache directoryForPackage: item]
				ifFalse: [map cache directoryForPackageRelease: item].
	win := FileList2 morphicViewOnDirectory: dir.
	"withLabel: item name, ' cache directory'."
	win openInWorld! !
!SMLoaderPlus methodsFor: 'interface' stamp: 'btr 12/5/2006 06:56'!
buildButtonBarWith: aBuilder
	^ aBuilder pluggablePanelSpec new
		model: self;
		layout: #horizontal;
		children: (self commandSpecs select: [ :spec | spec fourth includes: #all]
				thenCollect: [ :spec |
					aBuilder pluggableActionButtonSpec new
						model: self;
						label: spec first;
						action: spec second;
						help: spec third;
						enabled: ((spec fourth includes: #item) ifTrue: [#hasSelectedItem]);
						yourself]);
		name: #buttonBar;
		yourself! !
!SMLoaderPlus methodsFor: 'interface' stamp: 'btr 11/22/2006 15:02'!
buildButtonNamed: labelText helpText: balloon action: action
	| btn |
	btn := PluggableButtonMorph on: self getState: nil action: action.
	btn color: Color transparent;
		hResizing: #shrinkWrap;
		vResizing: #spaceFill;
		label: labelText;
		setBalloonText: balloon;
		onColor: Color transparent offColor: Color transparent.
	^ btn! !
!SMLoaderPlus methodsFor: 'interface' stamp: 'btr 12/5/2006 06:56'!
buildCategoriesListWith: aBuilder 
	"Create the hierarchical list holding the category tree."
	^ aBuilder pluggableTreeSpec new model: self;
		 roots: #categoryList;
		 getSelectedPath: #selectedCategoryPath;
		 getChildren: #categoryChildren:;
		 hasChildren: #categoryHasChildren:;
		 setSelected: #selectedCategory:;
		 menu: #categoriesMenu:;
		 label: #categoryLabel:;
		 autoDeselect: true;
		 wantsDrop: true;
		 name: #categoriesList;
		 yourself! !
!SMLoaderPlus methodsFor: 'interface' stamp: 'btr 12/5/2006 06:57'!
buildPackagePaneWith: aBuilder
	"Create the text area to the right in the loader."

	^ aBuilder pluggableTextSpec new model: self; getText: #itemDescription; name: #packagePane; yourself! !
!SMLoaderPlus methodsFor: 'interface' stamp: 'btr 12/5/2006 06:57'!
buildPackagesListWith: aBuilder 
	"Create the hierarchical list holding the packages and releases."
	^ aBuilder pluggableTreeSpec new model: self;
		 roots: #packageList;
		 getSelectedPath: #selectedItemPath;
		 setSelected: #selectedItem:;
		 menu: #packagesMenu:;
		 label: #itemLabel:;
		 getChildren: #itemChildren:;
		 hasChildren: #itemHasChildren:;
		 autoDeselect: true;
		 wantsDrop: true;
		 name: #packagesList;
		 yourself! !
!SMLoaderPlus methodsFor: 'interface' stamp: 'btr 12/5/2006 06:57'!
buildSearchPaneWith: aBuilder
	^ aBuilder pluggableInputFieldSpec new model: self;
		selection: #searchSelection;
		getText: #searchText; setText: #findPackage:notifying:; name: #search; yourself! !
!SMLoaderPlus methodsFor: 'interface' stamp: 'btr 12/5/2006 06:54'!
buildWith: aBuilder 
	"Create the package loader window."
	| buttonBarHeight vertDivide horizDivide |
	buttonBarHeight := 0.07.
	vertDivide := 0.6.
	horizDivide := 0.3.
	builder := aBuilder.
	window := builder build: (builder pluggableWindowSpec new model: self;
					 label: #label;
					 children: (OrderedCollection new
						add: ((self buildButtonBarWith: builder)
							frame: (0 @ 0 corner: 1 @ buttonBarHeight));
						add: ((self buildSearchPaneWith: builder)
							frame: (0 @ buttonBarHeight corner: horizDivide @ (buttonBarHeight * 2)));
						add: ((self buildPackagesListWith: builder)
							frame: (0 @ (buttonBarHeight * 2) corner: horizDivide @ vertDivide));
						add: ((self buildCategoriesListWith: builder)
							frame: (0 @ vertDivide corner: horizDivide @ 1));
						add: ((self buildPackagePaneWith: builder)
								frame: (horizDivide @ buttonBarHeight corner: 1 @ 1));
						 yourself);
					 yourself).
	window on: #mouseEnter send: #paneTransition: to: window.
	window on: #mouseLeave send: #paneTransition: to: window.
	window extent: self initialExtent.
	^ window! !
!SMLoaderPlus methodsFor: 'actions' stamp: 'btr 12/1/2006 01:38'!
cachePackageReleaseAndOfferToCopy
	"Cache package release, then offer to copy it somewhere.
	Answer the chosen file's location after copy,
	or the cache location if no directory was chosen."

	| release installer newDir newName newFile oldFile oldName |
	release := self selectedPackageOrRelease.
	release isPackageRelease ifFalse: [ self error: 'Should be a package release!!'].
	installer := SMInstaller forPackageRelease: release.
	[UIManager default informUser: 'Caching ' , release asString during: [installer cache]] on: Error do: [:ex |
		| msg |
		msg := ex messageText ifNil: [ex asString].
		self informException: ex msg: ('Error occurred during download:\', msg, '\') withCRs.
		^nil ].
	installer isCached ifFalse: [self inform: 'Download failed, see transcript for details'. ^nil].
	oldName := installer fullFileName.
	newDir := FileList2 modalFolderSelector: installer directory.
	newDir ifNil: [ ^oldName ].
	newDir = installer directory ifTrue: [ ^oldName ].
	newName := newDir fullNameFor: installer fileName.
	newFile := FileStream newFileNamed: newName.
	newFile ifNil: [ ^oldName ].
	newFile binary.
	oldFile := FileStream readOnlyFileNamed: oldName.
	oldFile ifNil: [ ^nil ].
	oldFile binary.
	[[ newDir copyFile: oldFile toFile: newFile ] ensure: [ oldFile close. newFile close ]] on: Error do: [ :ex | ^oldName ].
	^newName! !
!SMLoaderPlus methodsFor: 'menus' stamp: 'btr 11/22/2006 15:02'!
categoriesMenu: aMenu 
	"Answer the categories-list menu."

	self selectedCategory 
		ifNotNil: [aMenu addList: self categorySpecificOptions; addLine].
	aMenu addList: self generalOptions.
	self addFiltersToMenu: aMenu.
	^aMenu! !
!SMLoaderPlus methodsFor: 'interface' stamp: 'btr 11/24/2006 14:44'!
categoryChildren: aCategory
	^ aCategory subCategories! !
!SMLoaderPlus methodsFor: 'interface' stamp: 'btr 11/24/2006 14:45'!
categoryHasChildren: aCategory
	^ aCategory hasSubCategories! !
!SMLoaderPlus methodsFor: 'interface' stamp: 'btr 11/24/2006 14:46'!
categoryLabel: aCategory
	^ aCategory name! !
!SMLoaderPlus methodsFor: 'lists' stamp: 'btr 11/30/2006 21:01'!
categoryList
	"Create the category list for the hierarchical list.
	We sort the categories by name but ensure that 'Squeak versions'
	is first if it exists."
	| list first |
	list := (map categories
				select: [:each | each parent isNil]) asArray
				sort: [:c1 :c2 | c1 name <= c2 name].
	first := list
				detect: [:any | any name = 'Squeak versions']
				ifNone: [].
	first
		ifNotNil: [list := list copyWithout: first.
			list := {first} , list].
	^ list! !
!SMLoaderPlus methodsFor: 'menus' stamp: 'btr 11/22/2006 15:02'!
categorySpecificOptions
	| choices |
	choices := OrderedCollection new.
	(categoriesToFilterIds includes: self selectedCategory id)
		ifTrue: [
			choices add: #('Remove filter' #removeSelectedCategoryAsFilter 'Remove the filter for the selected category.')]
		ifFalse: [
			choices add: #('Add as filter' #addSelectedCategoryAsFilter 'Add the selection as a filter to hide unrelated packages.')].
	categoriesToFilterIds isEmpty ifFalse: [
		choices add: #('Remove all filters' #removeCategoryFilters 'Remove all category filters.')].
	^ choices! !
!SMLoaderPlus methodsFor: 'accessing' stamp: 'btr 11/22/2006 15:02'!
changeFilters: anObject 
	"Update my selection."

	| oldItem index |
	oldItem := self selectedPackageOrRelease.
	filters := anObject.
	self packagesListIndex: ((index := self packageList indexOf: oldItem) 
				ifNil: [0]
				ifNotNil: [index]).
	self noteChanged! !
!SMLoaderPlus methodsFor: 'menus' stamp: 'btr 11/22/2006 18:01'!
commandSpecFor: selector
	^ self commandSpecs detect: [:spec | spec second = selector]! !
!SMLoaderPlus methodsFor: 'menus' stamp: 'btr 11/22/2006 18:00'!
commandSpecs
	^ #(('Install' installPackageRelease 'Install the latest version from the server.' (item all))
		('Email' emailPackageMaintainers 'Open an editor to send an email to the owner and co-maintainers of this package.' (item all))
		('Browse cache' browseCacheDirectory 'Browse cache directory of the selection.' (item all))
		('Copy from cache' cachePackageReleaseAndOfferToCopy 'Download selected release into cache first if needed, and then offer to copy it somewhere else.' (item))
		('Force download into cache' downloadPackageRelease 'Force a download of the selected release into the cache.' (item))
		('Update' loadUpdates 'Update the package index from the servers.' (all))
		('Upgrade All' upgradeInstalledPackagesConfirm 'Upgrade all installed packages (conf8irming each).' (all))
		('Upgrade all installed packages' upgradeInstalledPackagesNoConfirm '' (item))
		('Upgrade all installed packages confirming each' upgradeInstalledPackagesConfirm '' (item))
		('Copy list' listInPasteBuffer 'Puts the list as text into the clipboard.' (all))
		('Save filters' saveFiltersAsDefault 'Saves the current filters as default.' (all))
		('Help' help 'What is this?' (all)))! !
!SMLoaderPlus methodsFor: 'interface' stamp: 'btr 11/22/2006 15:02'!
defaultButtonPaneHeight
	"Answer the user's preferred default height for new button panes."

	^ Preferences parameterAt: #defaultButtonPaneHeight ifAbsentPut: [25]! !
!SMLoaderPlus methodsFor: 'lists' stamp: 'btr 12/1/2006 01:50'!
defaultLabel
	^ 'SqueakMap Package Loader'! !
!SMLoaderPlus methodsFor: 'actions' stamp: 'btr 12/1/2006 01:38'!
downloadPackageRelease
	"Force a download of the selected package release into the cache."

	| release |
	release := self selectedPackageOrRelease.
	release isPackageRelease ifFalse: [ self error: 'Should be a package release!!'].
	[UIManager default informUser: 'Downloading ' , release asString during: [
		(SMInstaller forPackageRelease: release) download]
	] on: Error do: [:ex |
		| msg | 
		msg := ex messageText ifNil: [ex asString].
		self informException: ex msg: ('Error occurred during download:\', msg, '\') withCRs]! !
!SMLoaderPlus methodsFor: 'actions' stamp: 'btr 11/22/2006 15:02'!
emailPackageMaintainers
	"Send mail to package owner and co-maintainers."

	| item package toAddresses |
	item := self selectedPackageOrRelease ifNil: [^ nil].
	package := item isPackageRelease ifTrue: [item package] ifFalse: [item].

	"(this logic should be moved to MailMessage as soon as it can handle 
multiple To: addresses)"
	toAddresses := '<', package owner email, '>'.
	package maintainers ifNotNil: [
		package maintainers do: [:maintainer |
			toAddresses := toAddresses, ', <', maintainer email, '>']].

	SMUtilities sendMailTo: toAddresses regardingPackageRelease: item! !
!SMLoaderPlus methodsFor: 'filter utilities' stamp: 'btr 11/22/2006 15:02'!
filterAdd: anObject

	self changeFilters: (self filters copyWith: anObject)
! !
!SMLoaderPlus methodsFor: 'filters' stamp: 'btr 11/22/2006 15:02'!
filterAutoInstall
	^[:package | package isInstallable]! !
!SMLoaderPlus methodsFor: 'filters' stamp: 'btr 12/1/2006 01:42'!
filterAvailable
	^[:package | package isAvailable]! !
!SMLoaderPlus methodsFor: 'filters' stamp: 'btr 11/22/2006 15:02'!
filterInstalled
	^[:package | package isInstalled]! !
!SMLoaderPlus methodsFor: 'filters' stamp: 'btr 11/22/2006 15:02'!
filterNotInstalledYet
	^[:package | package isInstalled not]! !
!SMLoaderPlus methodsFor: 'filters' stamp: 'btr 12/1/2006 01:42'!
filterNotUptoDate
	^[:package | package isAvailable]! !
!SMLoaderPlus methodsFor: 'filters' stamp: 'btr 11/22/2006 15:02'!
filterPublished
	^[:package | package isPublished]! !
!SMLoaderPlus methodsFor: 'filter utilities' stamp: 'btr 11/22/2006 15:02'!
filterRemove: anObject

	self changeFilters: (self filters copyWithout: anObject)
! !
!SMLoaderPlus methodsFor: 'filters' stamp: 'btr 12/1/2006 01:43'!
filterSafelyAvailable
	^[:package | package isSafelyAvailable]! !
!SMLoaderPlus methodsFor: 'filter utilities' stamp: 'btr 11/30/2006 21:07'!
filterSpecs
	"Return a specification for the filter menu. Is called each time."
	| specs |
	specs := #(#('Auto-installable packages' #filterAutoInstall 'display only packages that can be installed automatically') #('New available packages' #filterAvailable 'display only packages that are not installed or that have newer releases available.') #('New safely-available packages' #filterSafelyAvailable 'display only packages that are not installed or that have newer releases available that are safe to install, meaning that they are published and meant for the current version of Squeak.') #('Installed packages' #filterInstalled 'Display only packages that are installed.') #('Published packages' #filterPublished 'Display only packages that have at least one published release.') ) asOrderedCollection.
	categoriesToFilterIds
		do: [:catId | specs add: {'Packages in ' , (map object: catId) name. catId. 'Display only packages that are in the category.'}].
	^ specs! !
!SMLoaderPlus methodsFor: 'filters' stamp: 'btr 12/1/2006 01:43'!
filterVersion
	"Ignore spaces in the version string, they're sometimes spurious.
	Not used anymore."
	^[:package | package categories anySatisfy:  
		[:cat | (cat name, '*') match: (Smalltalk version copyWithout: $ ) ]]! !
!SMLoaderPlus methodsFor: 'filter utilities' stamp: 'btr 11/22/2006 15:02'!
filters
	^filters! !
!SMLoaderPlus methodsFor: 'actions' stamp: 'btr 11/24/2006 13:49'!
findPackage: aString notifying: aView 
	"Search and select a package with the given (sub) string in the name or
	description. "
	| index list match descriptions |
	match := aString asString asLowercase.
	index := self packagesListIndex.
	list := self packageNameList.
	list isEmpty
		ifTrue: [^ self].
	descriptions := self packageList collect: [:e | e description].
	index + 1
		to: list size
		do: [:i | (((list at: i)
						includesSubstring: match
						caseSensitive: false)
					or: [(descriptions at: i)
							includesSubstring: match
							caseSensitive: false])
				ifTrue: [^ self packagesListIndex: i]].
	"wrap around"
	1
		to: index
		do: [:i | (((list at: i)
						includesSubstring: match
						caseSensitive: false)
					or: [(descriptions at: i)
							includesSubstring: match
							caseSensitive: false])
				ifTrue: [^ self packagesListIndex: i]].
	self inform: 'No package matching ' , aString asString! !
!SMLoaderPlus methodsFor: 'menus' stamp: 'btr 11/22/2006 15:02'!
generalOptions
	^#( #('Upgrade all installed packages' upgradeInstalledPackagesNoConfirm)
		#('Upgrade all installed packages confirming each' upgradeInstalledPackagesConfirm)
		#('Put list in paste buffer' listInPasteBuffer)
		#('Save filters as default' saveFiltersAsDefault)
		#- )

! !
!SMLoaderPlus methodsFor: 'private' stamp: 'btr 11/22/2006 18:36'!
hasSelectedItem
	^ self selectedPackageOrRelease notNil! !
!SMLoaderPlus methodsFor: 'interface' stamp: 'btr 12/1/2006 01:44'!
help
	"Present help text. If there is a web server available, offer to open it.
	Use the WebBrowser registry if possible, or Scamper if available."
	| message browserClass |
	message := 'Welcome to the SqueakMap package loader. 
The names of packages are followed by versions: (installed -> latest).
If there is no arrow, your installed version of the package is the latest.
Bold packages and releases have been installed.
The checkbox menu items modify which packages you''ll see.
Take a look at them - only some packages are shown initially.
The options available for a package depend on how it was packaged.
Comment on a package by emailing the author or the squeak list.'.

	browserClass := Smalltalk at: #WebBrowser ifPresent: [ :registry | registry default ].
	browserClass := browserClass ifNil: [ Smalltalk at: #Scamper ifAbsent: [ ^self inform: message ]].

	(self confirm: message, '
Would you like to view more detailed help on the SqueakMap swiki page?') 
	ifTrue: [ browserClass openOnUrl: 'http://wiki.squeak.org/2726' asUrl]! !
!SMLoaderPlus methodsFor: 'private' stamp: 'btr 11/22/2006 15:02'!
informException: ex msg: msg 
	"Tell the user that an error has occurred.
	Offer to open debug notifier."

	(self confirm: msg, 'Would you like to open a debugger?')
		ifTrue: [ex pass]! !
!SMLoaderPlus methodsFor: 'interface' stamp: 'btr 12/5/2006 05:28'!
initialExtent
	^500 at 400! !
!SMLoaderPlus methodsFor: 'actions' stamp: 'btr 11/22/2006 15:02'!
installPackageRelease
	"Install selected package or release.
	The cache is used."

	| item release |
	item := self selectedPackageOrRelease ifNil: [^ nil].
	item isPackageRelease
		ifTrue: [
			(item isPublished or: [self confirm: 'Selected release is not published yet, install anyway?'])
				ifTrue: [^self installPackageRelease: item]]
		ifFalse: [
			release := item lastPublishedReleaseForCurrentSystemVersion.
			release ifNil: [
				(self confirm: 'The package has no published release for your Squeak version, try releases for any Squeak version?')
					ifTrue: [
						release := item lastPublishedRelease.
						release ifNil: [
							(self confirm: 'The package has no published release at all, take the latest of the unpublished releases?')
								ifTrue: [release := item lastRelease]]]].
			release ifNotNil: [^self installPackageRelease: release]]! !
!SMLoaderPlus methodsFor: 'private' stamp: 'btr 12/1/2006 01:53'!
installPackageRelease: aRelease
	"Install a package release. The cache is used."

	| myRelease installer |
	aRelease isCompatibleWithCurrentSystemVersion ifFalse:
		[(self confirm:
'The package you are about to install is not listed as
being compatible with your image version (', SystemVersion current majorMinorVersion, '),
so the package may not work properly.
Do you still want to proceed with the install?')
			ifFalse: [^ self]].
	myRelease := self installedReleaseOfMe.
	installer := SMInstaller forPackageRelease: aRelease.
	[UIManager default informUser: 'Downloading ' , aRelease asString during:
		[installer download].
	UIManager default informUser: 'Installing ' , aRelease asString during: [
		installer install.
		myRelease = self installedReleaseOfMe
					ifFalse: [self reOpen]
					ifTrue: [self noteChanged]]
	] on: Error do: [:ex |
		| msg |
		msg := ex messageText ifNil:[ex asString].
		self informException: ex msg: ('Error occurred during install:\', msg, '\') withCRs].! !
!SMLoaderPlus methodsFor: 'private' stamp: 'btr 11/22/2006 15:02'!
installedReleaseOfMe
	"Return the release of the installed package loader."

	^SMSqueakMap default installedReleaseOf: (SMSqueakMap default packageWithId: '941c0108-4039-4071-9863-a8d7d2b3d4a3').! !
!SMLoaderPlus methodsFor: 'interface' stamp: 'btr 11/24/2006 14:44'!
itemChildren: anItem 
	^ anItem isPackage
		ifTrue: [anItem releases]
		ifFalse: [#()]! !
!SMLoaderPlus methodsFor: 'private' stamp: 'btr 11/22/2006 19:56'!
itemDescription
	^ self selectedPackageOrRelease
		ifNil: ['']
		ifNotNilDo: [:item | item fullDescription]! !
!SMLoaderPlus methodsFor: 'interface' stamp: 'btr 11/24/2006 14:45'!
itemHasChildren: anItem 
	^ anItem isPackage and: [anItem releases notEmpty]! !
!SMLoaderPlus methodsFor: 'interface' stamp: 'btr 12/1/2006 01:44'!
itemLabel: anItem 
	| label |
	label := anItem isPackage
				ifTrue: [anItem name
						, (anItem versionLabel
								ifEmpty: ['']
								ifNotEmptyDo: [:lbl | ' (' , anItem versionLabel , ')'])]
				ifFalse: [anItem smartVersion].
	^ anItem isInstalled
		ifTrue: [label asText allBold]
		ifFalse: [label]! !
!SMLoaderPlus methodsFor: 'lists' stamp: 'btr 11/24/2006 17:17'!
label
	^ self
		labelForShown: (packagesList
				ifNil: [self packageList])! !
!SMLoaderPlus methodsFor: 'filter utilities' stamp: 'btr 11/22/2006 15:02'!
labelForFilter: aFilterSymbol 
	^(self filterSpecs detect: [:fs | fs second = aFilterSymbol]) first! !
!SMLoaderPlus methodsFor: 'lists' stamp: 'btr 12/1/2006 01:50'!
labelForShown: packagesShown
	"Update the label of the window."
	^ self defaultLabel , ' (',
		(packagesShown size < map packages size ifTrue: [packagesShown size printString,
		' shown out of '] ifFalse: ['']) , map packages size printString, ' packages)'! !
!SMLoaderPlus methodsFor: 'actions' stamp: 'btr 11/22/2006 15:02'!
listInPasteBuffer
	"Useful when talking with people etc.
	Uses the map to produce a nice String."

	Clipboard clipboardText:
		(String streamContents: [:s |
			packagesList do: [:p |
				s nextPutAll: p nameWithVersionLabel; cr ]]) asText! !
!SMLoaderPlus methodsFor: 'actions' stamp: 'btr 12/1/2006 01:31'!
loadUpdates
	[UIManager default informUser: 'Loading Updates' during: [
		map loadUpdates.
		self noteChanged ]
	] on: Error do: [:ex |
		self informException: ex msg: ('Error occurred when updating map:\', ex messageText, '\') withCRs]! !
!SMLoaderPlus methodsFor: 'private' stamp: 'btr 11/24/2006 14:05'!
noteChanged
	filters
		ifNil: [^ self reOpen].
	map
		ifNotNil: [packagesList := nil.
			selectedCategory := nil.
			self changed: #categoryList.
			self changed: #packageList.
			self changed: #packagesListIndex.
			"update my selection"
			self contentsChanged]! !
!SMLoaderPlus methodsFor: 'initialization' stamp: 'btr 11/22/2006 16:11'!
on: aSqueakMap 
	"Initialize instance."

	map := aSqueakMap.
	map synchWithDisk.
	filters := DefaultFilters copy.
	categoriesToFilterIds := DefaultCategoriesToFilterIds copy.
	self askToLoadUpdates! !
!SMLoaderPlus methodsFor: 'filter utilities' stamp: 'btr 11/22/2006 15:02'!
package: aPackage filteredByCategory: aCategory
	"Answer true if the package should be shown
	if we filter on . It should be shown
	if itself or any of its releases has the category."

	| releases |
	releases := aPackage releases.
	^(aPackage hasCategoryOrSubCategoryOf: aCategory) or: [
			releases anySatisfy: [:rel |
				rel hasCategoryOrSubCategoryOf: aCategory]]! !
!SMLoaderPlus methodsFor: 'lists' stamp: 'btr 12/1/2006 01:49'!
packageList
	"Return a list of the SMPackages that should be visible
	by applying all the filters. Also filter based on the currently
	selected category - if any."
	| list |
	list := packagesList ifNil: [packagesList := self packageListCalculated].
	selectedCategory ifNotNil: [
		list := list select: [:each | self package: each filteredByCategory: selectedCategory]].
	self updateLabel: list.
	^ list! !
!SMLoaderPlus methodsFor: 'lists' stamp: 'btr 12/1/2006 01:49'!
packageListCalculated
	"Return a list of the SMPackages that should be visible
	by applying all the filters. Also filter based on the currently
	selected category - if any."
	^ self packages select: [:p |
		filters allSatisfy: [:currFilter |
			currFilter isSymbol
				ifTrue: [(self perform: currFilter) value: p]
				ifFalse: [self package: p filteredByCategory: (map object: currFilter)]]]! !
!SMLoaderPlus methodsFor: 'lists' stamp: 'btr 12/1/2006 01:50'!
packageNameList
	^ self packageList collect: [:e | e name]! !
!SMLoaderPlus methodsFor: 'menus' stamp: 'btr 11/22/2006 18:30'!
packageSpecificOptions
	| choices packageOrRelease |
	packageOrRelease := self selectedPackageOrRelease.
	choices := OrderedCollection new.
	packageOrRelease isInstallable ifTrue: [
		choices add: (self commandSpecFor: #installPackageRelease)].
	(packageOrRelease isDownloadable and: [packageOrRelease isCached]) ifTrue: [
		choices add: (self commandSpecFor: #browseCacheDirectory)].
	(packageOrRelease isPackageRelease and: [packageOrRelease isDownloadable]) ifTrue: [
		choices add: (self commandSpecFor: #cachePackageReleaseAndOfferToCopy).
		choices add: (self commandSpecFor: #downloadPackageRelease)].
	choices add: (self commandSpecFor: #emailPackageMaintainers).
	^ choices! !
!SMLoaderPlus methodsFor: 'private' stamp: 'btr 11/22/2006 16:11'!
packages
	"We request the packages as sorted by name by default."

	^map packagesByName asArray
! !
!SMLoaderPlus methodsFor: 'accessing' stamp: 'btr 11/24/2006 14:01'!
packagesListIndex
	^ self packageList indexOf: self selectedItem! !
!SMLoaderPlus methodsFor: 'accessing' stamp: 'btr 11/24/2006 14:01'!
packagesListIndex: anObject 
	self
		selectedItem: (anObject = 0
				ifFalse: [self packageList at: anObject])! !
!SMLoaderPlus methodsFor: 'menus' stamp: 'btr 11/22/2006 15:02'!
packagesMenu: aMenu 
	"Answer the packages-list menu."

	self selectedPackageOrRelease 
		ifNotNil: [aMenu addList: self packageSpecificOptions; addLine].
	aMenu addList: self generalOptions.
	self addFiltersToMenu: aMenu.
	^aMenu! !
!SMLoaderPlus methodsFor: 'interface' stamp: 'btr 12/1/2006 01:45'!
perform: selector orSendTo: otherTarget 
	"Selector was just chosen from a menu by a user. If can respond, then  
	perform it on myself. If not, send it to otherTarget, presumably the  
	editPane from which the menu was invoked."

	^ (self respondsTo: selector)
		ifTrue: [self perform: selector]
		ifFalse: [super perform: selector orSendTo: otherTarget]! !
!SMLoaderPlus methodsFor: 'private' stamp: 'btr 11/26/2006 23:22'!
reOpen
	"Close this package loader, probably because it has been updated,
	and open a new one."
	self inform: 'This package loader has been upgraded and will be closed and reopened to avoid strange side effects.'.
	window delete.
	(Smalltalk at: self class name) open! !
!SMLoaderPlus methodsFor: 'actions' stamp: 'btr 11/22/2006 15:02'!
removeCategoryFilters
	"Remove all category filters."

	categoriesToFilterIds := OrderedCollection new! !
!SMLoaderPlus methodsFor: 'actions' stamp: 'btr 11/22/2006 15:02'!
removeSelectedCategoryAsFilter
	"Remove the filter that filters on the currently selected category."

	categoriesToFilterIds remove: self selectedCategory id! !
!SMLoaderPlus methodsFor: 'actions' stamp: 'btr 11/22/2006 15:02'!
saveFiltersAsDefault
	"Save the current filters as default so that they
	are selected the next time the loader is opened."

	DefaultFilters := filters copy.
	DefaultCategoriesToFilterIds := categoriesToFilterIds copy! !
!SMLoaderPlus methodsFor: 'interface' stamp: 'btr 11/24/2006 14:35'!
searchSelection
	"Selects all of the default search text so that a type-in overwrites it."
	^ {1. self searchText size}! !
!SMLoaderPlus methodsFor: 'interface' stamp: 'btr 11/24/2006 14:35'!
searchText
	"A dummy default search text so that the field describes its purpose."
	^ 'Search packages'! !
!SMLoaderPlus methodsFor: 'accessing' stamp: 'btr 11/24/2006 14:02'!
selectedCategory
	"Return selected category."
	^ selectedCategory! !
!SMLoaderPlus methodsFor: 'accessing' stamp: 'btr 12/1/2006 16:37'!
selectedCategory: anSMCategory 
	"Change the selected category."
	selectedCategory := anSMCategory.
	selectedCategory
		ifNotNil: [(selectedCategory objects includes: self selectedItem)
			ifFalse: [self selectedItem: nil]].
	self changed: #selectedCategory.
	self changed: #packageList! !
!SMLoaderPlus methodsFor: 'accessing' stamp: 'btr 11/24/2006 14:52'!
selectedCategoryPath
	"Return selected category's path."
	| path |
	path := #().
	selectedCategory
		ifNotNil: [selectedCategory parent
				ifNotNilDo: [:p | path := path copyWith: p].
			path := path copyWith: selectedCategory].
	^ path
		collect: [:cat | self categoryLabel: cat]! !
!SMLoaderPlus methodsFor: 'accessing' stamp: 'btr 11/24/2006 14:02'!
selectedItem
	^ selectedItem! !
!SMLoaderPlus methodsFor: 'accessing' stamp: 'btr 12/1/2006 16:27'!
selectedItem: anItem
	"This == workaround protects us from recursion since ToolBuilder's tree widgets will always tell us that the selection has been updated when we tell it that the selection path has been updated. Cleaner solutions invited."
	anItem == selectedItem ifFalse: [
		selectedItem := anItem.
		self changed: #selectedItemPath.
		self changed: #itemDescription.
		self changed: #hasSelectedItem]! !
!SMLoaderPlus methodsFor: 'accessing' stamp: 'btr 12/1/2006 16:16'!
selectedItemPath
	| path |
	path := #().
	(selectedItem isKindOf: SMPackageRelease)
		ifTrue: [path := path copyWith: selectedItem package].
	selectedItem
		ifNotNil: [path := path copyWith: selectedItem].
	^ path! !
!SMLoaderPlus methodsFor: 'accessing' stamp: 'btr 11/24/2006 14:03'!
selectedPackageOrRelease
	"Return selected package or package release."
	^ selectedItem! !
!SMLoaderPlus methodsFor: 'filter utilities' stamp: 'btr 11/22/2006 15:02'!
showFilterString: aFilterSymbol 
	^(self stateForFilter: aFilterSymbol), (self labelForFilter: aFilterSymbol)! !
!SMLoaderPlus methodsFor: 'filter utilities' stamp: 'btr 11/22/2006 15:02'!
stateForFilter: aFilterSymbol 
	^(self filters includes: aFilterSymbol) ifTrue: [''] ifFalse: ['']

! !
!SMLoaderPlus methodsFor: 'filter utilities' stamp: 'btr 11/22/2006 15:02'!
toggleFilterState: aFilterSymbol 

	^(self filters includes: (aFilterSymbol)) 
		ifTrue: [self filterRemove: aFilterSymbol]
		ifFalse: [self filterAdd: aFilterSymbol]! !
!SMLoaderPlus methodsFor: 'actions' stamp: 'btr 11/22/2006 15:02'!
uncheckFilters
	"Uncheck all filters."
	
	filters := OrderedCollection new.
	self noteChanged! !
!SMLoaderPlus methodsFor: 'lists' stamp: 'btr 12/1/2006 01:50'!
updateLabel: packagesShown
	"Update the label of the window."
	window ifNotNilDo: [:w | w setLabel: (self labelForShown: packagesShown)]! !
!SMLoaderPlus methodsFor: 'actions' stamp: 'btr 12/1/2006 01:29'!
upgradeInstalledPackages
	"Tries to upgrade all installed packages to the latest published release for this
	version of Squeak. So this is a conservative approach."

	| installed old myRelease toUpgrade info |
	installed := map installedPackages.
	old := map oldPackages.
	old isEmpty ifTrue: [
			^self inform: 'All ', installed size printString, ' installed packages are up to date.'].
	toUpgrade := map upgradeableAndOldPackages.
	toUpgrade isEmpty ifTrue: [
			^self inform: 'None of the ', old size printString, ' old packages of the ', installed size printString, ' installed can be automatically upgraded. You need to upgrade them manually.'].
	info := old size < toUpgrade size ifTrue: [
		'Of the ', old size printString, ' old packages only ', toUpgrade size printString, ' can be upgraded.
The following packages will not be upgraded:
',  (String streamContents: [:s | (old removeAll: toUpgrade; yourself)
	do: [:p | s nextPutAll: p nameWithVersionLabel; cr]])]
		ifFalse: ['All old packages upgradeable.'].
	(self confirm: info, '
About to upgrade the following packages:
', (String streamContents: [:s | toUpgrade do: [:p | s nextPutAll: p nameWithVersionLabel; cr]]), 'Proceed?') ifTrue: [
			myRelease := self installedReleaseOfMe.
			[UIManager default informUser: 'Upgrading Installed Packages' during: [
				map upgradeOldPackages.
				self inform: toUpgrade size printString, ' packages successfully upgraded.'.
				myRelease = self installedReleaseOfMe
					ifFalse: [self reOpen]
					ifTrue: [self noteChanged]]
			] on: Error do: [:ex |
				self informException: ex msg: ('Error occurred when upgrading old packages:\', ex messageText, '\') withCRs]]! !
!SMLoaderPlus methodsFor: 'actions' stamp: 'btr 11/22/2006 15:02'!
upgradeInstalledPackagesConfirm
	"Tries to upgrade all installed packages to the latest published release for this
	version of Squeak. Confirms on each upgrade."

	^ self upgradeInstalledPackagesConfirm: true! !
!SMLoaderPlus methodsFor: 'private' stamp: 'btr 12/1/2006 01:29'!
upgradeInstalledPackagesConfirm: confirmEach 
	"Tries to upgrade all installed packages to the latest published release for
	this version of Squeak. If confirmEach is true we ask for every
	upgrade. "
	| installed old myRelease toUpgrade info |
	installed := map installedPackages.
	old := map oldPackages.
	old isEmpty
		ifTrue: [^ self inform: 'All ' , installed size printString , ' installed packages are up to date.'].
	toUpgrade := map upgradeableAndOldPackages.
	toUpgrade isEmpty
		ifTrue: [^ self inform: 'None of the ' , old size printString , ' old packages of the ' , installed size printString , ' installed can be automatically upgraded. You need to upgrade them manually.'].
	info := old size < toUpgrade size
		ifTrue: ['Of the ' , old size printString , ' old packages only ' , toUpgrade size printString , ' can be upgraded.
The following packages will not be upgraded:
'
						, (String
								streamContents: [:s | (old removeAll: toUpgrade;
										 yourself)
										do: [:p | s nextPutAll: p nameWithVersionLabel;
												 cr]])]
		ifFalse: ['All old packages upgradeable.'].
	(self confirm: info , '
About to upgrade the following packages:
'
				, (String
						streamContents: [:s | toUpgrade
								do: [:p | s nextPutAll: p nameWithVersionLabel;
										 cr]]) , 'Proceed?')
		ifTrue: [myRelease := self installedReleaseOfMe.
			[UIManager default informUser: 'Upgrading Installed Packages' during:
					[confirmEach
						ifTrue: [map
								upgradeOldPackagesConfirmBlock: [:p | self confirm: 'Upgrade ' , p installedRelease packageNameWithVersion , ' to ' , (p lastPublishedReleaseForCurrentSystemVersionNewerThan: p installedRelease) listName , '?']]
						ifFalse: [map upgradeOldPackages].
					self inform: toUpgrade size printString , ' packages successfully processed.'.
					myRelease = self installedReleaseOfMe
						ifTrue: [self noteChanged]
						ifFalse: [self reOpen]]]
				on: Error
				do: [:ex | self informException: ex msg: ('Error occurred when upgrading old packages:\' , ex messageText , '\') withCRs]]! !
!SMLoaderPlus methodsFor: 'actions' stamp: 'btr 11/22/2006 15:02'!
upgradeInstalledPackagesNoConfirm
	"Tries to upgrade all installed packages to the latest published release for this
	version of Squeak. No confirmation on each upgrade."

	^ self upgradeInstalledPackagesConfirm: false! !
!SMPackageWrapper methodsFor: 'comparing' stamp: 'dvf 9/21/2003 16:25' prior: 27998626!
= anObject
	^self withoutListWrapper = anObject withoutListWrapper! !
!SMPackageWrapper methodsFor: 'converting' stamp: 'btr 11/22/2006 00:54' prior: 27998778!
asString
	| string |
	string := item name, ' (', item versionLabel, ')'.
	item isInstalled ifTrue: [string := string asText allBold].
	"(string includesSubString: '->') ifTrue: [string := string asText color: Color green]."
	^ string! !
!SMPackageWrapper methodsFor: 'accessing' stamp: 'dvf 10/14/2003 18:58' prior: 27998902!
contents
	^item releases reversed collect: [:e | SMPackageReleaseWrapper with: e]! !
!SMPackageWrapper methodsFor: 'testing' stamp: 'dvf 9/21/2003 16:25' prior: 27999070!
hash
	^self withoutListWrapper hash! !
!SMPackageWrapper methodsFor: 'accessing' stamp: 'btr 11/22/2006 16:55'!
help
	^ 'This shows all packages with their releases that should be displayed according the current filter.'! !
!SMPackageWrapper methodsFor: 'accessing' stamp: 'btr 11/22/2006 16:49'!
label
	^ self asString! !
!SMPackageWrapper methodsFor: 'printing' stamp: 'dvf 9/21/2003 16:22' prior: 27999192!
printOn: aStream
	aStream nextPutAll: 'wrapper for: ', item printString! !
!SMCategoryWrapper methodsFor: 'comparing' stamp: 'ar 2/9/2004 02:13' prior: 27849043!
= anObject
	^self withoutListWrapper = anObject withoutListWrapper! !
!SMCategoryWrapper methodsFor: 'converting' stamp: 'btr 11/30/2006 18:53' prior: 27849195!
asString
	^ item name , ' (' , self numberOfObjects printString , ')'! !
!SMCategoryWrapper methodsFor: 'accessing' stamp: 'ar 2/9/2004 02:35' prior: 27849301!
category
	^item! !
!SMCategoryWrapper methodsFor: 'accessing' stamp: 'btr 11/30/2006 21:02' prior: 27849402!
contents
	^ item subCategories
		collect: [:n | self class with: n model: n]! !
!SMCategoryWrapper methodsFor: 'model access' stamp: 'btr 11/30/2006 21:02'!
getList
	^ Array
		with: (self class with: self contents model: model)! !
!SMCategoryWrapper methodsFor: 'testing' stamp: 'btr 11/30/2006 18:53'!
hasContents
	^ item hasSubCategories! !
!SMCategoryWrapper methodsFor: 'comparing' stamp: 'ar 2/9/2004 02:13' prior: 27849700!
hash
	^self withoutListWrapper hash! !
!SMCategoryWrapper methodsFor: 'accessing' stamp: 'btr 11/22/2006 16:56'!
help
	^ 'The categories are structured in a tree. Packages and package releases belong to several categories. You can add one or more categories as filters and enable them in the menu.'! !
!SMCategoryWrapper methodsFor: 'accessing' stamp: 'BJP 11/22/2002 14:17'!
model
	^model! !
!SMCategoryWrapper methodsFor: 'accessing' stamp: 'btr 11/30/2006 18:53'!
numberOfObjects
"	| total |
	total _ 0.
	model allCategoriesDo: [:c |
		total _ total + c objects size].
	^total"
	^item objects size! !
!SMPackageReleaseWrapper methodsFor: 'converting' stamp: 'btr 11/30/2006 21:30' prior: 27997393!
asString
	"Show installed releases with a trailing asterisk."
	| string |
	string := item smartVersion.
	"Older SMBase versions don't have isInstalled.'"
	(item respondsTo: #isInstalled) ifTrue:
		[item isInstalled ifTrue: [string := (string , ' *') asText allBold]].
	^ string! !
!SMPackageReleaseWrapper methodsFor: 'accessing' stamp: 'btr 11/22/2006 17:14'!
contents
	^ #()! !
!SMPackageReleaseWrapper methodsFor: 'accessing' stamp: 'btr 11/22/2006 16:49'!
label
	^ self asString
	! !
!SMLoader class methodsFor: 'class initialization' stamp: 'btr 12/1/2006 15:47' prior: 27944626!
initialize
	"Hook us up in the world menu."
	"self initialize"
	Smalltalk
		at: #ToolBuilder
		ifAbsent: [self registerInFlapsRegistry.
			(Preferences windowColorFor: #SMLoader) = Color white
				ifTrue: ["not set"
					Preferences
						setWindowColorFor: #SMLoader
						to: (Color colorFrom: self windowColorSpecification brightColor)].
			(TheWorldMenu respondsTo: #registerOpenCommand:)
				ifTrue: [| oldCmds |
					oldCmds := TheWorldMenu registry select: [:cmd | cmd first includesSubString: 'Package Loader'].
					oldCmds do: [:cmd | TheWorldMenu unregisterOpenCommand: cmd first].
					TheWorldMenu registerOpenCommand: {self openMenuString. {self. #open}}]].
	DefaultFilters := OrderedCollection new.
	DefaultCategoriesToFilterIds := OrderedCollection new! !
!SMLoader class methodsFor: 'class initialization' stamp: 'btr 11/30/2006 21:52'!
openMenuString
	^ 'SqueakMap Catalog'! !
!SMLoader class methodsFor: 'class initialization' stamp: 'btr 11/30/2006 21:52' prior: 27945298!
unload
	(TheWorldMenu respondsTo: #registerOpenCommand:) ifTrue: 
		[TheWorldMenu unregisterOpenCommand: self openMenuString].
	self environment at: #Flaps ifPresent: [:cl |
	cl unregisterQuadsWithReceiver: self] ! !
!SMLoader methodsFor: 'menus' stamp: 'btr 11/21/2006 16:08' prior: 54331069!
addFiltersToMenu: aMenu
	| filterSymbol help |
	self filterSpecs do: [:filterArray | 
		filterSymbol := filterArray second.
		help := filterArray third.
		aMenu addUpdating: #showFilterString: target: self selector: #toggleFilterState: argumentList: (Array with: filterSymbol).
		aMenu balloonTextForLastItem: help].
	aMenu addLine;
		addList: #(('Clear all filters' uncheckFilters 'Unchecks all filters to list all packages'))
	! !
!SMLoader methodsFor: 'interface' stamp: 'btr 11/22/2006 01:15' prior: 27927912!
browseCacheDirectory
	"Open a FileList2 on the directory for the package or release."

	| item dir win |
	item := self selectedPackageOrRelease ifNil: [^ nil].
	item ifNil: [^nil].
	dir := item isPackage
				ifTrue: [model cache directoryForPackage: item]
				ifFalse: [model cache directoryForPackageRelease: item].
	win := FileList2 morphicViewOnDirectory: dir. " withLabel: item name, ' cache directory'."
	win openInWorld
! !
!SMLoader methodsFor: 'interface' stamp: 'btr 11/22/2006 14:52'!
buildButtonBar
	| aRow btn |
	aRow := AlignmentMorph newRow beSticky.
	aRow color: Color transparent;
		clipSubmorphs: true.
	self buttonSpecs do: [:spec |
		btn := self buildButtonNamed: spec first helpText: spec third action: spec second.
		aRow addMorphBack: btn]
		separatedBy: [aRow addTransparentSpacerOfSize: 3 at 0].
	^ aRow! !
!SMLoader methodsFor: 'interface' stamp: 'btr 11/22/2006 01:27'!
buildButtonNamed: labelText helpText: balloon action: action
	| btn |
	btn := PluggableButtonMorph on: self getState: nil action: action.
	btn color: Color transparent;
		hResizing: #shrinkWrap;
		vResizing: #spaceFill;
		label: labelText;
		setBalloonText: balloon;
		onColor: Color transparent offColor: Color transparent.
	^ btn! !
!SMLoader methodsFor: 'interface' stamp: 'btr 11/30/2006 19:04' prior: 27928394!
buildMorphicCategoriesList
	"Create the hierarchical list holding the category tree."
	| list |

From noreply at buildbot.pypy.org  Thu Nov 13 15:51:29 2014
From: noreply at buildbot.pypy.org (anton_gulenko)
Date: Thu, 13 Nov 2014 15:51:29 +0100 (CET)
Subject: [pypy-commit] lang-smalltalk default: Changed README to windows
	line endings.
Message-ID: <20141113145129.EEE501C3140@cobra.cs.uni-duesseldorf.de>

Author: Anton Gulenko 
Branch: 
Changeset: r1068:88a77f712b8d
Date: 2014-11-13 15:51 +0100
http://bitbucket.org/pypy/lang-smalltalk/changeset/88a77f712b8d/

Log:	Changed README to windows line endings.

diff --git a/README.md b/README.md
--- a/README.md
+++ b/README.md
@@ -1,89 +1,89 @@
-RSqueak
-=========
-
-A Squeak VM written in RPython.
-
-Setup
-----
-
-### Required Projects
-You need three repositories: 
-* [This one](https://bitbucket.org/pypy/lang-smalltalk)
-* [pypy/pypy](https://bitbucket.org/pypy/pypy)
-* [pypy/rsdl](https://bitbucket.org/pypy/rsdl)
-    * Alternatively download RSDL package from [PYPI](https://pypi.python.org/pypi/rsdl)
-    * Then unpack and install it using ```python setup.py install``` or ```pypy setup.py install```
-
-### Required packages
-You need the following packages on your OS. Install with your favorite package manager:
-* pypy
-    * For faster translation of the RSqueak VM. Alternatively use default Python.
-* libsdl-dev
-* libffi-dev
-
-### Adjusting the PYTHONPATH
-In order to allow the RPython toolchain to find the rsdl and pypy packages you have to add the two folders to the PYTHONPATH.
-
-If you are using the rsdl *repository*, you have to add the rsdl subfolder of the rsdl repository to the PYTHONPATH.
-
-```
-export PYTHONPATH=${PYTHONPATH}:.:[path/to/pypy]:[path/to/rsdl]
-```
-
-### Setting the SDL Driver
-For testing the basic functionality of the VM you might want to disable the UI. You can do so by setting the SDL_VIDEODRIVER environment variable to dummy.
-
-```
-export SDL_VIDEODRIVER=dummy
-```
-
-### Building & Tests
-Execute the following commands inside the main directory of this repository.
-
-To build the VM:
-
-```
-[path to pypy repository]/rpython/bin/rpython targetimageloadingsmalltalk.py
-```
-
-To build the VM with enabled just-in-time compiler:
-
-```
-[path to pypy repository]/rpython/bin/rpython -Ojit targetimageloadingsmalltalk.py
-```
-
-To run the tests (most do not require building):
-
-```
-[path to pypy repository]/pytest.py [--slow|--quick] spyvm/test
-```
-
-### Starting an image
-The build process will produce an executable called rsqueak.
-The ```image/``` directory contains two images you can open with the following.
-Use ```--help``` to see command line switches.
-
-```
-./rsqueak images/mini.image
-./rsqueak images/Squeak4.5-noBitBlt.image
-```
-
-
-
-
-STM-enabled Rsqueak
-===
-This is a branch of RSqueak which incorporates the RPython STM transformation. Most of the initial code base comes from the results of a project seminar (https://bitbucket.org/amintos/lang-smalltalk). The stmgc-c7 branch is based on this version and the 64bit branch.
-
-Setup for stm-enabled RSqueak
----
-You can see the current state of the integration of the RPython STM in our stmgc-c7 branch.
-Beware that you can only build this branch if you have a 64-bit linux. To build this branch you have to setup several things:
-
-1. Change your local pypy repository to the stm-gc7 branch, commit dd3c06b
-2. Get a clang which has the patches from ([Clang patches](https://bitbucket.org/pypy/stmgc/src/d164a5bcad5e7615b4362b6a1a49d51e2e06de0c/c7/llvmfix/?at=default)). If you have a Debian-based OS you can use the following package: [llvm-pypystm](https://launchpad.net/~malte.swart/+archive/ubuntu/llvm-pypystm).
-
-To build, use the following command:
-```
-[path to pypy repository]/rpython/bin/rpython --gc=stmgc targetimageloadingsmalltalk.py
-```
+RSqueak
+=========
+
+A Squeak VM written in RPython.
+
+Setup
+----
+
+### Required Projects
+You need three repositories: 
+* [This one](https://bitbucket.org/pypy/lang-smalltalk)
+* [pypy/pypy](https://bitbucket.org/pypy/pypy)
+* [pypy/rsdl](https://bitbucket.org/pypy/rsdl)
+    * Alternatively download RSDL package from [PYPI](https://pypi.python.org/pypi/rsdl)
+    * Then unpack and install it using ```python setup.py install``` or ```pypy setup.py install```
+
+### Required packages
+You need the following packages on your OS. Install with your favorite package manager:
+* pypy
+    * For faster translation of the RSqueak VM. Alternatively use default Python.
+* libsdl-dev
+* libffi-dev
+
+### Adjusting the PYTHONPATH
+In order to allow the RPython toolchain to find the rsdl and pypy packages you have to add the two folders to the PYTHONPATH.
+
+If you are using the rsdl *repository*, you have to add the rsdl subfolder of the rsdl repository to the PYTHONPATH.
+
+```
+export PYTHONPATH=${PYTHONPATH}:.:[path/to/pypy]:[path/to/rsdl]
+```
+
+### Setting the SDL Driver
+For testing the basic functionality of the VM you might want to disable the UI. You can do so by setting the SDL_VIDEODRIVER environment variable to dummy.
+
+```
+export SDL_VIDEODRIVER=dummy
+```
+
+### Building & Tests
+Execute the following commands inside the main directory of this repository.
+
+To build the VM:
+
+```
+[path to pypy repository]/rpython/bin/rpython targetimageloadingsmalltalk.py
+```
+
+To build the VM with enabled just-in-time compiler:
+
+```
+[path to pypy repository]/rpython/bin/rpython -Ojit targetimageloadingsmalltalk.py
+```
+
+To run the tests (most do not require building):
+
+```
+[path to pypy repository]/pytest.py [--slow|--quick] spyvm/test
+```
+
+### Starting an image
+The build process will produce an executable called rsqueak.
+The ```image/``` directory contains two images you can open with the following.
+Use ```--help``` to see command line switches.
+
+```
+./rsqueak images/mini.image
+./rsqueak images/Squeak4.5-noBitBlt.image
+```
+
+
+
+
+STM-enabled Rsqueak
+===
+This is a branch of RSqueak which incorporates the RPython STM transformation. Most of the initial code base comes from the results of a project seminar (https://bitbucket.org/amintos/lang-smalltalk). The stmgc-c7 branch is based on this version and the 64bit branch.
+
+Setup for stm-enabled RSqueak
+---
+You can see the current state of the integration of the RPython STM in our stmgc-c7 branch.
+Beware that you can only build this branch if you have a 64-bit linux. To build this branch you have to setup several things:
+
+1. Change your local pypy repository to the stm-gc7 branch, commit dd3c06b
+2. Get a clang which has the patches from ([Clang patches](https://bitbucket.org/pypy/stmgc/src/d164a5bcad5e7615b4362b6a1a49d51e2e06de0c/c7/llvmfix/?at=default)). If you have a Debian-based OS you can use the following package: [llvm-pypystm](https://launchpad.net/~malte.swart/+archive/ubuntu/llvm-pypystm).
+
+To build, use the following command:
+```
+[path to pypy repository]/rpython/bin/rpython --gc=stmgc targetimageloadingsmalltalk.py
+```

From noreply at buildbot.pypy.org  Thu Nov 13 15:54:37 2014
From: noreply at buildbot.pypy.org (anton_gulenko)
Date: Thu, 13 Nov 2014 15:54:37 +0100 (CET)
Subject: [pypy-commit] lang-smalltalk default: Trying to fix lists in README
	markdown.
Message-ID: <20141113145437.9132F1C3140@cobra.cs.uni-duesseldorf.de>

Author: Anton Gulenko 
Branch: 
Changeset: r1069:f78652a865c8
Date: 2014-11-13 15:54 +0100
http://bitbucket.org/pypy/lang-smalltalk/changeset/f78652a865c8/

Log:	Trying to fix lists in README markdown.

diff --git a/README.md b/README.md
--- a/README.md
+++ b/README.md
@@ -8,18 +8,18 @@
 
 ### Required Projects
 You need three repositories: 
-* [This one](https://bitbucket.org/pypy/lang-smalltalk)
-* [pypy/pypy](https://bitbucket.org/pypy/pypy)
-* [pypy/rsdl](https://bitbucket.org/pypy/rsdl)
+ * [This one](https://bitbucket.org/pypy/lang-smalltalk)
+ * [pypy/pypy](https://bitbucket.org/pypy/pypy)
+ * [pypy/rsdl](https://bitbucket.org/pypy/rsdl)
     * Alternatively download RSDL package from [PYPI](https://pypi.python.org/pypi/rsdl)
     * Then unpack and install it using ```python setup.py install``` or ```pypy setup.py install```
 
 ### Required packages
 You need the following packages on your OS. Install with your favorite package manager:
-* pypy
+ * pypy
     * For faster translation of the RSqueak VM. Alternatively use default Python.
-* libsdl-dev
-* libffi-dev
+ * libsdl-dev
+ * libffi-dev
 
 ### Adjusting the PYTHONPATH
 In order to allow the RPython toolchain to find the rsdl and pypy packages you have to add the two folders to the PYTHONPATH.

From noreply at buildbot.pypy.org  Thu Nov 13 15:57:50 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 13 Nov 2014 15:57:50 +0100 (CET)
Subject: [pypy-commit] pypy default: We must also add -fPIC here (building
 --shared with shadowstack)
Message-ID: <20141113145750.EEDB51C3140@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r74505:66fa58469820
Date: 2014-11-13 15:56 +0100
http://bitbucket.org/pypy/pypy/changeset/66fa58469820/

Log:	We must also add -fPIC here (building --shared with shadowstack)

diff --git a/rpython/translator/c/genc.py b/rpython/translator/c/genc.py
--- a/rpython/translator/c/genc.py
+++ b/rpython/translator/c/genc.py
@@ -466,7 +466,10 @@
             if self.translator.platform.name == 'msvc':
                 mk.definition('DEBUGFLAGS', '-MD -Zi')
             else:
-                mk.definition('DEBUGFLAGS', '-O1 -g')
+                if self.config.translation.shared:
+                    mk.definition('DEBUGFLAGS', '-O1 -g -fPIC')
+                else:
+                    mk.definition('DEBUGFLAGS', '-O1 -g')
         if self.translator.platform.name == 'msvc':
             mk.rule('debug_target', 'debugmode_$(DEFAULT_TARGET)', 'rem')
         else:

From noreply at buildbot.pypy.org  Thu Nov 13 15:57:52 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 13 Nov 2014 15:57:52 +0100 (CET)
Subject: [pypy-commit] pypy stmgc-c7: We must also add -fPIC here (building
 --shared with shadowstack)
Message-ID: <20141113145752.53EC71C3140@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: stmgc-c7
Changeset: r74506:a923842ee580
Date: 2014-11-13 15:56 +0100
http://bitbucket.org/pypy/pypy/changeset/a923842ee580/

Log:	We must also add -fPIC here (building --shared with shadowstack)

diff --git a/rpython/translator/c/genc.py b/rpython/translator/c/genc.py
--- a/rpython/translator/c/genc.py
+++ b/rpython/translator/c/genc.py
@@ -543,7 +543,10 @@
             if self.translator.platform.name == 'msvc':
                 mk.definition('DEBUGFLAGS', '-MD -Zi')
             else:
-                mk.definition('DEBUGFLAGS', '-O1 -g')
+                if self.config.translation.shared:
+                    mk.definition('DEBUGFLAGS', '-O1 -g -fPIC')
+                else:
+                    mk.definition('DEBUGFLAGS', '-O1 -g')
         if self.translator.platform.name == 'msvc':
             mk.rule('debug_target', 'debugmode_$(DEFAULT_TARGET)', 'rem')
         else:

From noreply at buildbot.pypy.org  Thu Nov 13 15:57:53 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 13 Nov 2014 15:57:53 +0100 (CET)
Subject: [pypy-commit] pypy default: merge heads
Message-ID: <20141113145753.98CB91C3140@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r74507:b8aadbe8980b
Date: 2014-11-13 15:57 +0100
http://bitbucket.org/pypy/pypy/changeset/b8aadbe8980b/

Log:	merge heads

diff --git a/rpython/translator/c/genc.py b/rpython/translator/c/genc.py
--- a/rpython/translator/c/genc.py
+++ b/rpython/translator/c/genc.py
@@ -466,7 +466,10 @@
             if self.translator.platform.name == 'msvc':
                 mk.definition('DEBUGFLAGS', '-MD -Zi')
             else:
-                mk.definition('DEBUGFLAGS', '-O1 -g')
+                if self.config.translation.shared:
+                    mk.definition('DEBUGFLAGS', '-O1 -g -fPIC')
+                else:
+                    mk.definition('DEBUGFLAGS', '-O1 -g')
         if self.translator.platform.name == 'msvc':
             mk.rule('debug_target', 'debugmode_$(DEFAULT_TARGET)', 'rem')
         else:

From noreply at buildbot.pypy.org  Thu Nov 13 16:01:48 2014
From: noreply at buildbot.pypy.org (anton_gulenko)
Date: Thu, 13 Nov 2014 16:01:48 +0100 (CET)
Subject: [pypy-commit] lang-smalltalk default: Unix line endings.
Message-ID: <20141113150148.C5ABF1C3140@cobra.cs.uni-duesseldorf.de>

Author: Anton Gulenko 
Branch: 
Changeset: r1070:7733e8653fd8
Date: 2014-11-13 15:56 +0100
http://bitbucket.org/pypy/lang-smalltalk/changeset/7733e8653fd8/

Log:	Unix line endings.

diff --git a/README.md b/README.md
--- a/README.md
+++ b/README.md
@@ -1,89 +1,89 @@
-RSqueak
-=========
-
-A Squeak VM written in RPython.
-
-Setup
-----
-
-### Required Projects
-You need three repositories: 
- * [This one](https://bitbucket.org/pypy/lang-smalltalk)
- * [pypy/pypy](https://bitbucket.org/pypy/pypy)
- * [pypy/rsdl](https://bitbucket.org/pypy/rsdl)
-    * Alternatively download RSDL package from [PYPI](https://pypi.python.org/pypi/rsdl)
-    * Then unpack and install it using ```python setup.py install``` or ```pypy setup.py install```
-
-### Required packages
-You need the following packages on your OS. Install with your favorite package manager:
- * pypy
-    * For faster translation of the RSqueak VM. Alternatively use default Python.
- * libsdl-dev
- * libffi-dev
-
-### Adjusting the PYTHONPATH
-In order to allow the RPython toolchain to find the rsdl and pypy packages you have to add the two folders to the PYTHONPATH.
-
-If you are using the rsdl *repository*, you have to add the rsdl subfolder of the rsdl repository to the PYTHONPATH.
-
-```
-export PYTHONPATH=${PYTHONPATH}:.:[path/to/pypy]:[path/to/rsdl]
-```
-
-### Setting the SDL Driver
-For testing the basic functionality of the VM you might want to disable the UI. You can do so by setting the SDL_VIDEODRIVER environment variable to dummy.
-
-```
-export SDL_VIDEODRIVER=dummy
-```
-
-### Building & Tests
-Execute the following commands inside the main directory of this repository.
-
-To build the VM:
-
-```
-[path to pypy repository]/rpython/bin/rpython targetimageloadingsmalltalk.py
-```
-
-To build the VM with enabled just-in-time compiler:
-
-```
-[path to pypy repository]/rpython/bin/rpython -Ojit targetimageloadingsmalltalk.py
-```
-
-To run the tests (most do not require building):
-
-```
-[path to pypy repository]/pytest.py [--slow|--quick] spyvm/test
-```
-
-### Starting an image
-The build process will produce an executable called rsqueak.
-The ```image/``` directory contains two images you can open with the following.
-Use ```--help``` to see command line switches.
-
-```
-./rsqueak images/mini.image
-./rsqueak images/Squeak4.5-noBitBlt.image
-```
-
-
-
-
-STM-enabled Rsqueak
-===
-This is a branch of RSqueak which incorporates the RPython STM transformation. Most of the initial code base comes from the results of a project seminar (https://bitbucket.org/amintos/lang-smalltalk). The stmgc-c7 branch is based on this version and the 64bit branch.
-
-Setup for stm-enabled RSqueak
----
-You can see the current state of the integration of the RPython STM in our stmgc-c7 branch.
-Beware that you can only build this branch if you have a 64-bit linux. To build this branch you have to setup several things:
-
-1. Change your local pypy repository to the stm-gc7 branch, commit dd3c06b
-2. Get a clang which has the patches from ([Clang patches](https://bitbucket.org/pypy/stmgc/src/d164a5bcad5e7615b4362b6a1a49d51e2e06de0c/c7/llvmfix/?at=default)). If you have a Debian-based OS you can use the following package: [llvm-pypystm](https://launchpad.net/~malte.swart/+archive/ubuntu/llvm-pypystm).
-
-To build, use the following command:
-```
-[path to pypy repository]/rpython/bin/rpython --gc=stmgc targetimageloadingsmalltalk.py
-```
+RSqueak
+=========
+
+A Squeak VM written in RPython.
+
+Setup
+----
+
+### Required Projects
+You need three repositories: 
+ * [This one](https://bitbucket.org/pypy/lang-smalltalk)
+ * [pypy/pypy](https://bitbucket.org/pypy/pypy)
+ * [pypy/rsdl](https://bitbucket.org/pypy/rsdl)
+    * Alternatively download RSDL package from [PYPI](https://pypi.python.org/pypi/rsdl)
+    * Then unpack and install it using ```python setup.py install``` or ```pypy setup.py install```
+
+### Required packages
+You need the following packages on your OS. Install with your favorite package manager:
+ * pypy
+    * For faster translation of the RSqueak VM. Alternatively use default Python.
+ * libsdl-dev
+ * libffi-dev
+
+### Adjusting the PYTHONPATH
+In order to allow the RPython toolchain to find the rsdl and pypy packages you have to add the two folders to the PYTHONPATH.
+
+If you are using the rsdl *repository*, you have to add the rsdl subfolder of the rsdl repository to the PYTHONPATH.
+
+```
+export PYTHONPATH=${PYTHONPATH}:.:[path/to/pypy]:[path/to/rsdl]
+```
+
+### Setting the SDL Driver
+For testing the basic functionality of the VM you might want to disable the UI. You can do so by setting the SDL_VIDEODRIVER environment variable to dummy.
+
+```
+export SDL_VIDEODRIVER=dummy
+```
+
+### Building & Tests
+Execute the following commands inside the main directory of this repository.
+
+To build the VM:
+
+```
+[path to pypy repository]/rpython/bin/rpython targetimageloadingsmalltalk.py
+```
+
+To build the VM with enabled just-in-time compiler:
+
+```
+[path to pypy repository]/rpython/bin/rpython -Ojit targetimageloadingsmalltalk.py
+```
+
+To run the tests (most do not require building):
+
+```
+[path to pypy repository]/pytest.py [--slow|--quick] spyvm/test
+```
+
+### Starting an image
+The build process will produce an executable called rsqueak.
+The ```image/``` directory contains two images you can open with the following.
+Use ```--help``` to see command line switches.
+
+```
+./rsqueak images/mini.image
+./rsqueak images/Squeak4.5-noBitBlt.image
+```
+
+
+
+
+STM-enabled Rsqueak
+===
+This is a branch of RSqueak which incorporates the RPython STM transformation. Most of the initial code base comes from the results of a project seminar (https://bitbucket.org/amintos/lang-smalltalk). The stmgc-c7 branch is based on this version and the 64bit branch.
+
+Setup for stm-enabled RSqueak
+---
+You can see the current state of the integration of the RPython STM in our stmgc-c7 branch.
+Beware that you can only build this branch if you have a 64-bit linux. To build this branch you have to setup several things:
+
+1. Change your local pypy repository to the stm-gc7 branch, commit dd3c06b
+2. Get a clang which has the patches from ([Clang patches](https://bitbucket.org/pypy/stmgc/src/d164a5bcad5e7615b4362b6a1a49d51e2e06de0c/c7/llvmfix/?at=default)). If you have a Debian-based OS you can use the following package: [llvm-pypystm](https://launchpad.net/~malte.swart/+archive/ubuntu/llvm-pypystm).
+
+To build, use the following command:
+```
+[path to pypy repository]/rpython/bin/rpython --gc=stmgc targetimageloadingsmalltalk.py
+```

From noreply at buildbot.pypy.org  Thu Nov 13 16:01:50 2014
From: noreply at buildbot.pypy.org (anton_gulenko)
Date: Thu, 13 Nov 2014 16:01:50 +0100 (CET)
Subject: [pypy-commit] lang-smalltalk default: Trying to fixed README lists.
Message-ID: <20141113150150.079DC1C3140@cobra.cs.uni-duesseldorf.de>

Author: Anton Gulenko 
Branch: 
Changeset: r1071:2a99a4827c84
Date: 2014-11-13 16:02 +0100
http://bitbucket.org/pypy/lang-smalltalk/changeset/2a99a4827c84/

Log:	Trying to fixed README lists.

diff --git a/README.md b/README.md
--- a/README.md
+++ b/README.md
@@ -7,19 +7,23 @@
 ----
 
 ### Required Projects
+
 You need three repositories: 
- * [This one](https://bitbucket.org/pypy/lang-smalltalk)
- * [pypy/pypy](https://bitbucket.org/pypy/pypy)
- * [pypy/rsdl](https://bitbucket.org/pypy/rsdl)
+
+* [This one](https://bitbucket.org/pypy/lang-smalltalk)
+* [pypy/pypy](https://bitbucket.org/pypy/pypy)
+* [pypy/rsdl](https://bitbucket.org/pypy/rsdl)
     * Alternatively download RSDL package from [PYPI](https://pypi.python.org/pypi/rsdl)
     * Then unpack and install it using ```python setup.py install``` or ```pypy setup.py install```
 
 ### Required packages
+
 You need the following packages on your OS. Install with your favorite package manager:
- * pypy
+
+* pypy
     * For faster translation of the RSqueak VM. Alternatively use default Python.
- * libsdl-dev
- * libffi-dev
+* libsdl-dev
+* libffi-dev
 
 ### Adjusting the PYTHONPATH
 In order to allow the RPython toolchain to find the rsdl and pypy packages you have to add the two folders to the PYTHONPATH.

From noreply at buildbot.pypy.org  Thu Nov 13 19:24:03 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 13 Nov 2014 19:24:03 +0100 (CET)
Subject: [pypy-commit] pypy stmgc-c7: (untested) rewrite this part of
 _write_barrier_fastpath to no
Message-ID: <20141113182403.D3E2B1D2859@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: stmgc-c7
Changeset: r74508:b39a540f8ed5
Date: 2014-11-13 19:23 +0100
http://bitbucket.org/pypy/pypy/changeset/b39a540f8ed5/

Log:	(untested) rewrite this part of _write_barrier_fastpath to no longer
	rely on &write_locks to be in the first 2GB, which is not the case
	on --shared.

diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py
--- a/rpython/jit/backend/x86/assembler.py
+++ b/rpython/jit/backend/x86/assembler.py
@@ -2310,19 +2310,32 @@
                 else:
                     raise AssertionError("CARD_SIZE should be 32/64/128")
                 #
-                # idea:  mov r11, loc_base    # the object
+                # idea:  mov r11, write_locks_base<<4
+                #        add r11, loc_base    # the object
                 #        and r11, ~15         # align
                 #        lea r11, [loc_index + r11<<(card_bits-4)]
                 #        shr r11, card_bits
-                #        mov [r11 + write_locks_base], card_marked
+                #        mov [r11], card_marked
+                #
+                # this assumes that the value computed up to the
+                # "shr r11, card_bits" instruction does not overflow
+                # the (64-card_bits) bound.  For now we silently assumes
+                # that it is the case.  As far as I can tell, this is
+                # clearly true on any reasonable setting (which assume
+                # that the non-kernel addresses are numbers between 0
+                # and 2**X, for X <= 56).
+                #
                 r11 = X86_64_SCRATCH_REG
+                initial_value = write_locks_base << 4
                 if isinstance(loc_index, RegLoc):
                     if isinstance(loc_base, RegLoc):
-                        mc.MOV_rr(r11.value, loc_base.value)
+                        mc.MOV_ri(r11, initial_value)
+                        mc.ADD_rr(r11.value, loc_base.value)
                         mc.AND_ri(r11.value, ~15)
                     else:
                         assert isinstance(loc_base, ImmedLoc)
-                        mc.MOV_ri(r11.value, loc_base.value & ~15)  # 32/64bit
+                        initial_value += loc_base.value & ~15
+                        mc.MOV_ri(r11.value, initial_value)
                     mc.LEA_ra(r11.value, (self.SEGMENT_NO,
                                           loc_index.value,
                                           r11.value,
@@ -2330,22 +2343,19 @@
                                           0))
                     mc.SHR_ri(r11.value, card_bits)
                 else:
-                    # XXX these cases could be slightly more optimized
                     assert isinstance(loc_index, ImmedLoc)
-                    cardindex = loc_index.value >> card_bits
+                    initial_value += (loc_index.value >> card_bits) << 4
                     if isinstance(loc_base, RegLoc):
-                        if rx86.fits_in_32bits(write_locks_base + cardindex):
-                            write_locks_base += cardindex
-                            mc.MOV_rr(r11.value, loc_base.value)
-                        else:
-                            mc.MOV_ri(r11.value, cardindex << 4)    # 32/64bit
-                            mc.ADD_rr(r11.value, loc_base.value)
+                        mc.MOV_ri(r11.value, initial_value)
+                        mc.ADD_rr(r11.value, loc_base.value)
                         mc.SHR_ri(r11.value, 4)
                     else:
-                        mc.MOV_ri(r11.value, cardindex + (loc_base.value >> 4))
+                        assert isinstance(loc_base, ImmedLoc)
+                        initial_value += loc_base.value
+                        initial_value >>= 4
+                        mc.MOV_ri(r11.value, initial_value)
                 #
-                assert rx86.fits_in_32bits(write_locks_base), "XXX"
-                mc.MOV8_mi((self.SEGMENT_NO, r11.value, write_locks_base),
+                mc.MOV8_mi((self.SEGMENT_NO, r11.value, 0),
                            rstm.CARD_MARKED)
 
             elif isinstance(loc_index, RegLoc):

From noreply at buildbot.pypy.org  Thu Nov 13 22:37:05 2014
From: noreply at buildbot.pypy.org (amauryfa)
Date: Thu, 13 Nov 2014 22:37:05 +0100 (CET)
Subject: [pypy-commit] pypy kill-rctime: Close branch about to be closed
Message-ID: <20141113213705.909421C0548@cobra.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc 
Branch: kill-rctime
Changeset: r74509:3c0cf064f9c4
Date: 2014-11-13 22:28 +0100
http://bitbucket.org/pypy/pypy/changeset/3c0cf064f9c4/

Log:	Close branch about to be closed

diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -43,3 +43,7 @@
 .. branch nditer-external_loop
 
 Implement `external_loop` arguement to numpy's nditer
+
+.. branch kill-rctime
+
+Rename pypy/module/rctime to pypy/module/time, since it contains the implementation of the 'time' module.

From noreply at buildbot.pypy.org  Thu Nov 13 22:37:08 2014
From: noreply at buildbot.pypy.org (amauryfa)
Date: Thu, 13 Nov 2014 22:37:08 +0100 (CET)
Subject: [pypy-commit] pypy default: Merge branch kill-rctime: finally
 rename pypy/module/rctime to pypy/module/time.
Message-ID: <20141113213708.052821C0548@cobra.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc 
Branch: 
Changeset: r74510:e9494abc8143
Date: 2014-11-13 22:34 +0100
http://bitbucket.org/pypy/pypy/changeset/e9494abc8143/

Log:	Merge branch kill-rctime: finally rename pypy/module/rctime to
	pypy/module/time.

	The 'rc' prefix comes from the early days where (a subset of) ctypes
	was the blessed way to write RPython extension modules...

diff too long, truncating to 2000 out of 2648 lines

diff --git a/lib-python/conftest.py b/lib-python/conftest.py
--- a/lib-python/conftest.py
+++ b/lib-python/conftest.py
@@ -59,7 +59,7 @@
     def __init__(self, basename, core=False, compiler=None, usemodules='',
                  skip=None):
         self.basename = basename
-        self._usemodules = usemodules.split() + ['signal', 'rctime', 'itertools', '_socket']
+        self._usemodules = usemodules.split() + ['signal', 'time', 'itertools', '_socket']
         self._compiler = compiler
         self.core = core
         self.skip = skip
diff --git a/pypy/config/pypyoption.py b/pypy/config/pypyoption.py
--- a/pypy/config/pypyoption.py
+++ b/pypy/config/pypyoption.py
@@ -29,7 +29,7 @@
 # --allworkingmodules
 working_modules = default_modules.copy()
 working_modules.update([
-    "_socket", "unicodedata", "mmap", "fcntl", "_locale", "pwd", "rctime" ,
+    "_socket", "unicodedata", "mmap", "fcntl", "_locale", "pwd", "time" ,
     "select", "zipimport", "_lsprof", "crypt", "signal", "_rawffi", "termios",
     "zlib", "bz2", "struct", "_hashlib", "_md5", "_sha", "_minimal_curses",
     "cStringIO", "thread", "itertools", "pyexpat", "_ssl", "cpyext", "array",
@@ -40,7 +40,7 @@
 
 translation_modules = default_modules.copy()
 translation_modules.update([
-    "fcntl", "rctime", "select", "signal", "_rawffi", "zlib", "struct", "_md5",
+    "fcntl", "time", "select", "signal", "_rawffi", "zlib", "struct", "_md5",
     "cStringIO", "array", "binascii",
     # the following are needed for pyrepl (and hence for the
     # interactive prompt/pdb)
@@ -65,18 +65,18 @@
 
 if sys.platform == "sunos5":
     working_modules.remove('mmap')   # depend on ctypes, can't get at c-level 'errono'
-    working_modules.remove('rctime') # depend on ctypes, missing tm_zone/tm_gmtoff
+    working_modules.remove('time')   # depend on ctypes, missing tm_zone/tm_gmtoff
     working_modules.remove('signal') # depend on ctypes, can't get at c-level 'errono'
     working_modules.remove('fcntl')  # LOCK_NB not defined
     working_modules.remove("_minimal_curses")
     working_modules.remove("termios")
-    working_modules.remove("_multiprocessing")   # depends on rctime
+    working_modules.remove("_multiprocessing")   # depends on time
     if "cppyy" in working_modules:
         working_modules.remove("cppyy")  # depends on ctypes
 
 
 module_dependencies = {
-    '_multiprocessing': [('objspace.usemodules.rctime', True),
+    '_multiprocessing': [('objspace.usemodules.time', True),
                          ('objspace.usemodules.thread', True)],
     'cpyext': [('objspace.usemodules.array', True)],
     'cppyy': [('objspace.usemodules.cpyext', True)],
diff --git a/pypy/doc/config/objspace.usemodules.rctime.txt b/pypy/doc/config/objspace.usemodules.rctime.txt
deleted file mode 100644
--- a/pypy/doc/config/objspace.usemodules.rctime.txt
+++ /dev/null
@@ -1,7 +0,0 @@
-Use the 'rctime' module. 
-
-'rctime' is our `rffi`_ based implementation of the builtin 'time' module.
-It supersedes the less complete :config:`objspace.usemodules.time`,
-at least for C-like targets (the C and LLVM backends).
-
-.. _`rffi`: ../rffi.html
diff --git a/pypy/doc/config/objspace.usemodules.time.txt b/pypy/doc/config/objspace.usemodules.time.txt
--- a/pypy/doc/config/objspace.usemodules.time.txt
+++ b/pypy/doc/config/objspace.usemodules.time.txt
@@ -1,5 +1,1 @@
 Use the 'time' module. 
-
-Obsolete; use :config:`objspace.usemodules.rctime` for our up-to-date version
-of the application-level 'time' module, at least for C-like targets (the C
-and LLVM backends).
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -43,3 +43,7 @@
 .. branch nditer-external_loop
 
 Implement `external_loop` arguement to numpy's nditer
+
+.. branch kill-rctime
+
+Rename pypy/module/rctime to pypy/module/time, since it contains the implementation of the 'time' module.
diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -522,11 +522,6 @@
                 if name not in modules:
                     modules.append(name)
 
-        # a bit of custom logic: rctime take precedence over time
-        # XXX this could probably be done as a "requires" in the config
-        if 'rctime' in modules and 'time' in modules:
-            modules.remove('time')
-
         self._builtinmodule_list = modules
         return self._builtinmodule_list
 
diff --git a/pypy/module/_file/test/test_file_extra.py b/pypy/module/_file/test/test_file_extra.py
--- a/pypy/module/_file/test/test_file_extra.py
+++ b/pypy/module/_file/test/test_file_extra.py
@@ -221,7 +221,7 @@
     expected_filename = str(udir.join('sample'))
     expected_mode = 'rb'
     extra_args = ()
-    spaceconfig = {'usemodules': ['binascii', 'rctime', 'struct']}
+    spaceconfig = {'usemodules': ['binascii', 'time', 'struct']}
 
     def setup_method(self, method):
         space = self.space
@@ -281,7 +281,7 @@
     expected_filename = ''
     expected_mode = 'rb'
     extra_args = ()
-    spaceconfig = {'usemodules': ['binascii', 'rctime', 'struct']}
+    spaceconfig = {'usemodules': ['binascii', 'time', 'struct']}
 
     def setup_method(self, method):
         space = self.space
@@ -359,7 +359,7 @@
 #  A few extra tests
 
 class AppTestAFewExtra:
-    spaceconfig = {'usemodules': ['_socket', 'array', 'binascii', 'rctime',
+    spaceconfig = {'usemodules': ['_socket', 'array', 'binascii', 'time',
                                   'struct']}
 
     def setup_method(self, method):
diff --git a/pypy/module/_lsprof/test/test_cprofile.py b/pypy/module/_lsprof/test/test_cprofile.py
--- a/pypy/module/_lsprof/test/test_cprofile.py
+++ b/pypy/module/_lsprof/test/test_cprofile.py
@@ -1,6 +1,6 @@
 class AppTestCProfile(object):
     spaceconfig = {
-        "usemodules": ['_lsprof', 'rctime'],
+        "usemodules": ['_lsprof', 'time'],
     }
 
     def setup_class(cls):
diff --git a/pypy/module/_md5/test/test_md5.py b/pypy/module/_md5/test/test_md5.py
--- a/pypy/module/_md5/test/test_md5.py
+++ b/pypy/module/_md5/test/test_md5.py
@@ -5,7 +5,7 @@
 
 class AppTestMD5(object):
     spaceconfig = {
-        'usemodules': ['_md5', 'binascii', 'rctime', 'struct'],
+        'usemodules': ['_md5', 'binascii', 'time', 'struct'],
     }
 
     def setup_class(cls):
diff --git a/pypy/module/_multiprocessing/interp_semaphore.py b/pypy/module/_multiprocessing/interp_semaphore.py
--- a/pypy/module/_multiprocessing/interp_semaphore.py
+++ b/pypy/module/_multiprocessing/interp_semaphore.py
@@ -254,7 +254,7 @@
         start = _GetTickCount()
 
         while True:
-            from pypy.module.rctime.interp_time import State
+            from pypy.module.time.interp_time import State
             interrupt_event = space.fromcache(State).get_interrupt_event()
             handles = [self.handle, interrupt_event]
 
diff --git a/pypy/module/_random/test/test_random.py b/pypy/module/_random/test/test_random.py
--- a/pypy/module/_random/test/test_random.py
+++ b/pypy/module/_random/test/test_random.py
@@ -1,6 +1,6 @@
 class AppTestRandom:
     spaceconfig = {
-        "usemodules": ['_random', 'rctime'],
+        "usemodules": ['_random', 'time'],
     }
 
     def test_dict(self):
diff --git a/pypy/module/_sha/test/test_sha.py b/pypy/module/_sha/test/test_sha.py
--- a/pypy/module/_sha/test/test_sha.py
+++ b/pypy/module/_sha/test/test_sha.py
@@ -5,7 +5,7 @@
 
 class AppTestSHA(object):
     spaceconfig = {
-        'usemodules': ['_sha', 'binascii', 'rctime', 'struct'],
+        'usemodules': ['_sha', 'binascii', 'time', 'struct'],
     }
 
     def setup_class(cls):
diff --git a/pypy/module/bz2/test/test_bz2_file.py b/pypy/module/bz2/test/test_bz2_file.py
--- a/pypy/module/bz2/test/test_bz2_file.py
+++ b/pypy/module/bz2/test/test_bz2_file.py
@@ -53,7 +53,7 @@
 
 class AppTestBZ2File(CheckAllocation):
     spaceconfig = {
-        'usemodules': ['bz2', 'binascii', 'rctime', 'struct']
+        'usemodules': ['bz2', 'binascii', 'time', 'struct']
     }
 
     def setup_class(cls):
diff --git a/pypy/module/cpyext/test/conftest.py b/pypy/module/cpyext/test/conftest.py
--- a/pypy/module/cpyext/test/conftest.py
+++ b/pypy/module/cpyext/test/conftest.py
@@ -7,7 +7,7 @@
     # it's necessary to run "import time" at least once before any
     # other cpyext test, otherwise the same statement will fail in
     # test_datetime.py.
-    space = gettestobjspace(usemodules=['rctime'])
+    space = gettestobjspace(usemodules=['time'])
     space.getbuiltinmodule("time")
 
 def pytest_ignore_collect(path, config):
diff --git a/pypy/module/cpyext/test/test_cpyext.py b/pypy/module/cpyext/test/test_cpyext.py
--- a/pypy/module/cpyext/test/test_cpyext.py
+++ b/pypy/module/cpyext/test/test_cpyext.py
@@ -102,7 +102,7 @@
 class LeakCheckingTest(object):
     """Base class for all cpyext tests."""
     spaceconfig = dict(usemodules=['cpyext', 'thread', '_rawffi', 'array',
-                                   'itertools', 'rctime', 'binascii', 'micronumpy'])
+                                   'itertools', 'time', 'binascii', 'micronumpy'])
     spaceconfig['std.withmethodcache'] = True
 
     enable_leak_checking = True
diff --git a/pypy/module/fcntl/test/test_fcntl.py b/pypy/module/fcntl/test/test_fcntl.py
--- a/pypy/module/fcntl/test/test_fcntl.py
+++ b/pypy/module/fcntl/test/test_fcntl.py
@@ -12,7 +12,7 @@
 
 class AppTestFcntl:
     spaceconfig = dict(usemodules=('fcntl', 'array', 'struct', 'termios',
-                                   'select', 'rctime'))
+                                   'select', 'time'))
 
     def setup_class(cls):
         tmpprefix = str(udir.ensure('test_fcntl', dir=1).join('tmp_'))
diff --git a/pypy/module/imp/test/test_app.py b/pypy/module/imp/test/test_app.py
--- a/pypy/module/imp/test/test_app.py
+++ b/pypy/module/imp/test/test_app.py
@@ -4,7 +4,7 @@
 
 class AppTestImpModule:
     spaceconfig = {
-        'usemodules': ['binascii', 'imp', 'itertools', 'rctime', 'struct'],
+        'usemodules': ['binascii', 'imp', 'itertools', 'time', 'struct'],
     }
 
     def setup_class(cls):
diff --git a/pypy/module/imp/test/test_import.py b/pypy/module/imp/test/test_import.py
--- a/pypy/module/imp/test/test_import.py
+++ b/pypy/module/imp/test/test_import.py
@@ -149,7 +149,7 @@
 
 class AppTestImport:
     spaceconfig = {
-        "usemodules": ['_md5', 'rctime'],
+        "usemodules": ['_md5', 'time'],
     }
 
     def setup_class(cls):
@@ -1044,7 +1044,7 @@
 
 class AppTestImportHooks(object):
     spaceconfig = {
-        "usemodules": ['struct', 'itertools', 'rctime'],
+        "usemodules": ['struct', 'itertools', 'time'],
     }
 
     def setup_class(cls):
@@ -1304,7 +1304,7 @@
 
 
 class AppTestMultithreadedImp(object):
-    spaceconfig = dict(usemodules=['thread', 'rctime'])
+    spaceconfig = dict(usemodules=['thread', 'time'])
 
     def setup_class(cls):
         #if not conftest.option.runappdirect:
diff --git a/pypy/module/math/test/test_math.py b/pypy/module/math/test/test_math.py
--- a/pypy/module/math/test/test_math.py
+++ b/pypy/module/math/test/test_math.py
@@ -7,7 +7,7 @@
 
 class AppTestMath:
     spaceconfig = {
-        "usemodules": ['math', 'struct', 'itertools', 'rctime', 'binascii'],
+        "usemodules": ['math', 'struct', 'itertools', 'time', 'binascii'],
     }
 
     def setup_class(cls):
diff --git a/pypy/module/posix/test/test_posix2.py b/pypy/module/posix/test/test_posix2.py
--- a/pypy/module/posix/test/test_posix2.py
+++ b/pypy/module/posix/test/test_posix2.py
@@ -14,7 +14,7 @@
 import signal
 
 def setup_module(mod):
-    usemodules = ['binascii', 'posix', 'struct', 'rctime']
+    usemodules = ['binascii', 'posix', 'struct', 'time']
     if os.name != 'nt':
         usemodules += ['fcntl']
     else:
diff --git a/pypy/module/posix/test/test_posix_libfile.py b/pypy/module/posix/test/test_posix_libfile.py
--- a/pypy/module/posix/test/test_posix_libfile.py
+++ b/pypy/module/posix/test/test_posix_libfile.py
@@ -10,7 +10,7 @@
 
 class AppTestPosix:
     spaceconfig = {
-        "usemodules": ['posix', 'rctime'],
+        "usemodules": ['posix', 'time'],
     }
 
     def setup_class(cls):
diff --git a/pypy/module/pyexpat/test/test_parser.py b/pypy/module/pyexpat/test/test_parser.py
--- a/pypy/module/pyexpat/test/test_parser.py
+++ b/pypy/module/pyexpat/test/test_parser.py
@@ -177,7 +177,7 @@
 
 class AppTestPyexpat2:
     spaceconfig = dict(usemodules=['pyexpat', 'itertools', '_socket',
-                                   'rctime', 'struct', 'binascii'])
+                                   'time', 'struct', 'binascii'])
 
     def test_django_bug(self):
         xml_str = ''
diff --git a/pypy/module/pypyjit/test/test_policy.py b/pypy/module/pypyjit/test/test_policy.py
--- a/pypy/module/pypyjit/test/test_policy.py
+++ b/pypy/module/pypyjit/test/test_policy.py
@@ -29,7 +29,7 @@
     assert pypypolicy.look_inside_function(get_ident)
 
 def test_time():
-    from pypy.module.rctime.interp_time import time
+    from pypy.module.time.interp_time import time
     assert pypypolicy.look_inside_function(time)
 
 def test_io():
diff --git a/pypy/module/rctime/__init__.py b/pypy/module/rctime/__init__.py
deleted file mode 100644
--- a/pypy/module/rctime/__init__.py
+++ /dev/null
@@ -1,42 +0,0 @@
-
-from pypy.interpreter.mixedmodule import MixedModule
-import os
-
-_WIN = os.name == "nt"
-
-class Module(MixedModule):
-    applevel_name = 'time'
-
-    interpleveldefs = {
-        'time': 'interp_time.time',
-        'clock': 'interp_time.clock',
-        'ctime': 'interp_time.ctime',
-        'asctime': 'interp_time.asctime',
-        'gmtime': 'interp_time.gmtime',
-        'localtime': 'interp_time.localtime',
-        'mktime': 'interp_time.mktime',
-        'strftime': 'interp_time.strftime',
-        'sleep' : 'interp_time.sleep',
-    }
-
-    if os.name == "posix":
-        interpleveldefs['tzset'] = 'interp_time.tzset'
-
-    appleveldefs = {
-        'struct_time': 'app_time.struct_time',
-        '__doc__': 'app_time.__doc__',
-        'strptime': 'app_time.strptime',
-    }
-
-    def startup(self, space):
-        if _WIN:
-            from pypy.module.rctime.interp_time import State
-            space.fromcache(State).startup(space)
-
-        # this machinery is needed to expose constants
-        # that have to be initialized one time only
-        from pypy.module.rctime import interp_time
-
-        interp_time._init_timezone(space)
-        interp_time._init_accept2dyear(space)
-
diff --git a/pypy/module/rctime/interp_time.py b/pypy/module/rctime/interp_time.py
deleted file mode 100644
--- a/pypy/module/rctime/interp_time.py
+++ /dev/null
@@ -1,670 +0,0 @@
-from rpython.rtyper.tool import rffi_platform as platform
-from rpython.rtyper.lltypesystem import rffi
-from pypy.interpreter.error import OperationError, oefmt
-from pypy.interpreter.gateway import unwrap_spec
-from rpython.rtyper.lltypesystem import lltype
-from rpython.rlib.rarithmetic import intmask
-from rpython.rlib import rposix
-from rpython.translator.tool.cbuild import ExternalCompilationInfo
-import os
-import sys
-import time as pytime
-
-_POSIX = os.name == "posix"
-_WIN = os.name == "nt"
-_CYGWIN = sys.platform == "cygwin"
-
-_time_zones = []
-if _CYGWIN:
-    _time_zones = ["GMT-12", "GMT-11", "GMT-10", "GMT-9", "GMT-8", "GMT-7",
-                   "GMT-6", "GMT-5", "GMT-4", "GMT-3", "GMT-2", "GMT-1",
-                   "GMT",  "GMT+1", "GMT+2", "GMT+3", "GMT+4", "GMT+5",
-                   "GMT+6",  "GMT+7", "GMT+8", "GMT+9", "GMT+10", "GMT+11",
-                   "GMT+12",  "GMT+13", "GMT+14"]
-
-if _WIN:
-    # Interruptible sleeps on Windows:
-    # We install a specific Console Ctrl Handler which sets an 'event'.
-    # time.sleep() will actually call WaitForSingleObject with the desired
-    # timeout.  On Ctrl-C, the signal handler is called, the event is set,
-    # and the wait function exits.
-    from rpython.rlib import rwin32
-    from pypy.interpreter.error import wrap_windowserror, wrap_oserror
-    from rpython.rlib import rthread as thread
-
-    eci = ExternalCompilationInfo(
-        includes = ['windows.h'],
-        post_include_bits = [
-            "RPY_EXTERN\n"
-            "BOOL pypy_timemodule_setCtrlHandler(HANDLE event);"],
-        separate_module_sources=['''
-            static HANDLE interrupt_event;
-
-            static BOOL WINAPI CtrlHandlerRoutine(
-              DWORD dwCtrlType)
-            {
-                SetEvent(interrupt_event);
-                /* allow other default handlers to be called.
-                 * Default Python handler will setup the
-                 * KeyboardInterrupt exception.
-                 */
-                return 0;
-            }
-
-            BOOL pypy_timemodule_setCtrlHandler(HANDLE event)
-            {
-                interrupt_event = event;
-                return SetConsoleCtrlHandler(CtrlHandlerRoutine, TRUE);
-            }
-
-        '''],
-        )
-    _setCtrlHandlerRoutine = rffi.llexternal(
-        'pypy_timemodule_setCtrlHandler',
-        [rwin32.HANDLE], rwin32.BOOL,
-        compilation_info=eci)
-
-    class GlobalState:
-        def __init__(self):
-            self.init()
-
-        def init(self):
-            self.interrupt_event = rwin32.NULL_HANDLE
-
-        def startup(self, space):
-            # Initialize the event handle used to signal Ctrl-C
-            try:
-                globalState.interrupt_event = rwin32.CreateEvent(
-                    rffi.NULL, True, False, rffi.NULL)
-            except WindowsError, e:
-                raise wrap_windowserror(space, e)
-            if not _setCtrlHandlerRoutine(globalState.interrupt_event):
-                raise wrap_windowserror(
-                    space, rwin32.lastWindowsError("SetConsoleCtrlHandler"))
-
-    globalState = GlobalState()
-
-    class State:
-        def __init__(self, space):
-            self.main_thread = 0
-
-        def _cleanup_(self):
-            self.main_thread = 0
-            globalState.init()
-
-        def startup(self, space):
-            self.main_thread = thread.get_ident()
-            globalState.startup(space)
-
-        def get_interrupt_event(self):
-            return globalState.interrupt_event
-
-
-_includes = ["time.h"]
-if _POSIX:
-    _includes.append('sys/time.h')
-
-class CConfig:
-    _compilation_info_ = ExternalCompilationInfo(
-        includes = _includes
-    )
-    CLOCKS_PER_SEC = platform.ConstantInteger("CLOCKS_PER_SEC")
-    clock_t = platform.SimpleType("clock_t", rffi.ULONG)
-    has_gettimeofday = platform.Has('gettimeofday')
-
-if _POSIX:
-    calling_conv = 'c'
-    CConfig.timeval = platform.Struct("struct timeval",
-                                      [("tv_sec", rffi.INT),
-                                       ("tv_usec", rffi.INT)])
-    if _CYGWIN:
-        CConfig.tm = platform.Struct("struct tm", [("tm_sec", rffi.INT),
-            ("tm_min", rffi.INT), ("tm_hour", rffi.INT), ("tm_mday", rffi.INT),
-            ("tm_mon", rffi.INT), ("tm_year", rffi.INT), ("tm_wday", rffi.INT),
-            ("tm_yday", rffi.INT), ("tm_isdst", rffi.INT)])
-    else:
-        CConfig.tm = platform.Struct("struct tm", [("tm_sec", rffi.INT),
-            ("tm_min", rffi.INT), ("tm_hour", rffi.INT), ("tm_mday", rffi.INT),
-            ("tm_mon", rffi.INT), ("tm_year", rffi.INT), ("tm_wday", rffi.INT),
-            ("tm_yday", rffi.INT), ("tm_isdst", rffi.INT), ("tm_gmtoff", rffi.LONG),
-            ("tm_zone", rffi.CCHARP)])
-elif _WIN:
-    calling_conv = 'win'
-    CConfig.tm = platform.Struct("struct tm", [("tm_sec", rffi.INT),
-        ("tm_min", rffi.INT), ("tm_hour", rffi.INT), ("tm_mday", rffi.INT),
-        ("tm_mon", rffi.INT), ("tm_year", rffi.INT), ("tm_wday", rffi.INT),
-        ("tm_yday", rffi.INT), ("tm_isdst", rffi.INT)])
-
-class cConfig:
-    pass
-
-for k, v in platform.configure(CConfig).items():
-    setattr(cConfig, k, v)
-cConfig.tm.__name__ = "_tm"
-
-def external(name, args, result, eci=CConfig._compilation_info_):
-    if _WIN and rffi.sizeof(rffi.TIME_T) == 8:
-        # Recent Microsoft compilers use 64bit time_t and
-        # the corresponding functions are named differently
-        if (rffi.TIME_T in args or rffi.TIME_TP in args
-            or result in (rffi.TIME_T, rffi.TIME_TP)):
-            name = '_' + name + '64'
-    return rffi.llexternal(name, args, result,
-                           compilation_info=eci,
-                           calling_conv=calling_conv,
-                           releasegil=False)
-
-if _POSIX:
-    cConfig.timeval.__name__ = "_timeval"
-    timeval = cConfig.timeval
-
-CLOCKS_PER_SEC = cConfig.CLOCKS_PER_SEC
-clock_t = cConfig.clock_t
-tm = cConfig.tm
-glob_buf = lltype.malloc(tm, flavor='raw', zero=True, immortal=True)
-
-if cConfig.has_gettimeofday:
-    c_gettimeofday = external('gettimeofday', [rffi.VOIDP, rffi.VOIDP], rffi.INT)
-TM_P = lltype.Ptr(tm)
-c_clock = external('clock', [rffi.TIME_TP], clock_t)
-c_time = external('time', [rffi.TIME_TP], rffi.TIME_T)
-c_ctime = external('ctime', [rffi.TIME_TP], rffi.CCHARP)
-c_gmtime = external('gmtime', [rffi.TIME_TP], TM_P)
-c_mktime = external('mktime', [TM_P], rffi.TIME_T)
-c_asctime = external('asctime', [TM_P], rffi.CCHARP)
-c_localtime = external('localtime', [rffi.TIME_TP], TM_P)
-if _POSIX:
-    c_tzset = external('tzset', [], lltype.Void)
-if _WIN:
-    win_eci = ExternalCompilationInfo(
-        includes = ["time.h"],
-        post_include_bits = ["RPY_EXTERN "
-                             "long pypy_get_timezone();\n"
-                             "RPY_EXTERN "
-                             "int pypy_get_daylight();\n"
-                             "RPY_EXTERN "
-                             "char** pypy_get_tzname();\n"
-                             "RPY_EXTERN "
-                             "void pypy__tzset();"],
-        separate_module_sources = ["""
-        long pypy_get_timezone() { return timezone; }
-        int pypy_get_daylight() { return daylight; }
-        char** pypy_get_tzname() { return tzname; }
-        void pypy__tzset() { _tzset(); }
-        """])
-    # Ensure sure that we use _tzset() and timezone from the same C Runtime.
-    c_tzset = external('pypy__tzset', [], lltype.Void, win_eci)
-    c_get_timezone = external('pypy_get_timezone', [], rffi.LONG, win_eci)
-    c_get_daylight = external('pypy_get_daylight', [], rffi.INT, win_eci)
-    c_get_tzname = external('pypy_get_tzname', [], rffi.CCHARPP, win_eci)
-
-c_strftime = external('strftime', [rffi.CCHARP, rffi.SIZE_T, rffi.CCHARP, TM_P],
-                      rffi.SIZE_T)
-
-def _init_accept2dyear(space):
-    if os.environ.get("PYTHONY2K"):
-        accept2dyear = 0
-    else:
-        accept2dyear = 1
-    _set_module_object(space, "accept2dyear", space.wrap(accept2dyear))
-
-def _init_timezone(space):
-    timezone = daylight = altzone = 0
-    tzname = ["", ""]
-
-    if _WIN:
-        c_tzset()
-        timezone = c_get_timezone()
-        altzone = timezone - 3600
-        daylight = c_get_daylight()
-        tzname_ptr = c_get_tzname()
-        tzname = rffi.charp2str(tzname_ptr[0]), rffi.charp2str(tzname_ptr[1])
-
-    if _POSIX:
-        if _CYGWIN:
-            YEAR = (365 * 24 + 6) * 3600
-
-            # about January 11th
-            t = (((c_time(lltype.nullptr(rffi.TIME_TP.TO))) / YEAR) * YEAR + 10 * 24 * 3600)
-            # we cannot have reference to stack variable, put it on the heap
-            t_ref = lltype.malloc(rffi.TIME_TP.TO, 1, flavor='raw')
-            t_ref[0] = rffi.cast(rffi.TIME_T, t)
-            p = c_localtime(t_ref)
-            q = c_gmtime(t_ref)
-            janzone = (p.c_tm_hour + 24 * p.c_tm_mday) - (q.c_tm_hour + 24 * q.c_tm_mday)
-            if janzone < -12:
-                janname = "   "
-            elif janzone > 14:
-                janname = "   "
-            else:
-                janname = _time_zones[janzone - 12]
-            janzone = janzone * 3600
-            # about July 11th
-            tt = t + YEAR / 2
-            t_ref[0] = rffi.cast(rffi.TIME_T, tt)
-            p = c_localtime(t_ref)
-            q = c_gmtime(t_ref)
-            julyzone = (p.c_tm_hour + 24 * p.c_tm_mday) - (q.c_tm_hour + 24 * q.c_tm_mday)
-            if julyzone < -12:
-                julyname = "   "
-            elif julyzone > 14:
-                julyname = "   "
-            else:
-                julyname = _time_zones[julyzone - 12]
-            julyzone = julyzone * 3600
-            lltype.free(t_ref, flavor='raw')
-
-            if janzone < julyzone:
-                # DST is reversed in the southern hemisphere
-                timezone = julyzone
-                altzone = janzone
-                daylight = int(janzone != julyzone)
-                tzname = [julyname, janname]
-            else:
-                timezone = janzone
-                altzone = julyzone
-                daylight = int(janzone != julyzone)
-                tzname = [janname, julyname]
-
-        else:
-            YEAR = (365 * 24 + 6) * 3600
-
-            t = (((c_time(lltype.nullptr(rffi.TIME_TP.TO))) / YEAR) * YEAR)
-            # we cannot have reference to stack variable, put it on the heap
-            t_ref = lltype.malloc(rffi.TIME_TP.TO, 1, flavor='raw')
-            t_ref[0] = rffi.cast(rffi.TIME_T, t)
-            p = c_localtime(t_ref)
-            janzone = -p.c_tm_gmtoff
-            tm_zone = rffi.charp2str(p.c_tm_zone)
-            janname = ["   ", tm_zone][bool(tm_zone)]
-            tt = t + YEAR / 2
-            t_ref[0] = rffi.cast(rffi.TIME_T, tt)
-            p = c_localtime(t_ref)
-            lltype.free(t_ref, flavor='raw')
-            tm_zone = rffi.charp2str(p.c_tm_zone)
-            julyzone = -p.c_tm_gmtoff
-            julyname = ["   ", tm_zone][bool(tm_zone)]
-
-            if janzone < julyzone:
-                # DST is reversed in the southern hemisphere
-                timezone = julyzone
-                altzone = janzone
-                daylight = int(janzone != julyzone)
-                tzname = [julyname, janname]
-            else:
-                timezone = janzone
-                altzone = julyzone
-                daylight = int(janzone != julyzone)
-                tzname = [janname, julyname]
-
-    _set_module_object(space, "timezone", space.wrap(timezone))
-    _set_module_object(space, 'daylight', space.wrap(daylight))
-    tzname_w = [space.wrap(tzname[0]), space.wrap(tzname[1])]
-    _set_module_object(space, 'tzname', space.newtuple(tzname_w))
-    _set_module_object(space, 'altzone', space.wrap(altzone))
-
-def _get_error_msg():
-    errno = rposix.get_errno()
-    return os.strerror(errno)
-
-if sys.platform != 'win32':
-    @unwrap_spec(secs=float)
-    def sleep(space, secs):
-        if secs < 0:
-            raise OperationError(space.w_IOError,
-                                 space.wrap("Invalid argument: negative time in sleep"))
-        pytime.sleep(secs)
-else:
-    from rpython.rlib import rwin32
-    from errno import EINTR
-    def _simple_sleep(space, secs, interruptible):
-        if secs == 0.0 or not interruptible:
-            pytime.sleep(secs)
-        else:
-            millisecs = int(secs * 1000)
-            interrupt_event = space.fromcache(State).get_interrupt_event()
-            rwin32.ResetEvent(interrupt_event)
-            rc = rwin32.WaitForSingleObject(interrupt_event, millisecs)
-            if rc == rwin32.WAIT_OBJECT_0:
-                # Yield to make sure real Python signal handler
-                # called.
-                pytime.sleep(0.001)
-                raise wrap_oserror(space,
-                                   OSError(EINTR, "sleep() interrupted"))
-    @unwrap_spec(secs=float)
-    def sleep(space, secs):
-        if secs < 0:
-            raise OperationError(space.w_IOError,
-                                 space.wrap("Invalid argument: negative time in sleep"))
-        # as decreed by Guido, only the main thread can be
-        # interrupted.
-        main_thread = space.fromcache(State).main_thread
-        interruptible = (main_thread == thread.get_ident())
-        MAX = sys.maxint / 1000.0 # > 24 days
-        while secs > MAX:
-            _simple_sleep(space, MAX, interruptible)
-            secs -= MAX
-        _simple_sleep(space, secs, interruptible)
-
-def _get_module_object(space, obj_name):
-    w_module = space.getbuiltinmodule('time')
-    w_obj = space.getattr(w_module, space.wrap(obj_name))
-    return w_obj
-
-def _set_module_object(space, obj_name, w_obj_value):
-    w_module = space.getbuiltinmodule('time')
-    space.setattr(w_module, space.wrap(obj_name), w_obj_value)
-
-def _get_inttime(space, w_seconds):
-    # w_seconds can be a wrapped None (it will be automatically wrapped
-    # in the callers, so we never get a real None here).
-    if space.is_none(w_seconds):
-        seconds = pytime.time()
-    else:
-        seconds = space.float_w(w_seconds)
-    #
-    t = rffi.cast(rffi.TIME_T, seconds)
-    #
-    # Logic from CPython: How much info did we lose?  We assume that
-    # time_t is an integral type.  If we lost a second or more, the
-    # input doesn't fit in a time_t; call it an error.
-    diff = seconds - rffi.cast(lltype.Float, t)
-    if diff <= -1.0 or diff >= 1.0:
-        raise OperationError(space.w_ValueError,
-                      space.wrap("timestamp out of range for platform time_t"))
-    return t
-
-def _tm_to_tuple(space, t):
-    time_tuple = [
-        space.wrap(rffi.getintfield(t, 'c_tm_year') + 1900),
-        space.wrap(rffi.getintfield(t, 'c_tm_mon') + 1), # want january == 1
-        space.wrap(rffi.getintfield(t, 'c_tm_mday')),
-        space.wrap(rffi.getintfield(t, 'c_tm_hour')),
-        space.wrap(rffi.getintfield(t, 'c_tm_min')),
-        space.wrap(rffi.getintfield(t, 'c_tm_sec')),
-        space.wrap((rffi.getintfield(t, 'c_tm_wday') + 6) % 7), # want monday == 0
-        space.wrap(rffi.getintfield(t, 'c_tm_yday') + 1), # want january, 1 == 1
-        space.wrap(rffi.getintfield(t, 'c_tm_isdst'))]
-
-    w_struct_time = _get_module_object(space, 'struct_time')
-    w_time_tuple = space.newtuple(time_tuple)
-    return space.call_function(w_struct_time, w_time_tuple)
-
-def _gettmarg(space, w_tup, allowNone=True):
-    if space.is_none(w_tup):
-        if not allowNone:
-            raise OperationError(space.w_TypeError,
-                                 space.wrap("tuple expected"))
-        # default to the current local time
-        tt = rffi.r_time_t(int(pytime.time()))
-        t_ref = lltype.malloc(rffi.TIME_TP.TO, 1, flavor='raw')
-        t_ref[0] = tt
-        pbuf = c_localtime(t_ref)
-        lltype.free(t_ref, flavor='raw')
-        if not pbuf:
-            raise OperationError(space.w_ValueError,
-                space.wrap(_get_error_msg()))
-        return pbuf
-
-    tup_w = space.fixedview(w_tup)
-    if len(tup_w) != 9:
-        raise oefmt(space.w_TypeError,
-                    "argument must be sequence of length 9, not %d",
-                    len(tup_w))
-
-    y = space.int_w(tup_w[0])
-    tm_mon = space.int_w(tup_w[1])
-    if tm_mon == 0:
-        tm_mon = 1
-    tm_mday = space.int_w(tup_w[2])
-    if tm_mday == 0:
-        tm_mday = 1
-    tm_yday = space.int_w(tup_w[7])
-    if tm_yday == 0:
-        tm_yday = 1
-    rffi.setintfield(glob_buf, 'c_tm_mon', tm_mon)
-    rffi.setintfield(glob_buf, 'c_tm_mday', tm_mday)
-    rffi.setintfield(glob_buf, 'c_tm_hour', space.int_w(tup_w[3]))
-    rffi.setintfield(glob_buf, 'c_tm_min', space.int_w(tup_w[4]))
-    rffi.setintfield(glob_buf, 'c_tm_sec', space.int_w(tup_w[5]))
-    rffi.setintfield(glob_buf, 'c_tm_wday', space.int_w(tup_w[6]))
-    rffi.setintfield(glob_buf, 'c_tm_yday', tm_yday)
-    rffi.setintfield(glob_buf, 'c_tm_isdst', space.int_w(tup_w[8]))
-    if _POSIX:
-        if _CYGWIN:
-            pass
-        else:
-            # actually never happens, but makes annotator happy
-            glob_buf.c_tm_zone = lltype.nullptr(rffi.CCHARP.TO)
-            rffi.setintfield(glob_buf, 'c_tm_gmtoff', 0)
-
-    if y < 1900:
-        w_accept2dyear = _get_module_object(space, "accept2dyear")
-        accept2dyear = space.int_w(w_accept2dyear)
-
-        if not accept2dyear:
-            raise OperationError(space.w_ValueError,
-                space.wrap("year >= 1900 required"))
-
-        if 69 <= y <= 99:
-            y += 1900
-        elif 0 <= y <= 68:
-            y += 2000
-        else:
-            raise OperationError(space.w_ValueError,
-                space.wrap("year out of range"))
-
-    # tm_wday does not need checking of its upper-bound since taking "%
-    #  7" in gettmarg() automatically restricts the range.
-    if rffi.getintfield(glob_buf, 'c_tm_wday') < -1:
-        raise OperationError(space.w_ValueError,
-                             space.wrap("day of week out of range"))
-
-    rffi.setintfield(glob_buf, 'c_tm_year', y - 1900)
-    rffi.setintfield(glob_buf, 'c_tm_mon',
-                     rffi.getintfield(glob_buf, 'c_tm_mon') - 1)
-    rffi.setintfield(glob_buf, 'c_tm_wday',
-                     (rffi.getintfield(glob_buf, 'c_tm_wday') + 1) % 7)
-    rffi.setintfield(glob_buf, 'c_tm_yday',
-                     rffi.getintfield(glob_buf, 'c_tm_yday') - 1)
-
-    return glob_buf
-
-def time(space):
-    """time() -> floating point number
-
-    Return the current time in seconds since the Epoch.
-    Fractions of a second may be present if the system clock provides them."""
-
-    secs = pytime.time()
-    return space.wrap(secs)
-
-if _WIN:
-    class PCCache:
-        pass
-    pccache = PCCache()
-    pccache.divisor = 0.0
-    pccache.ctrStart = 0
-
-def clock(space):
-    """clock() -> floating point number
-
-    Return the CPU time or real time since the start of the process or since
-    the first call to clock().  This has as much precision as the system
-    records."""
-
-    return space.wrap(pytime.clock())
-
-def ctime(space, w_seconds=None):
-    """ctime([seconds]) -> string
-
-    Convert a time in seconds since the Epoch to a string in local time.
-    This is equivalent to asctime(localtime(seconds)). When the time tuple is
-    not present, current time as returned by localtime() is used."""
-
-    seconds = _get_inttime(space, w_seconds)
-
-    t_ref = lltype.malloc(rffi.TIME_TP.TO, 1, flavor='raw')
-    t_ref[0] = seconds
-    p = c_ctime(t_ref)
-    lltype.free(t_ref, flavor='raw')
-    if not p:
-        raise OperationError(space.w_ValueError,
-            space.wrap("unconvertible time"))
-
-    return space.wrap(rffi.charp2str(p)[:-1]) # get rid of new line
-
-# by now w_tup is an optional argument (and not *args)
-# because of the ext. compiler bugs in handling such arguments (*args, **kwds)
-def asctime(space, w_tup=None):
-    """asctime([tuple]) -> string
-
-    Convert a time tuple to a string, e.g. 'Sat Jun 06 16:26:11 1998'.
-    When the time tuple is not present, current time as returned by localtime()
-    is used."""
-    buf_value = _gettmarg(space, w_tup)
-    p = c_asctime(buf_value)
-    if not p:
-        raise OperationError(space.w_ValueError,
-            space.wrap("unconvertible time"))
-
-    return space.wrap(rffi.charp2str(p)[:-1]) # get rid of new line
-
-def gmtime(space, w_seconds=None):
-    """gmtime([seconds]) -> (tm_year, tm_mon, tm_day, tm_hour, tm_min,
-                          tm_sec, tm_wday, tm_yday, tm_isdst)
-
-    Convert seconds since the Epoch to a time tuple expressing UTC (a.k.a.
-    GMT).  When 'seconds' is not passed in, convert the current time instead.
-    """
-
-    # rpython does not support that a variable has two incompatible builtins
-    # as value so we have to duplicate the code. NOT GOOD! see localtime() too
-    seconds = _get_inttime(space, w_seconds)
-    t_ref = lltype.malloc(rffi.TIME_TP.TO, 1, flavor='raw')
-    t_ref[0] = seconds
-    p = c_gmtime(t_ref)
-    lltype.free(t_ref, flavor='raw')
-
-    if not p:
-        raise OperationError(space.w_ValueError, space.wrap(_get_error_msg()))
-    return _tm_to_tuple(space, p)
-
-def localtime(space, w_seconds=None):
-    """localtime([seconds]) -> (tm_year, tm_mon, tm_day, tm_hour, tm_min,
-                             tm_sec, tm_wday, tm_yday, tm_isdst)
-
-    Convert seconds since the Epoch to a time tuple expressing local time.
-    When 'seconds' is not passed in, convert the current time instead."""
-
-    seconds = _get_inttime(space, w_seconds)
-    t_ref = lltype.malloc(rffi.TIME_TP.TO, 1, flavor='raw')
-    t_ref[0] = seconds
-    p = c_localtime(t_ref)
-    lltype.free(t_ref, flavor='raw')
-
-    if not p:
-        raise OperationError(space.w_ValueError, space.wrap(_get_error_msg()))
-    return _tm_to_tuple(space, p)
-
-def mktime(space, w_tup):
-    """mktime(tuple) -> floating point number
-
-    Convert a time tuple in local time to seconds since the Epoch."""
-
-    buf = _gettmarg(space, w_tup, allowNone=False)
-    rffi.setintfield(buf, "c_tm_wday", -1)
-    tt = c_mktime(buf)
-    # A return value of -1 does not necessarily mean an error, but tm_wday
-    # cannot remain set to -1 if mktime succeeds.
-    if tt == -1 and rffi.getintfield(buf, "c_tm_wday") == -1:
-        raise OperationError(space.w_OverflowError,
-            space.wrap("mktime argument out of range"))
-
-    return space.wrap(float(tt))
-
-if _POSIX:
-    def tzset(space):
-        """tzset()
-
-        Initialize, or reinitialize, the local timezone to the value stored in
-        os.environ['TZ']. The TZ environment variable should be specified in
-        standard Unix timezone format as documented in the tzset man page
-        (eg. 'US/Eastern', 'Europe/Amsterdam'). Unknown timezones will silently
-        fall back to UTC. If the TZ environment variable is not set, the local
-        timezone is set to the systems best guess of wallclock time.
-        Changing the TZ environment variable without calling tzset *may* change
-        the local timezone used by methods such as localtime, but this behaviour
-        should not be relied on"""
-
-        c_tzset()
-
-        # reset timezone, altzone, daylight and tzname
-        _init_timezone(space)
-
- at unwrap_spec(format=str)
-def strftime(space, format, w_tup=None):
-    """strftime(format[, tuple]) -> string
-
-    Convert a time tuple to a string according to a format specification.
-    See the library reference manual for formatting codes. When the time tuple
-    is not present, current time as returned by localtime() is used."""
-    buf_value = _gettmarg(space, w_tup)
-
-    # Checks added to make sure strftime() does not crash Python by
-    # indexing blindly into some array for a textual representation
-    # by some bad index (fixes bug #897625).
-    # No check for year since handled in gettmarg().
-    if rffi.getintfield(buf_value, 'c_tm_mon') < 0 or rffi.getintfield(buf_value, 'c_tm_mon') > 11:
-        raise OperationError(space.w_ValueError,
-                             space.wrap("month out of range"))
-    if rffi.getintfield(buf_value, 'c_tm_mday') < 1 or rffi.getintfield(buf_value, 'c_tm_mday') > 31:
-        raise OperationError(space.w_ValueError,
-                             space.wrap("day of month out of range"))
-    if rffi.getintfield(buf_value, 'c_tm_hour') < 0 or rffi.getintfield(buf_value, 'c_tm_hour') > 23:
-        raise OperationError(space.w_ValueError,
-                             space.wrap("hour out of range"))
-    if rffi.getintfield(buf_value, 'c_tm_min') < 0 or rffi.getintfield(buf_value, 'c_tm_min') > 59:
-        raise OperationError(space.w_ValueError,
-                             space.wrap("minute out of range"))
-    if rffi.getintfield(buf_value, 'c_tm_sec') < 0 or rffi.getintfield(buf_value, 'c_tm_sec') > 61:
-        raise OperationError(space.w_ValueError,
-                             space.wrap("seconds out of range"))
-    if rffi.getintfield(buf_value, 'c_tm_yday') < 0 or rffi.getintfield(buf_value, 'c_tm_yday') > 365:
-        raise OperationError(space.w_ValueError,
-                             space.wrap("day of year out of range"))
-    if rffi.getintfield(buf_value, 'c_tm_isdst') < -1 or rffi.getintfield(buf_value, 'c_tm_isdst') > 1:
-        raise OperationError(space.w_ValueError,
-                             space.wrap("daylight savings flag out of range"))
-
-    if _WIN:
-        # check that the format string contains only valid directives
-        length = len(format)
-        i = 0
-        while i < length:
-            if format[i] == '%':
-                i += 1
-                if i < length and format[i] == '#':
-                    # not documented by python
-                    i += 1
-                if i >= length or format[i] not in "aAbBcdHIjmMpSUwWxXyYzZ%":
-                    raise OperationError(space.w_ValueError,
-                                         space.wrap("invalid format string"))
-            i += 1
-
-    i = 1024
-    while True:
-        outbuf = lltype.malloc(rffi.CCHARP.TO, i, flavor='raw')
-        try:
-            buflen = c_strftime(outbuf, i, format, buf_value)
-            if buflen > 0 or i >= 256 * len(format):
-                # if the buffer is 256 times as long as the format,
-                # it's probably not failing for lack of room!
-                # More likely, the format yields an empty result,
-                # e.g. an empty format, or %Z when the timezone
-                # is unknown.
-                result = rffi.charp2strn(outbuf, intmask(buflen))
-                return space.wrap(result)
-        finally:
-            lltype.free(outbuf, flavor='raw')
-        i += i
diff --git a/pypy/module/rctime/test/test_rctime.py b/pypy/module/rctime/test/test_rctime.py
deleted file mode 100644
--- a/pypy/module/rctime/test/test_rctime.py
+++ /dev/null
@@ -1,333 +0,0 @@
-class AppTestRCTime:
-    spaceconfig = {
-        "usemodules": ['rctime', 'struct', 'binascii'],
-    }
-
-    def test_attributes(self):
-        import time as rctime
-        assert isinstance(rctime.accept2dyear, int)
-        assert isinstance(rctime.altzone, int)
-        assert isinstance(rctime.daylight, int)
-        assert isinstance(rctime.timezone, int)
-        assert isinstance(rctime.tzname, tuple)
-        assert isinstance(rctime.__doc__, str)
-
-    def test_sleep(self):
-        import time as rctime
-        import sys
-        import os
-        raises(TypeError, rctime.sleep, "foo")
-        rctime.sleep(0.12345)
-        raises(IOError, rctime.sleep, -1.0)
-
-    def test_clock(self):
-        import time as rctime
-        rctime.clock()
-        assert isinstance(rctime.clock(), float)
-
-    def test_time(self):
-        import time as rctime
-        t1 = rctime.time()
-        assert isinstance(rctime.time(), float)
-        assert rctime.time() != 0.0 # 0.0 means failure
-        rctime.sleep(0.02)
-        t2 = rctime.time()
-        assert t1 != t2       # the resolution should be at least 0.01 secs
-
-    def test_ctime(self):
-        import time as rctime
-        raises(TypeError, rctime.ctime, "foo")
-        rctime.ctime(None)
-        rctime.ctime()
-        res = rctime.ctime(0)
-        assert isinstance(res, str)
-        rctime.ctime(rctime.time())
-        raises(ValueError, rctime.ctime, 1E200)
-        raises(OverflowError, rctime.ctime, 10**900)
-
-    def test_gmtime(self):
-        import time as rctime
-        raises(TypeError, rctime.gmtime, "foo")
-        rctime.gmtime()
-        rctime.gmtime(None)
-        rctime.gmtime(0)
-        res = rctime.gmtime(rctime.time())
-        assert isinstance(res, rctime.struct_time)
-        assert res[-1] == 0 # DST is always zero in gmtime()
-        t0 = rctime.mktime(rctime.gmtime())
-        t1 = rctime.mktime(rctime.gmtime(None))
-        assert 0 <= (t1 - t0) < 1.2
-        t = rctime.time()
-        assert rctime.gmtime(t) == rctime.gmtime(t)
-        raises(ValueError, rctime.gmtime, 2**64)
-        raises(ValueError, rctime.gmtime, -2**64)
-
-    def test_localtime(self):
-        import time as rctime
-        import os
-        raises(TypeError, rctime.localtime, "foo")
-        rctime.localtime()
-        rctime.localtime(None)
-        rctime.localtime(0)
-        res = rctime.localtime(rctime.time())
-        assert isinstance(res, rctime.struct_time)
-        t0 = rctime.mktime(rctime.localtime())
-        t1 = rctime.mktime(rctime.localtime(None))
-        assert 0 <= (t1 - t0) < 1.2
-        t = rctime.time()
-        assert rctime.localtime(t) == rctime.localtime(t)
-        if os.name == 'nt':
-            raises(ValueError, rctime.localtime, -1)
-        else:
-            rctime.localtime(-1)
-
-    def test_mktime(self):
-        import time as rctime
-        import os, sys
-        raises(TypeError, rctime.mktime, "foo")
-        raises(TypeError, rctime.mktime, None)
-        raises(TypeError, rctime.mktime, (1, 2))
-        raises(TypeError, rctime.mktime, (1, 2, 3, 4, 5, 6, 'f', 8, 9))
-        res = rctime.mktime(rctime.localtime())
-        assert isinstance(res, float)
-
-        ltime = rctime.localtime()
-        rctime.accept2dyear == 0
-        ltime = list(ltime)
-        ltime[0] = 1899
-        raises(ValueError, rctime.mktime, tuple(ltime))
-        rctime.accept2dyear == 1
-
-        ltime = list(ltime)
-        ltime[0] = 67
-        ltime = tuple(ltime)
-        if os.name != "nt" and sys.maxint < 1<<32:   # time_t may be 64bit
-            raises(OverflowError, rctime.mktime, ltime)
-
-        ltime = list(ltime)
-        ltime[0] = 100
-        raises(ValueError, rctime.mktime, tuple(ltime))
-
-        t = rctime.time()
-        assert long(rctime.mktime(rctime.localtime(t))) == long(t)
-        assert long(rctime.mktime(rctime.gmtime(t))) - rctime.timezone == long(t)
-        ltime = rctime.localtime()
-        assert rctime.mktime(tuple(ltime)) == rctime.mktime(ltime)
-        if os.name != 'nt':
-            assert rctime.mktime(rctime.localtime(-1)) == -1
-
-        res = rctime.mktime((2000, 1, 1, 0, 0, 0, -1, -1, -1))
-        if os.name == 'nt':
-            assert rctime.ctime(res) == 'Sat Jan 01 00:00:00 2000'
-        else:
-            assert rctime.ctime(res) == 'Sat Jan  1 00:00:00 2000'
-
-    def test_asctime(self):
-        import time as rctime
-        rctime.asctime()
-        # raises(TypeError, rctime.asctime, None)
-        raises(TypeError, rctime.asctime, ())
-        raises(TypeError, rctime.asctime, (1,))
-        raises(TypeError, rctime.asctime, range(8))
-        raises(TypeError, rctime.asctime, (1, 2))
-        raises(TypeError, rctime.asctime, (1, 2, 3, 4, 5, 6, 'f', 8, 9))
-        raises(TypeError, rctime.asctime, "foo")
-        res = rctime.asctime()
-        assert isinstance(res, str)
-        rctime.asctime(rctime.localtime())
-        t = rctime.time()
-        assert rctime.ctime(t) == rctime.asctime(rctime.localtime(t))
-        if rctime.timezone:
-            assert rctime.ctime(t) != rctime.asctime(rctime.gmtime(t))
-        ltime = rctime.localtime()
-        assert rctime.asctime(tuple(ltime)) == rctime.asctime(ltime)
-        try:
-            rctime.asctime((12345,) + (0,) * 8)  # assert this doesn't crash
-        except ValueError:
-            pass  # some OS (ie POSIXes besides Linux) reject year > 9999
-
-    def test_accept2dyear_access(self):
-        import time as rctime
-
-        accept2dyear = rctime.accept2dyear
-        del rctime.accept2dyear
-        try:
-            # with year >= 1900 this shouldn't need to access accept2dyear
-            assert rctime.asctime((2000,) + (0,) * 8).split()[-1] == '2000'
-        finally:
-            rctime.accept2dyear = accept2dyear
-
-    def test_struct_time(self):
-        import time as rctime
-        raises(TypeError, rctime.struct_time)
-        raises(TypeError, rctime.struct_time, "foo")
-        raises(TypeError, rctime.struct_time, (1, 2, 3))
-        tup = (1, 2, 3, 4, 5, 6, 7, 8, 9)
-        st_time = rctime.struct_time(tup)
-        assert str(st_time).startswith('time.struct_time(tm_year=1, ')
-        assert len(st_time) == len(tup)
-
-    def test_tzset(self):
-        import time as rctime
-        import os
-
-        if not os.name == "posix":
-            skip("tzset available only under Unix")
-
-        # epoch time of midnight Dec 25th 2002. Never DST in northern
-        # hemisphere.
-        xmas2002 = 1040774400.0
-
-        # these formats are correct for 2002, and possibly future years
-        # this format is the 'standard' as documented at:
-        # http://www.opengroup.org/onlinepubs/007904975/basedefs/xbd_chap08.html
-        # They are also documented in the tzset(3) man page on most Unix
-        # systems.
-        eastern = 'EST+05EDT,M4.1.0,M10.5.0'
-        victoria = 'AEST-10AEDT-11,M10.5.0,M3.5.0'
-        utc = 'UTC+0'
-
-        org_TZ = os.environ.get('TZ', None)
-        try:
-            # Make sure we can switch to UTC time and results are correct
-            # Note that unknown timezones default to UTC.
-            # Note that altzone is undefined in UTC, as there is no DST
-            os.environ['TZ'] = eastern
-            rctime.tzset()
-            os.environ['TZ'] = utc
-            rctime.tzset()
-            assert rctime.gmtime(xmas2002) == rctime.localtime(xmas2002)
-            assert rctime.daylight == 0
-            assert rctime.timezone == 0
-            assert rctime.localtime(xmas2002).tm_isdst == 0
-
-            # make sure we can switch to US/Eastern
-            os.environ['TZ'] = eastern
-            rctime.tzset()
-            assert rctime.gmtime(xmas2002) != rctime.localtime(xmas2002)
-            assert rctime.tzname == ('EST', 'EDT')
-            assert len(rctime.tzname) == 2
-            assert rctime.daylight == 1
-            assert rctime.timezone == 18000
-            assert rctime.altzone == 14400
-            assert rctime.localtime(xmas2002).tm_isdst == 0
-
-            # now go to the southern hemisphere.
-            os.environ['TZ'] = victoria
-            rctime.tzset()
-            assert rctime.gmtime(xmas2002) != rctime.localtime(xmas2002)
-            assert rctime.tzname[0] == 'AEST'
-            assert rctime.tzname[1] == 'AEDT'
-            assert len(rctime.tzname) == 2
-            assert rctime.daylight == 1
-            assert rctime.timezone == -36000
-            assert rctime.altzone == -39600
-            assert rctime.localtime(xmas2002).tm_isdst == 1
-        finally:
-            # repair TZ environment variable in case any other tests
-            # rely on it.
-            if org_TZ is not None:
-                os.environ['TZ'] = org_TZ
-            elif os.environ.has_key('TZ'):
-                del os.environ['TZ']
-            rctime.tzset()
-
-    def test_strftime(self):
-        import time as rctime
-        import os, sys
-
-        t = rctime.time()
-        tt = rctime.gmtime(t)
-        for directive in ('a', 'A', 'b', 'B', 'c', 'd', 'H', 'I',
-                          'j', 'm', 'M', 'p', 'S',
-                          'U', 'w', 'W', 'x', 'X', 'y', 'Y', 'Z', '%'):
-            format = ' %' + directive
-            rctime.strftime(format, tt)
-
-        raises(TypeError, rctime.strftime, ())
-        raises(TypeError, rctime.strftime, (1,))
-        raises(TypeError, rctime.strftime, range(8))
-        exp = '2000 01 01 00 00 00 1 001'
-        assert rctime.strftime("%Y %m %d %H %M %S %w %j", (0,)*9) == exp
-
-        # Guard against invalid/non-supported format string
-        # so that Python don't crash (Windows crashes when the format string
-        # input to [w]strftime is not kosher.
-        if os.name == 'nt':
-            raises(ValueError, rctime.strftime, '%f')
-        elif sys.platform == 'darwin' or 'bsd' in sys.platform:
-            # darwin strips % of unknown format codes
-            # http://bugs.python.org/issue9811
-            assert rctime.strftime('%f') == 'f'
-        else:
-            assert rctime.strftime('%f') == '%f'
-
-    def test_strftime_ext(self):
-        import time as rctime
-
-        tt = rctime.gmtime()
-        try:
-            result = rctime.strftime('%D', tt)
-        except ValueError:
-            pass
-        else:
-            assert result == rctime.strftime('%m/%d/%y', tt)
-
-    def test_strftime_bounds_checking(self):
-        import time as rctime
-
-        # make sure that strftime() checks the bounds of the various parts
-        # of the time tuple.
-
-        # check year
-        raises(ValueError, rctime.strftime, '', (1899, 1, 1, 0, 0, 0, 0, 1, -1))
-        if rctime.accept2dyear:
-            raises(ValueError, rctime.strftime, '', (-1, 1, 1, 0, 0, 0, 0, 1, -1))
-            raises(ValueError, rctime.strftime, '', (100, 1, 1, 0, 0, 0, 0, 1, -1))
-        # check month
-        raises(ValueError, rctime.strftime, '', (1900, 13, 1, 0, 0, 0, 0, 1, -1))
-        # check day of month
-        raises(ValueError, rctime.strftime, '', (1900, 1, 32, 0, 0, 0, 0, 1, -1))
-        # check hour
-        raises(ValueError, rctime.strftime, '', (1900, 1, 1, -1, 0, 0, 0, 1, -1))
-        raises(ValueError, rctime.strftime, '', (1900, 1, 1, 24, 0, 0, 0, 1, -1))
-        # check minute
-        raises(ValueError, rctime.strftime, '', (1900, 1, 1, 0, -1, 0, 0, 1, -1))
-        raises(ValueError, rctime.strftime, '', (1900, 1, 1, 0, 60, 0, 0, 1, -1))
-        # check second
-        raises(ValueError, rctime.strftime, '', (1900, 1, 1, 0, 0, -1, 0, 1, -1))
-        # C99 only requires allowing for one leap second, but Python's docs say
-        # allow two leap seconds (0..61)
-        raises(ValueError, rctime.strftime, '', (1900, 1, 1, 0, 0, 62, 0, 1, -1))
-        # no check for upper-bound day of week;
-        #  value forced into range by a "% 7" calculation.
-        # start check at -2 since gettmarg() increments value before taking
-        #  modulo.
-        raises(ValueError, rctime.strftime, '', (1900, 1, 1, 0, 0, 0, -2, 1, -1))
-        # check day of the year
-        raises(ValueError, rctime.strftime, '', (1900, 1, 1, 0, 0, 0, 0, 367, -1))
-        # check daylight savings flag
-        raises(ValueError, rctime.strftime, '', (1900, 1, 1, 0, 0, 0, 0, 1, -2))
-        raises(ValueError, rctime.strftime, '', (1900, 1, 1, 0, 0, 0, 0, 1, 2))
-
-    def test_strptime(self):
-        import time as rctime
-
-        t = rctime.time()
-        tt = rctime.gmtime(t)
-        assert isinstance(rctime.strptime("", ""), type(tt))
-
-        for directive in ('a', 'A', 'b', 'B', 'c', 'd', 'H', 'I',
-                          'j', 'm', 'M', 'p', 'S',
-                          'U', 'w', 'W', 'x', 'X', 'y', 'Y', 'Z', '%'):
-            format = ' %' + directive
-            print format
-            rctime.strptime(rctime.strftime(format, tt), format)
-
-    def test_pickle(self):
-        import pickle
-        import time as rctime
-        now = rctime.localtime()
-        new = pickle.loads(pickle.dumps(now))
-        assert new == now
-        assert type(new) is type(now)
diff --git a/pypy/module/select/test/test_epoll.py b/pypy/module/select/test/test_epoll.py
--- a/pypy/module/select/test/test_epoll.py
+++ b/pypy/module/select/test/test_epoll.py
@@ -7,7 +7,7 @@
 
 class AppTestEpoll(object):
     spaceconfig = {
-        "usemodules": ["select", "_socket", "posix", "rctime"],
+        "usemodules": ["select", "_socket", "posix", "time"],
     }
 
     def setup_class(cls):
diff --git a/pypy/module/select/test/test_kqueue.py b/pypy/module/select/test/test_kqueue.py
--- a/pypy/module/select/test/test_kqueue.py
+++ b/pypy/module/select/test/test_kqueue.py
@@ -4,7 +4,7 @@
 import sys
 
 class AppTestKqueue(object):
-    spaceconfig = dict(usemodules=["select", "_socket", "posix", "rctime"])
+    spaceconfig = dict(usemodules=["select", "_socket", "posix", "time"])
 
     def setup_class(cls):
         if not 'bsd' in sys.platform and \
diff --git a/pypy/module/select/test/test_select.py b/pypy/module/select/test/test_select.py
--- a/pypy/module/select/test/test_select.py
+++ b/pypy/module/select/test/test_select.py
@@ -249,7 +249,7 @@
 class AppTestSelectWithPipes(_AppTestSelect):
     "Use a pipe to get pairs of file descriptors"
     spaceconfig = {
-        "usemodules": ["select", "rctime", "thread"]
+        "usemodules": ["select", "time", "thread"]
     }
 
     def setup_class(cls):
@@ -325,7 +325,7 @@
     so we start our own server.
     """
     spaceconfig = {
-        "usemodules": ["select", "_socket", "rctime", "thread"],
+        "usemodules": ["select", "_socket", "time", "thread"],
     }
 
     def w_make_server(self):
diff --git a/pypy/module/signal/test/test_signal.py b/pypy/module/signal/test/test_signal.py
--- a/pypy/module/signal/test/test_signal.py
+++ b/pypy/module/signal/test/test_signal.py
@@ -36,7 +36,7 @@
 
 class AppTestSignal:
     spaceconfig = {
-        "usemodules": ['signal', 'rctime'] + (['fcntl'] if os.name != 'nt' else []),
+        "usemodules": ['signal', 'time'] + (['fcntl'] if os.name != 'nt' else []),
     }
 
     def setup_class(cls):
diff --git a/pypy/module/sys/test/test_sysmodule.py b/pypy/module/sys/test/test_sysmodule.py
--- a/pypy/module/sys/test/test_sysmodule.py
+++ b/pypy/module/sys/test/test_sysmodule.py
@@ -619,7 +619,7 @@
 
 class AppTestCurrentFramesWithThread(AppTestCurrentFrames):
     spaceconfig = {
-        "usemodules": ["rctime", "thread"],
+        "usemodules": ["time", "thread"],
     }
 
     def test_current_frames(self):
diff --git a/pypy/module/thread/test/support.py b/pypy/module/thread/test/support.py
--- a/pypy/module/thread/test/support.py
+++ b/pypy/module/thread/test/support.py
@@ -41,7 +41,7 @@
 
 
 class GenericTestThread:
-    spaceconfig = dict(usemodules=('thread', 'rctime', 'signal'))
+    spaceconfig = dict(usemodules=('thread', 'time', 'signal'))
 
     def setup_class(cls):
         cls.w_runappdirect = cls.space.wrap(cls.runappdirect)
diff --git a/pypy/module/time/__init__.py b/pypy/module/time/__init__.py
--- a/pypy/module/time/__init__.py
+++ b/pypy/module/time/__init__.py
@@ -1,13 +1,42 @@
+
 from pypy.interpreter.mixedmodule import MixedModule
+import os
+
+_WIN = os.name == "nt"
 
 class Module(MixedModule):
-    """time module"""
+    applevel_name = 'time'
+
+    interpleveldefs = {
+        'time': 'interp_time.time',
+        'clock': 'interp_time.clock',
+        'ctime': 'interp_time.ctime',
+        'asctime': 'interp_time.asctime',
+        'gmtime': 'interp_time.gmtime',
+        'localtime': 'interp_time.localtime',
+        'mktime': 'interp_time.mktime',
+        'strftime': 'interp_time.strftime',
+        'sleep' : 'interp_time.sleep',
+    }
+
+    if os.name == "posix":
+        interpleveldefs['tzset'] = 'interp_time.tzset'
 
     appleveldefs = {
+        'struct_time': 'app_time.struct_time',
+        '__doc__': 'app_time.__doc__',
+        'strptime': 'app_time.strptime',
     }
 
-    interpleveldefs = {
-        'clock'    : 'interp_time.clock',
-        'time'     : 'interp_time.time_',
-        'sleep'    : 'interp_time.sleep',
-    }
+    def startup(self, space):
+        if _WIN:
+            from pypy.module.time.interp_time import State
+            space.fromcache(State).startup(space)
+
+        # this machinery is needed to expose constants
+        # that have to be initialized one time only
+        from pypy.module.time import interp_time
+
+        interp_time._init_timezone(space)
+        interp_time._init_accept2dyear(space)
+
diff --git a/pypy/module/rctime/app_time.py b/pypy/module/time/app_time.py
rename from pypy/module/rctime/app_time.py
rename to pypy/module/time/app_time.py
diff --git a/pypy/module/time/interp_time.py b/pypy/module/time/interp_time.py
--- a/pypy/module/time/interp_time.py
+++ b/pypy/module/time/interp_time.py
@@ -1,20 +1,670 @@
-import time
+from rpython.rtyper.tool import rffi_platform as platform
+from rpython.rtyper.lltypesystem import rffi
+from pypy.interpreter.error import OperationError, oefmt
 from pypy.interpreter.gateway import unwrap_spec
+from rpython.rtyper.lltypesystem import lltype
+from rpython.rlib.rarithmetic import intmask
+from rpython.rlib import rposix
+from rpython.translator.tool.cbuild import ExternalCompilationInfo
+import os
+import sys
+import time as pytime
 
+_POSIX = os.name == "posix"
+_WIN = os.name == "nt"
+_CYGWIN = sys.platform == "cygwin"
+
+_time_zones = []
+if _CYGWIN:
+    _time_zones = ["GMT-12", "GMT-11", "GMT-10", "GMT-9", "GMT-8", "GMT-7",
+                   "GMT-6", "GMT-5", "GMT-4", "GMT-3", "GMT-2", "GMT-1",
+                   "GMT",  "GMT+1", "GMT+2", "GMT+3", "GMT+4", "GMT+5",
+                   "GMT+6",  "GMT+7", "GMT+8", "GMT+9", "GMT+10", "GMT+11",
+                   "GMT+12",  "GMT+13", "GMT+14"]
+
+if _WIN:
+    # Interruptible sleeps on Windows:
+    # We install a specific Console Ctrl Handler which sets an 'event'.
+    # time.sleep() will actually call WaitForSingleObject with the desired
+    # timeout.  On Ctrl-C, the signal handler is called, the event is set,
+    # and the wait function exits.
+    from rpython.rlib import rwin32
+    from pypy.interpreter.error import wrap_windowserror, wrap_oserror
+    from rpython.rlib import rthread as thread
+
+    eci = ExternalCompilationInfo(
+        includes = ['windows.h'],
+        post_include_bits = [
+            "RPY_EXTERN\n"
+            "BOOL pypy_timemodule_setCtrlHandler(HANDLE event);"],
+        separate_module_sources=['''
+            static HANDLE interrupt_event;
+
+            static BOOL WINAPI CtrlHandlerRoutine(
+              DWORD dwCtrlType)
+            {
+                SetEvent(interrupt_event);
+                /* allow other default handlers to be called.
+                 * Default Python handler will setup the
+                 * KeyboardInterrupt exception.
+                 */
+                return 0;
+            }
+
+            BOOL pypy_timemodule_setCtrlHandler(HANDLE event)
+            {
+                interrupt_event = event;
+                return SetConsoleCtrlHandler(CtrlHandlerRoutine, TRUE);
+            }
+
+        '''],
+        )
+    _setCtrlHandlerRoutine = rffi.llexternal(
+        'pypy_timemodule_setCtrlHandler',
+        [rwin32.HANDLE], rwin32.BOOL,
+        compilation_info=eci)
+
+    class GlobalState:
+        def __init__(self):
+            self.init()
+
+        def init(self):
+            self.interrupt_event = rwin32.NULL_HANDLE
+
+        def startup(self, space):
+            # Initialize the event handle used to signal Ctrl-C
+            try:
+                globalState.interrupt_event = rwin32.CreateEvent(
+                    rffi.NULL, True, False, rffi.NULL)
+            except WindowsError, e:
+                raise wrap_windowserror(space, e)
+            if not _setCtrlHandlerRoutine(globalState.interrupt_event):
+                raise wrap_windowserror(
+                    space, rwin32.lastWindowsError("SetConsoleCtrlHandler"))
+
+    globalState = GlobalState()
+
+    class State:
+        def __init__(self, space):
+            self.main_thread = 0
+
+        def _cleanup_(self):
+            self.main_thread = 0
+            globalState.init()
+
+        def startup(self, space):
+            self.main_thread = thread.get_ident()
+            globalState.startup(space)
+
+        def get_interrupt_event(self):
+            return globalState.interrupt_event
+
+
+_includes = ["time.h"]
+if _POSIX:
+    _includes.append('sys/time.h')
+
+class CConfig:
+    _compilation_info_ = ExternalCompilationInfo(
+        includes = _includes
+    )
+    CLOCKS_PER_SEC = platform.ConstantInteger("CLOCKS_PER_SEC")
+    clock_t = platform.SimpleType("clock_t", rffi.ULONG)
+    has_gettimeofday = platform.Has('gettimeofday')
+
+if _POSIX:
+    calling_conv = 'c'
+    CConfig.timeval = platform.Struct("struct timeval",
+                                      [("tv_sec", rffi.INT),
+                                       ("tv_usec", rffi.INT)])
+    if _CYGWIN:
+        CConfig.tm = platform.Struct("struct tm", [("tm_sec", rffi.INT),
+            ("tm_min", rffi.INT), ("tm_hour", rffi.INT), ("tm_mday", rffi.INT),
+            ("tm_mon", rffi.INT), ("tm_year", rffi.INT), ("tm_wday", rffi.INT),
+            ("tm_yday", rffi.INT), ("tm_isdst", rffi.INT)])
+    else:
+        CConfig.tm = platform.Struct("struct tm", [("tm_sec", rffi.INT),
+            ("tm_min", rffi.INT), ("tm_hour", rffi.INT), ("tm_mday", rffi.INT),
+            ("tm_mon", rffi.INT), ("tm_year", rffi.INT), ("tm_wday", rffi.INT),
+            ("tm_yday", rffi.INT), ("tm_isdst", rffi.INT), ("tm_gmtoff", rffi.LONG),
+            ("tm_zone", rffi.CCHARP)])
+elif _WIN:
+    calling_conv = 'win'
+    CConfig.tm = platform.Struct("struct tm", [("tm_sec", rffi.INT),
+        ("tm_min", rffi.INT), ("tm_hour", rffi.INT), ("tm_mday", rffi.INT),
+        ("tm_mon", rffi.INT), ("tm_year", rffi.INT), ("tm_wday", rffi.INT),
+        ("tm_yday", rffi.INT), ("tm_isdst", rffi.INT)])
+
+class cConfig:
+    pass
+
+for k, v in platform.configure(CConfig).items():
+    setattr(cConfig, k, v)
+cConfig.tm.__name__ = "_tm"
+
+def external(name, args, result, eci=CConfig._compilation_info_):
+    if _WIN and rffi.sizeof(rffi.TIME_T) == 8:
+        # Recent Microsoft compilers use 64bit time_t and
+        # the corresponding functions are named differently
+        if (rffi.TIME_T in args or rffi.TIME_TP in args
+            or result in (rffi.TIME_T, rffi.TIME_TP)):
+            name = '_' + name + '64'
+    return rffi.llexternal(name, args, result,
+                           compilation_info=eci,
+                           calling_conv=calling_conv,
+                           releasegil=False)
+
+if _POSIX:
+    cConfig.timeval.__name__ = "_timeval"
+    timeval = cConfig.timeval
+
+CLOCKS_PER_SEC = cConfig.CLOCKS_PER_SEC
+clock_t = cConfig.clock_t
+tm = cConfig.tm
+glob_buf = lltype.malloc(tm, flavor='raw', zero=True, immortal=True)
+
+if cConfig.has_gettimeofday:
+    c_gettimeofday = external('gettimeofday', [rffi.VOIDP, rffi.VOIDP], rffi.INT)
+TM_P = lltype.Ptr(tm)
+c_clock = external('clock', [rffi.TIME_TP], clock_t)
+c_time = external('time', [rffi.TIME_TP], rffi.TIME_T)
+c_ctime = external('ctime', [rffi.TIME_TP], rffi.CCHARP)
+c_gmtime = external('gmtime', [rffi.TIME_TP], TM_P)
+c_mktime = external('mktime', [TM_P], rffi.TIME_T)
+c_asctime = external('asctime', [TM_P], rffi.CCHARP)
+c_localtime = external('localtime', [rffi.TIME_TP], TM_P)
+if _POSIX:
+    c_tzset = external('tzset', [], lltype.Void)
+if _WIN:
+    win_eci = ExternalCompilationInfo(
+        includes = ["time.h"],
+        post_include_bits = ["RPY_EXTERN "
+                             "long pypy_get_timezone();\n"
+                             "RPY_EXTERN "
+                             "int pypy_get_daylight();\n"
+                             "RPY_EXTERN "
+                             "char** pypy_get_tzname();\n"
+                             "RPY_EXTERN "
+                             "void pypy__tzset();"],
+        separate_module_sources = ["""
+        long pypy_get_timezone() { return timezone; }
+        int pypy_get_daylight() { return daylight; }
+        char** pypy_get_tzname() { return tzname; }
+        void pypy__tzset() { _tzset(); }
+        """])
+    # Ensure sure that we use _tzset() and timezone from the same C Runtime.
+    c_tzset = external('pypy__tzset', [], lltype.Void, win_eci)
+    c_get_timezone = external('pypy_get_timezone', [], rffi.LONG, win_eci)
+    c_get_daylight = external('pypy_get_daylight', [], rffi.INT, win_eci)
+    c_get_tzname = external('pypy_get_tzname', [], rffi.CCHARPP, win_eci)
+
+c_strftime = external('strftime', [rffi.CCHARP, rffi.SIZE_T, rffi.CCHARP, TM_P],
+                      rffi.SIZE_T)
+
+def _init_accept2dyear(space):
+    if os.environ.get("PYTHONY2K"):
+        accept2dyear = 0
+    else:
+        accept2dyear = 1
+    _set_module_object(space, "accept2dyear", space.wrap(accept2dyear))
+
+def _init_timezone(space):
+    timezone = daylight = altzone = 0
+    tzname = ["", ""]
+
+    if _WIN:
+        c_tzset()
+        timezone = c_get_timezone()
+        altzone = timezone - 3600
+        daylight = c_get_daylight()
+        tzname_ptr = c_get_tzname()
+        tzname = rffi.charp2str(tzname_ptr[0]), rffi.charp2str(tzname_ptr[1])
+
+    if _POSIX:
+        if _CYGWIN:
+            YEAR = (365 * 24 + 6) * 3600
+
+            # about January 11th
+            t = (((c_time(lltype.nullptr(rffi.TIME_TP.TO))) / YEAR) * YEAR + 10 * 24 * 3600)
+            # we cannot have reference to stack variable, put it on the heap
+            t_ref = lltype.malloc(rffi.TIME_TP.TO, 1, flavor='raw')
+            t_ref[0] = rffi.cast(rffi.TIME_T, t)
+            p = c_localtime(t_ref)
+            q = c_gmtime(t_ref)
+            janzone = (p.c_tm_hour + 24 * p.c_tm_mday) - (q.c_tm_hour + 24 * q.c_tm_mday)
+            if janzone < -12:
+                janname = "   "
+            elif janzone > 14:
+                janname = "   "
+            else:
+                janname = _time_zones[janzone - 12]
+            janzone = janzone * 3600
+            # about July 11th
+            tt = t + YEAR / 2
+            t_ref[0] = rffi.cast(rffi.TIME_T, tt)
+            p = c_localtime(t_ref)
+            q = c_gmtime(t_ref)
+            julyzone = (p.c_tm_hour + 24 * p.c_tm_mday) - (q.c_tm_hour + 24 * q.c_tm_mday)
+            if julyzone < -12:
+                julyname = "   "
+            elif julyzone > 14:
+                julyname = "   "
+            else:
+                julyname = _time_zones[julyzone - 12]
+            julyzone = julyzone * 3600
+            lltype.free(t_ref, flavor='raw')
+
+            if janzone < julyzone:
+                # DST is reversed in the southern hemisphere
+                timezone = julyzone
+                altzone = janzone
+                daylight = int(janzone != julyzone)
+                tzname = [julyname, janname]
+            else:
+                timezone = janzone
+                altzone = julyzone
+                daylight = int(janzone != julyzone)
+                tzname = [janname, julyname]
+
+        else:
+            YEAR = (365 * 24 + 6) * 3600
+
+            t = (((c_time(lltype.nullptr(rffi.TIME_TP.TO))) / YEAR) * YEAR)
+            # we cannot have reference to stack variable, put it on the heap
+            t_ref = lltype.malloc(rffi.TIME_TP.TO, 1, flavor='raw')
+            t_ref[0] = rffi.cast(rffi.TIME_T, t)
+            p = c_localtime(t_ref)
+            janzone = -p.c_tm_gmtoff
+            tm_zone = rffi.charp2str(p.c_tm_zone)
+            janname = ["   ", tm_zone][bool(tm_zone)]
+            tt = t + YEAR / 2
+            t_ref[0] = rffi.cast(rffi.TIME_T, tt)
+            p = c_localtime(t_ref)
+            lltype.free(t_ref, flavor='raw')
+            tm_zone = rffi.charp2str(p.c_tm_zone)
+            julyzone = -p.c_tm_gmtoff
+            julyname = ["   ", tm_zone][bool(tm_zone)]
+
+            if janzone < julyzone:
+                # DST is reversed in the southern hemisphere
+                timezone = julyzone
+                altzone = janzone
+                daylight = int(janzone != julyzone)
+                tzname = [julyname, janname]
+            else:
+                timezone = janzone
+                altzone = julyzone
+                daylight = int(janzone != julyzone)
+                tzname = [janname, julyname]
+
+    _set_module_object(space, "timezone", space.wrap(timezone))
+    _set_module_object(space, 'daylight', space.wrap(daylight))
+    tzname_w = [space.wrap(tzname[0]), space.wrap(tzname[1])]
+    _set_module_object(space, 'tzname', space.newtuple(tzname_w))
+    _set_module_object(space, 'altzone', space.wrap(altzone))
+
+def _get_error_msg():
+    errno = rposix.get_errno()
+    return os.strerror(errno)
+
+if sys.platform != 'win32':
+    @unwrap_spec(secs=float)
+    def sleep(space, secs):
+        if secs < 0:
+            raise OperationError(space.w_IOError,
+                                 space.wrap("Invalid argument: negative time in sleep"))
+        pytime.sleep(secs)
+else:
+    from rpython.rlib import rwin32
+    from errno import EINTR
+    def _simple_sleep(space, secs, interruptible):
+        if secs == 0.0 or not interruptible:
+            pytime.sleep(secs)
+        else:
+            millisecs = int(secs * 1000)
+            interrupt_event = space.fromcache(State).get_interrupt_event()
+            rwin32.ResetEvent(interrupt_event)
+            rc = rwin32.WaitForSingleObject(interrupt_event, millisecs)
+            if rc == rwin32.WAIT_OBJECT_0:
+                # Yield to make sure real Python signal handler
+                # called.
+                pytime.sleep(0.001)
+                raise wrap_oserror(space,
+                                   OSError(EINTR, "sleep() interrupted"))
+    @unwrap_spec(secs=float)
+    def sleep(space, secs):
+        if secs < 0:
+            raise OperationError(space.w_IOError,
+                                 space.wrap("Invalid argument: negative time in sleep"))
+        # as decreed by Guido, only the main thread can be
+        # interrupted.
+        main_thread = space.fromcache(State).main_thread
+        interruptible = (main_thread == thread.get_ident())
+        MAX = sys.maxint / 1000.0 # > 24 days
+        while secs > MAX:
+            _simple_sleep(space, MAX, interruptible)
+            secs -= MAX
+        _simple_sleep(space, secs, interruptible)
+
+def _get_module_object(space, obj_name):
+    w_module = space.getbuiltinmodule('time')
+    w_obj = space.getattr(w_module, space.wrap(obj_name))
+    return w_obj
+
+def _set_module_object(space, obj_name, w_obj_value):
+    w_module = space.getbuiltinmodule('time')
+    space.setattr(w_module, space.wrap(obj_name), w_obj_value)
+
+def _get_inttime(space, w_seconds):
+    # w_seconds can be a wrapped None (it will be automatically wrapped
+    # in the callers, so we never get a real None here).
+    if space.is_none(w_seconds):
+        seconds = pytime.time()
+    else:
+        seconds = space.float_w(w_seconds)
+    #
+    t = rffi.cast(rffi.TIME_T, seconds)
+    #
+    # Logic from CPython: How much info did we lose?  We assume that
+    # time_t is an integral type.  If we lost a second or more, the
+    # input doesn't fit in a time_t; call it an error.
+    diff = seconds - rffi.cast(lltype.Float, t)
+    if diff <= -1.0 or diff >= 1.0:
+        raise OperationError(space.w_ValueError,
+                      space.wrap("timestamp out of range for platform time_t"))
+    return t
+
+def _tm_to_tuple(space, t):
+    time_tuple = [
+        space.wrap(rffi.getintfield(t, 'c_tm_year') + 1900),
+        space.wrap(rffi.getintfield(t, 'c_tm_mon') + 1), # want january == 1
+        space.wrap(rffi.getintfield(t, 'c_tm_mday')),
+        space.wrap(rffi.getintfield(t, 'c_tm_hour')),
+        space.wrap(rffi.getintfield(t, 'c_tm_min')),
+        space.wrap(rffi.getintfield(t, 'c_tm_sec')),
+        space.wrap((rffi.getintfield(t, 'c_tm_wday') + 6) % 7), # want monday == 0
+        space.wrap(rffi.getintfield(t, 'c_tm_yday') + 1), # want january, 1 == 1
+        space.wrap(rffi.getintfield(t, 'c_tm_isdst'))]
+
+    w_struct_time = _get_module_object(space, 'struct_time')
+    w_time_tuple = space.newtuple(time_tuple)
+    return space.call_function(w_struct_time, w_time_tuple)
+
+def _gettmarg(space, w_tup, allowNone=True):
+    if space.is_none(w_tup):
+        if not allowNone:
+            raise OperationError(space.w_TypeError,
+                                 space.wrap("tuple expected"))
+        # default to the current local time
+        tt = rffi.r_time_t(int(pytime.time()))
+        t_ref = lltype.malloc(rffi.TIME_TP.TO, 1, flavor='raw')
+        t_ref[0] = tt
+        pbuf = c_localtime(t_ref)
+        lltype.free(t_ref, flavor='raw')
+        if not pbuf:
+            raise OperationError(space.w_ValueError,
+                space.wrap(_get_error_msg()))
+        return pbuf
+
+    tup_w = space.fixedview(w_tup)
+    if len(tup_w) != 9:
+        raise oefmt(space.w_TypeError,
+                    "argument must be sequence of length 9, not %d",
+                    len(tup_w))
+
+    y = space.int_w(tup_w[0])
+    tm_mon = space.int_w(tup_w[1])
+    if tm_mon == 0:
+        tm_mon = 1
+    tm_mday = space.int_w(tup_w[2])
+    if tm_mday == 0:
+        tm_mday = 1
+    tm_yday = space.int_w(tup_w[7])
+    if tm_yday == 0:
+        tm_yday = 1
+    rffi.setintfield(glob_buf, 'c_tm_mon', tm_mon)
+    rffi.setintfield(glob_buf, 'c_tm_mday', tm_mday)
+    rffi.setintfield(glob_buf, 'c_tm_hour', space.int_w(tup_w[3]))
+    rffi.setintfield(glob_buf, 'c_tm_min', space.int_w(tup_w[4]))
+    rffi.setintfield(glob_buf, 'c_tm_sec', space.int_w(tup_w[5]))
+    rffi.setintfield(glob_buf, 'c_tm_wday', space.int_w(tup_w[6]))
+    rffi.setintfield(glob_buf, 'c_tm_yday', tm_yday)
+    rffi.setintfield(glob_buf, 'c_tm_isdst', space.int_w(tup_w[8]))
+    if _POSIX:
+        if _CYGWIN:
+            pass
+        else:
+            # actually never happens, but makes annotator happy
+            glob_buf.c_tm_zone = lltype.nullptr(rffi.CCHARP.TO)
+            rffi.setintfield(glob_buf, 'c_tm_gmtoff', 0)
+
+    if y < 1900:
+        w_accept2dyear = _get_module_object(space, "accept2dyear")
+        accept2dyear = space.int_w(w_accept2dyear)
+
+        if not accept2dyear:
+            raise OperationError(space.w_ValueError,
+                space.wrap("year >= 1900 required"))
+
+        if 69 <= y <= 99:
+            y += 1900
+        elif 0 <= y <= 68:
+            y += 2000
+        else:
+            raise OperationError(space.w_ValueError,
+                space.wrap("year out of range"))
+
+    # tm_wday does not need checking of its upper-bound since taking "%
+    #  7" in gettmarg() automatically restricts the range.
+    if rffi.getintfield(glob_buf, 'c_tm_wday') < -1:

From noreply at buildbot.pypy.org  Thu Nov 13 22:37:09 2014
From: noreply at buildbot.pypy.org (amauryfa)
Date: Thu, 13 Nov 2014 22:37:09 +0100 (CET)
Subject: [pypy-commit] pypy default: Don't know whether these modules
 actually work on sunos5,
 but the excuse 'depends on ctypes' does not stand anymore.
Message-ID: <20141113213709.4D8D61C0548@cobra.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc 
Branch: 
Changeset: r74511:c611645f2e73
Date: 2014-11-13 22:36 +0100
http://bitbucket.org/pypy/pypy/changeset/c611645f2e73/

Log:	Don't know whether these modules actually work on sunos5, but the
	excuse 'depends on ctypes' does not stand anymore.

diff --git a/pypy/config/pypyoption.py b/pypy/config/pypyoption.py
--- a/pypy/config/pypyoption.py
+++ b/pypy/config/pypyoption.py
@@ -64,13 +64,9 @@
     default_modules.add("_locale")
 
 if sys.platform == "sunos5":
-    working_modules.remove('mmap')   # depend on ctypes, can't get at c-level 'errono'
-    working_modules.remove('time')   # depend on ctypes, missing tm_zone/tm_gmtoff
-    working_modules.remove('signal') # depend on ctypes, can't get at c-level 'errono'
     working_modules.remove('fcntl')  # LOCK_NB not defined
     working_modules.remove("_minimal_curses")
     working_modules.remove("termios")
-    working_modules.remove("_multiprocessing")   # depends on time
     if "cppyy" in working_modules:
         working_modules.remove("cppyy")  # depends on ctypes
 

From noreply at buildbot.pypy.org  Thu Nov 13 23:26:42 2014
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Thu, 13 Nov 2014 23:26:42 +0100 (CET)
Subject: [pypy-commit] pypy default: (amaury,
 alex) Remove some now-pointless code since rctime is dead.
Message-ID: <20141113222642.983191C0548@cobra.cs.uni-duesseldorf.de>

Author: Alex Gaynor 
Branch: 
Changeset: r74512:a4894448b138
Date: 2014-11-13 14:26 -0800
http://bitbucket.org/pypy/pypy/changeset/a4894448b138/

Log:	(amaury, alex) Remove some now-pointless code since rctime is dead.

diff --git a/pypy/tool/pytest/objspace.py b/pypy/tool/pytest/objspace.py
--- a/pypy/tool/pytest/objspace.py
+++ b/pypy/tool/pytest/objspace.py
@@ -48,9 +48,6 @@
             if key == 'usemodules':
                 if info is not None:
                     for modname in value:
-                        if modname == 'time':
-                            continue   # always either 'time' or 'rctime',
-                                       # and any is fine
                         ok = info.get('objspace.usemodules.%s' % modname,
                                       False)
                         if not ok:

From noreply at buildbot.pypy.org  Fri Nov 14 00:00:10 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Fri, 14 Nov 2014 00:00:10 +0100 (CET)
Subject: [pypy-commit] pypy stmgc-c7: fix
Message-ID: <20141113230010.83CF71D2859@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: stmgc-c7
Changeset: r74513:6371061c0bca
Date: 2014-11-13 23:59 +0100
http://bitbucket.org/pypy/pypy/changeset/6371061c0bca/

Log:	fix

diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py
--- a/rpython/jit/backend/x86/assembler.py
+++ b/rpython/jit/backend/x86/assembler.py
@@ -2329,7 +2329,7 @@
                 initial_value = write_locks_base << 4
                 if isinstance(loc_index, RegLoc):
                     if isinstance(loc_base, RegLoc):
-                        mc.MOV_ri(r11, initial_value)
+                        mc.MOV_ri(r11.value, initial_value)
                         mc.ADD_rr(r11.value, loc_base.value)
                         mc.AND_ri(r11.value, ~15)
                     else:

From noreply at buildbot.pypy.org  Fri Nov 14 00:50:25 2014
From: noreply at buildbot.pypy.org (rlamy)
Date: Fri, 14 Nov 2014 00:50:25 +0100 (CET)
Subject: [pypy-commit] pypy ssa-flow: create remove_trivial_links(): the SSA
 version of join_blocks()
Message-ID: <20141113235025.2360B1C093D@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy 
Branch: ssa-flow
Changeset: r74514:273f7c704efb
Date: 2014-11-13 23:49 +0000
http://bitbucket.org/pypy/pypy/changeset/273f7c704efb/

Log:	create remove_trivial_links(): the SSA version of join_blocks()

diff --git a/rpython/translator/simplify.py b/rpython/translator/simplify.py
--- a/rpython/translator/simplify.py
+++ b/rpython/translator/simplify.py
@@ -246,6 +246,44 @@
             seen.append(case)
         block.recloseblock(*exits)
 
+def remove_trivial_links(graph):
+    """Remove trivial links by merging their source and target blocks
+
+    A link is trivial if it only renames variables, is the single exit of its
+    source and the single parent of its target.
+    """
+    entrymap = mkentrymap(graph)
+    block = graph.startblock
+    seen = set([block])
+    stack = list(block.exits)
+    renaming = {}
+    while stack:
+        link = stack.pop()
+        if link.target in seen:
+            continue
+        source = link.prevblock
+        target = link.target
+        if (source.exitswitch is None and len(entrymap[target]) == 1 and
+                target.exits):  # stop at the returnblock
+            assert len(source.exits) == 1
+            for vprev, vtarg in zip(link.args, target.inputargs):
+                while vprev in renaming:
+                    vprev = renaming[vprev]
+                renaming[vtarg] = vprev
+            target.renamevariables(renaming)
+            source.operations.extend(target.operations)
+            source.exitswitch = newexitswitch = target.exitswitch
+            exits = target.exits
+            source.recloseblock(*exits)
+            if isinstance(newexitswitch, Constant) and newexitswitch != c_last_exception:
+                exits = replace_exitswitch_by_constant(source, newexitswitch)
+            stack.extend(exits)
+        else:
+            target.renamevariables(renaming)
+            seen.add(target)
+            stack.extend(target.exits)
+
+
 def join_blocks(graph):
     """Links can be deleted if they are the single exit of a block and
     the single entry point of the next block.  When this happens, we can
@@ -962,9 +1000,9 @@
 
 all_passes = [
     eliminate_empty_blocks,
+    remove_assertion_errors,
+    remove_trivial_links,
     SSA_to_SSI,
-    remove_assertion_errors,
-    join_blocks,
     coalesce_bool,
     transform_dead_op_vars,
     remove_identical_vars,

From noreply at buildbot.pypy.org  Fri Nov 14 07:14:46 2014
From: noreply at buildbot.pypy.org (fijal)
Date: Fri, 14 Nov 2014 07:14:46 +0100 (CET)
Subject: [pypy-commit] pypy default: fix test_executor
Message-ID: <20141114061446.09A5F1C0548@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: 
Changeset: r74515:1a994c305af0
Date: 2014-11-14 08:14 +0200
http://bitbucket.org/pypy/pypy/changeset/1a994c305af0/

Log:	fix test_executor

diff --git a/rpython/jit/metainterp/test/test_executor.py b/rpython/jit/metainterp/test/test_executor.py
--- a/rpython/jit/metainterp/test/test_executor.py
+++ b/rpython/jit/metainterp/test/test_executor.py
@@ -4,7 +4,7 @@
 from rpython.rtyper.lltypesystem import lltype, llmemory
 from rpython.jit.metainterp.executor import execute
 from rpython.jit.metainterp.executor import execute_varargs, execute_nonspec
-from rpython.jit.metainterp.resoperation import rop, opboolinvers, opboolreflex, opname
+from rpython.jit.metainterp.resoperation import rop, opname, opclasses
 from rpython.jit.metainterp.history import BoxInt, ConstInt
 from rpython.jit.metainterp.history import BoxPtr, ConstPtr
 from rpython.jit.metainterp.history import BoxFloat, ConstFloat
@@ -343,7 +343,11 @@
 
 def test_opboolinvers():
     cpu = FakeCPU()
-    for op1, op2 in opboolinvers.items():
+    for op1 in opclasses:
+        if op1 is None or op1.boolinverse == -1:
+            continue
+        op2 = opclasses[op1.boolinverse].opnum
+        op1 = op1.opnum
         for a in (1,2,3):
             for b in (1,2,3):
                 arg1, arg2 = make_args_for_op(op1, a, b)
@@ -353,7 +357,11 @@
 
 def test_opboolreflex():
     cpu = FakeCPU()
-    for op1, op2 in opboolreflex.items():
+    for op1 in opclasses:
+        if op1 is None or op1.boolreflex == -1:
+            continue
+        op2 = opclasses[op1.boolreflex].opnum
+        op1 = op1.opnum
         for a in (1,2,3):
             for b in (1,2,3):
                 arg1, arg2 = make_args_for_op(op1, a, b)

From noreply at buildbot.pypy.org  Fri Nov 14 11:13:44 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Fri, 14 Nov 2014 11:13:44 +0100 (CET)
Subject: [pypy-commit] pypy stmgc-c7: Update "import_stmgc.py" to show which
 new files need to
Message-ID: <20141114101344.6D70D1C093D@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: stmgc-c7
Changeset: r74516:f47c518719ce
Date: 2014-11-14 11:13 +0100
http://bitbucket.org/pypy/pypy/changeset/f47c518719ce/

Log:	Update "import_stmgc.py" to show which new files need to be tracked
	and which old files need to be deleted. Remove three old files.

diff --git a/rpython/translator/stm/import_stmgc.py b/rpython/translator/stm/import_stmgc.py
--- a/rpython/translator/stm/import_stmgc.py
+++ b/rpython/translator/stm/import_stmgc.py
@@ -7,7 +7,7 @@
 The working copy comes from:  hg clone https://bitbucket.org/pypy/stmgc
 """
 
-import sys, py, subprocess
+import sys, py, subprocess, os
 
 def mangle(lines):
     yield "/* Imported by rpython/translator/stm/import_stmgc.py */\n"
@@ -40,6 +40,12 @@
     #
     stmgc_dest.join('revision').write('%s\n' % rev)
     print rev
+    #
+    print 'The differences between which files are tracked are:'
+    os.system("bash -c 'diff <(cd '%s' && hg status -macn stm/ | sort)"
+              "              <(cd '%s' && hg status -macn stm/ | sort)'"
+        % (stmgc_dest, stmgc_dir))
+    print 'Unless none are listed, use "hg add" or "hg remove".'
 
 if __name__ == '__main__':
     if len(sys.argv) != 2:
diff --git a/rpython/translator/stm/src_stm/stm/prof.h b/rpython/translator/stm/src_stm/stm/prof.h
deleted file mode 100644
--- a/rpython/translator/stm/src_stm/stm/prof.h
+++ /dev/null
@@ -1,3 +0,0 @@
-/* Imported by rpython/translator/stm/import_stmgc.py */
-
-static void forksupport_open_new_profiling_file(void);
diff --git a/rpython/translator/stm/src_stm/stm/timing.c b/rpython/translator/stm/src_stm/stm/timing.c
deleted file mode 100644
--- a/rpython/translator/stm/src_stm/stm/timing.c
+++ /dev/null
@@ -1,92 +0,0 @@
-/* Imported by rpython/translator/stm/import_stmgc.py */
-#ifndef _STM_CORE_H_
-# error "must be compiled via stmgc.c"
-#endif
-
-
-static inline void add_timing(stm_thread_local_t *tl, enum stm_time_e category,
-                              double elapsed)
-{
-    tl->timing[category] += elapsed;
-    tl->events[category] += 1;
-}
-
-#define TIMING_CHANGE(tl, newstate)                     \
-    double curtime = get_stm_time();                    \
-    double elasped = curtime - tl->_timing_cur_start;   \
-    enum stm_time_e oldstate = tl->_timing_cur_state;   \
-    add_timing(tl, oldstate, elasped);                  \
-    tl->_timing_cur_state = newstate;                   \
-    tl->_timing_cur_start = curtime
-
-static enum stm_time_e change_timing_state(enum stm_time_e newstate)
-{
-    stm_thread_local_t *tl = STM_SEGMENT->running_thread;
-    TIMING_CHANGE(tl, newstate);
-    return oldstate;
-}
-
-static double change_timing_state_tl(stm_thread_local_t *tl,
-                                     enum stm_time_e newstate)
-{
-    TIMING_CHANGE(tl, newstate);
-    return elasped;
-}
-
-static void timing_end_transaction(enum stm_time_e attribute_to)
-{
-    stm_thread_local_t *tl = STM_SEGMENT->running_thread;
-    TIMING_CHANGE(tl, STM_TIME_OUTSIDE_TRANSACTION);
-    double time_this_transaction = tl->timing[STM_TIME_RUN_CURRENT];
-    add_timing(tl, attribute_to, time_this_transaction);
-    tl->timing[STM_TIME_RUN_CURRENT] = 0.0f;
-
-    if (attribute_to != STM_TIME_RUN_COMMITTED) {
-        struct stm_priv_segment_info_s *pseg =
-            get_priv_segment(STM_SEGMENT->segment_num);
-        marker_copy(tl, pseg, attribute_to, time_this_transaction);
-    }
-}
-
-static const char *timer_names[] = {
-    "outside transaction",
-    "run current",
-    "run committed",
-    "run aborted write write",
-    "run aborted write read",
-    "run aborted inevitable",
-    "run aborted other",
-    "wait free segment",
-    "wait write read",
-    "wait inevitable",
-    "wait other",
-    "sync commit soon",
-    "bookkeeping",
-    "minor gc",
-    "major gc",
-    "sync pause",
-};
-
-void stm_flush_timing(stm_thread_local_t *tl, int verbose)
-{
-    enum stm_time_e category = tl->_timing_cur_state;
-    uint64_t oldevents = tl->events[category];
-    TIMING_CHANGE(tl, category);
-    tl->events[category] = oldevents;
-
-    assert((sizeof(timer_names) / sizeof(timer_names[0])) == _STM_TIME_N);
-    if (verbose > 0) {
-        int i;
-        s_mutex_lock();
-        fprintf(stderr, "thread %p:\n", tl);
-        for (i = 0; i < _STM_TIME_N; i++) {
-            fprintf(stderr, "    %-24s %9u %8.3f s\n",
-                    timer_names[i], tl->events[i], (double)tl->timing[i]);
-        }
-        fprintf(stderr, "    %-24s %6s %11.6f s\n",
-                "longest recorded marker", "", tl->longest_marker_time);
-        fprintf(stderr, "    \"%.*s\"\n",
-                (int)_STM_MARKER_LEN, tl->longest_marker_self);
-        s_mutex_unlock();
-    }
-}
diff --git a/rpython/translator/stm/src_stm/stm/timing.h b/rpython/translator/stm/src_stm/stm/timing.h
deleted file mode 100644
--- a/rpython/translator/stm/src_stm/stm/timing.h
+++ /dev/null
@@ -1,15 +0,0 @@
-/* Imported by rpython/translator/stm/import_stmgc.py */
-#include 
-
-static inline double get_stm_time(void)
-{
-    struct timespec tp;
-    clock_gettime(CLOCK_MONOTONIC, &tp);
-    return tp.tv_sec + tp.tv_nsec * 0.000000001;
-}
-
-static enum stm_time_e change_timing_state(enum stm_time_e newstate);
-static double change_timing_state_tl(stm_thread_local_t *tl,
-                                     enum stm_time_e newstate);
-
-static void timing_end_transaction(enum stm_time_e attribute_to);

From noreply at buildbot.pypy.org  Fri Nov 14 11:32:44 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Fri, 14 Nov 2014 11:32:44 +0100 (CET)
Subject: [pypy-commit] pypy stmgc-c7: Add the missing part about recording
	pauses.
Message-ID: <20141114103244.225591D2CED@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: stmgc-c7
Changeset: r74517:31351821e9e1
Date: 2014-11-14 11:32 +0100
http://bitbucket.org/pypy/pypy/changeset/31351821e9e1/

Log:	Add the missing part about recording pauses.

diff --git a/pypy/stm/print_stm_log.py b/pypy/stm/print_stm_log.py
--- a/pypy/stm/print_stm_log.py
+++ b/pypy/stm/print_stm_log.py
@@ -85,16 +85,35 @@
     def transaction_start(self, entry):
         self._start = entry
         self._conflict = None
+        self._pause = None
+        self._paused_time = 0.0
 
     def transaction_stop(self, entry):
         transaction_time = entry.timestamp - self._start.timestamp
+        transaction_time -= self._paused_time
+        assert transaction_time >= 0.0
         self.cpu_time += transaction_time
         self._start = None
+        self._pause = None
         if self._conflict and entry.event == STM_TRANSACTION_ABORT:
             c = self._conflict[1]
             c.aborted_time += transaction_time
             self._conflict = None
 
+    def transaction_pause(self, entry):
+        self._pause = entry
+
+    def transaction_unpause(self, entry):
+        if self._pause is None:
+            return
+        pause_time = entry.timestamp - self._pause.timestamp
+        self._paused_time += pause_time
+        self._pause = None
+        if self._conflict and self._conflict[0] == "local":
+            c = self._conflict[1]
+            c.paused_time += pause_time
+            self._conflict = None
+
     def in_transaction(self):
         return self._start is not None
 
@@ -165,7 +184,14 @@
                 t2 = threads.get(entry.otherthreadnum)
                 if t2 is not None and t2.in_transaction():
                     t2._conflict = ("remote", c, entry)
-        #elif entry.event == ...STM_WAIT...
+        elif entry.event in (STM_WAIT_SYNC_PAUSE, STM_WAIT_CONTENTION):
+            t = threads.get(entry.threadnum)
+            if t is not None and t.in_transaction():
+                t.transaction_pause(entry)
+        elif entry.event == STM_WAIT_DONE:
+            t = threads.get(entry.threadnum)
+            if t is not None and t.in_transaction():
+                t.transaction_unpause(entry)
     #
     total_cpu_time = sum([t.cpu_time for t in threads.values()])
     print 'Total CPU time in STM mode:  %.3fs (%s)' % (

From noreply at buildbot.pypy.org  Fri Nov 14 11:46:44 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Fri, 14 Nov 2014 11:46:44 +0100 (CET)
Subject: [pypy-commit] pypy stmgc-c7: update from the changes done in the
	list of operations
Message-ID: <20141114104644.EE93D1C345F@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: stmgc-c7
Changeset: r74518:4491d856926c
Date: 2014-11-14 11:46 +0100
http://bitbucket.org/pypy/pypy/changeset/4491d856926c/

Log:	update from the changes done in the list of operations

diff --git a/rpython/translator/stm/inevitable.py b/rpython/translator/stm/inevitable.py
--- a/rpython/translator/stm/inevitable.py
+++ b/rpython/translator/stm/inevitable.py
@@ -31,11 +31,14 @@
 GETTERS = set(['getfield', 'getarrayitem', 'getinteriorfield', 'raw_load'])
 SETTERS = set(['setfield', 'setarrayitem', 'setinteriorfield', 'raw_store'])
 MALLOCS = set(['malloc', 'malloc_varsize',
-               'malloc_nonmovable', 'malloc_nonmovable_varsize',
                'raw_malloc',
+               'do_malloc_fixedsize', 'do_malloc_varsize',
                'do_malloc_fixedsize_clear', 'do_malloc_varsize_clear'])
 FREES   = set(['free', 'raw_free'])
 
+for opname in ALWAYS_ALLOW_OPERATIONS | GETTERS | SETTERS | MALLOCS | FREES:
+    getattr(lloperation.llop, opname)   # the opname must exist!
+
 # ____________________________________________________________
 
 def should_turn_inevitable_getter_setter(op):

From noreply at buildbot.pypy.org  Fri Nov 14 13:29:05 2014
From: noreply at buildbot.pypy.org (Raemi)
Date: Fri, 14 Nov 2014 13:29:05 +0100 (CET)
Subject: [pypy-commit] pypy stmgc-c7: add useful time line to print_stm_log
 in order to see if the time was lost in
Message-ID: <20141114122905.60DF01C345F@cobra.cs.uni-duesseldorf.de>

Author: Remi Meier 
Branch: stmgc-c7
Changeset: r74519:91bce27dd903
Date: 2014-11-14 13:29 +0100
http://bitbucket.org/pypy/pypy/changeset/91bce27dd903/

Log:	add useful time line to print_stm_log in order to see if the time
	was lost in the initialization or shutdown phases instead of the
	actual runtime.

diff --git a/pypy/stm/print_stm_log.py b/pypy/stm/print_stm_log.py
--- a/pypy/stm/print_stm_log.py
+++ b/pypy/stm/print_stm_log.py
@@ -126,6 +126,7 @@
         self.aborted_time = 0.0
         self.paused_time = 0.0
         self.num_events = 0
+        self.timestamps = []
 
     def sortkey(self):
         return self.aborted_time + self.paused_time
@@ -149,7 +150,8 @@
     return r + '%'
 
 def dump(logentries):
-    total_time = logentries[-1].timestamp - logentries[0].timestamp
+    start_time, stop_time = logentries[0].timestamp, logentries[-1].timestamp
+    total_time = stop_time - start_time
     print 'Total real time:             %.3fs' % (total_time,)
     #
     threads = {}
@@ -173,6 +175,7 @@
             if c is None:
                 c = conflicts[summary] = ConflictSummary(*summary)
             c.num_events += 1
+            c.timestamps.append(entry.timestamp)
             t = threads.get(entry.threadnum)
             if t is not None and t.in_transaction():
                 t._conflict = ("local", c, entry)
@@ -193,18 +196,25 @@
             if t is not None and t.in_transaction():
                 t.transaction_unpause(entry)
     #
-    total_cpu_time = sum([t.cpu_time for t in threads.values()])
+    total_cpu_time = sum([v.cpu_time for v in threads.values()])
     print 'Total CPU time in STM mode:  %.3fs (%s)' % (
         total_cpu_time, percent(total_cpu_time, total_time))
     print
     #
     values = sorted(conflicts.values(), key=ConflictSummary.sortkey)
     for c in values[-1:-15:-1]:
+        intervals = 48
+        timeline = [0] * intervals
+        for t in c.timestamps:
+            idx = int((t - start_time) / total_time * intervals)
+            timeline[idx] += 1
+
         print '%.3fs lost in aborts, %.3fs paused (%dx %s)' % (
             c.aborted_time, c.paused_time, c.num_events, event_name[c.event])
         print_marker(c.marker1)
         if c.marker2:
             print_marker(c.marker2)
+        print "time line:", "".join(['x' if i else '.' for i in timeline])
         print
 
 

From noreply at buildbot.pypy.org  Fri Nov 14 15:13:45 2014
From: noreply at buildbot.pypy.org (bivab)
Date: Fri, 14 Nov 2014 15:13:45 +0100 (CET)
Subject: [pypy-commit] buildbot default: Copy libpypy-c.so to pypy checkout
 for translated test build
Message-ID: <20141114141345.820541C093D@cobra.cs.uni-duesseldorf.de>

Author: David Schneider 
Branch: 
Changeset: r922:b699815f9193
Date: 2014-11-14 14:53 +0100
http://bitbucket.org/pypy/buildbot/changeset/b699815f9193/

Log:	Copy libpypy-c.so to pypy checkout for translated test build

diff --git a/bot2/pypybuildbot/builds.py b/bot2/pypybuildbot/builds.py
--- a/bot2/pypybuildbot/builds.py
+++ b/bot2/pypybuildbot/builds.py
@@ -564,6 +564,13 @@
             command=command,
             haltOnFailure=True,
             workdir='.'))
+        # copy libpypy-c.so to the expected location within the pypy source checkout, if available
+        command = 'if [ -e pypy-c/bin/libpypy-c.so ]; then cp -v pypy-c/bin/libpypy-c.so build/pypy/goal/; fi;'
+        self.addStep(ShellCmd(
+            description="move libpypy-c.so",
+            command=command,
+            haltOnFailure=True,
+            workdir='.'))
         # copy generated and copied header files to build/include
         self.addStep(ShellCmd(
             description="move header files",

From noreply at buildbot.pypy.org  Fri Nov 14 17:16:02 2014
From: noreply at buildbot.pypy.org (fijal)
Date: Fri, 14 Nov 2014 17:16:02 +0100 (CET)
Subject: [pypy-commit] pypy optresult: hack enough to make the first test
 from optimizebasic pass
Message-ID: <20141114161602.3031C1C345F@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: optresult
Changeset: r74520:6dfe20d64577
Date: 2014-11-14 10:44 +0200
http://bitbucket.org/pypy/pypy/changeset/6dfe20d64577/

Log:	hack enough to make the first test from optimizebasic pass

diff --git a/rpython/jit/metainterp/optimizeopt/intbounds.py b/rpython/jit/metainterp/optimizeopt/intbounds.py
--- a/rpython/jit/metainterp/optimizeopt/intbounds.py
+++ b/rpython/jit/metainterp/optimizeopt/intbounds.py
@@ -6,7 +6,7 @@
 from rpython.jit.metainterp.optimizeopt.optimizer import (Optimization, CONST_1,
     CONST_0, MODE_ARRAY, MODE_STR, MODE_UNICODE)
 from rpython.jit.metainterp.optimizeopt.util import make_dispatcher_method
-from rpython.jit.metainterp.resoperation import rop
+from rpython.jit.metainterp.resoperation import rop, AbstractResOp
 from rpython.jit.backend.llsupport import symbolic
 
 
@@ -58,11 +58,8 @@
         if b.has_lower and b.has_upper and b.lower == b.upper:
             v.make_constant(ConstInt(b.lower))
 
-        try:
-            op = self.optimizer.producer[box]
-        except KeyError:
-            return
-        dispatch_bounds_ops(self, op)
+        if isinstance(box, AbstractResOp):
+            dispatch_bounds_ops(self, box)
 
     def optimize_GUARD_TRUE(self, op):
         self.emit_operation(op)
@@ -83,7 +80,7 @@
         self.emit_operation(op)
         if v1.intbound.known_ge(IntBound(0, 0)) and \
            v2.intbound.known_ge(IntBound(0, 0)):
-            r = self.getvalue(op.result)
+            r = self.getvalue(op)
             mostsignificant = v1.intbound.upper | v2.intbound.upper
             r.intbound.intersect(IntBound(0, next_pow2_m1(mostsignificant)))
 
@@ -95,7 +92,7 @@
         v2 = self.getvalue(op.getarg(1))
         self.emit_operation(op)
 
-        r = self.getvalue(op.result)
+        r = self.getvalue(op)
         if v2.is_constant():
             val = v2.box.getint()
             if val >= 0:
@@ -113,7 +110,7 @@
         v1 = self.getvalue(op.getarg(0))
         v2 = self.getvalue(op.getarg(1))
         self.emit_operation(op)
-        r = self.getvalue(op.result)
+        r = self.getvalue(op)
         b = v1.intbound.sub_bound(v2.intbound)
         if b.bounded():
             r.intbound.intersect(b)
@@ -122,7 +119,7 @@
         v1 = self.getvalue(op.getarg(0))
         v2 = self.getvalue(op.getarg(1))
         self.emit_operation(op)
-        r = self.getvalue(op.result)
+        r = self.getvalue(op)
         b = v1.intbound.add_bound(v2.intbound)
         if b.bounded():
             r.intbound.intersect(b)
@@ -131,7 +128,7 @@
         v1 = self.getvalue(op.getarg(0))
         v2 = self.getvalue(op.getarg(1))
         self.emit_operation(op)
-        r = self.getvalue(op.result)
+        r = self.getvalue(op)
         b = v1.intbound.mul_bound(v2.intbound)
         if b.bounded():
             r.intbound.intersect(b)
@@ -140,7 +137,7 @@
         v1 = self.getvalue(op.getarg(0))
         v2 = self.getvalue(op.getarg(1))
         self.emit_operation(op)
-        r = self.getvalue(op.result)
+        r = self.getvalue(op)
         r.intbound.intersect(v1.intbound.div_bound(v2.intbound))
 
     def optimize_INT_MOD(self, op):
@@ -158,7 +155,7 @@
         self.emit_operation(op)
         if v2.is_constant():
             val = v2.box.getint()
-            r = self.getvalue(op.result)
+            r = self.getvalue(op)
             if val < 0:
                 if val == -sys.maxint-1:
                     return     # give up
@@ -173,7 +170,7 @@
         v1 = self.getvalue(op.getarg(0))
         v2 = self.getvalue(op.getarg(1))
         self.emit_operation(op)
-        r = self.getvalue(op.result)
+        r = self.getvalue(op)
         b = v1.intbound.lshift_bound(v2.intbound)
         r.intbound.intersect(b)
         # intbound.lshift_bound checks for an overflow and if the
@@ -192,7 +189,7 @@
             self.make_constant_int(op.result, b.lower)
         else:
             self.emit_operation(op)
-            r = self.getvalue(op.result)
+            r = self.getvalue(op)
             r.intbound.intersect(b)
 
     def optimize_GUARD_NO_OVERFLOW(self, op):
@@ -245,7 +242,7 @@
             # optimize_GUARD_OVERFLOW, then InvalidLoop.
             op = op.copy_and_change(rop.INT_ADD)
         self.emit_operation(op) # emit the op
-        r = self.getvalue(op.result)
+        r = self.getvalue(op)
         r.intbound.intersect(resbound)
 
     def optimize_INT_SUB_OVF(self, op):
@@ -258,7 +255,7 @@
         if resbound.bounded():
             op = op.copy_and_change(rop.INT_SUB)
         self.emit_operation(op) # emit the op
-        r = self.getvalue(op.result)
+        r = self.getvalue(op)
         r.intbound.intersect(resbound)
 
     def optimize_INT_MUL_OVF(self, op):
@@ -268,7 +265,7 @@
         if resbound.bounded():
             op = op.copy_and_change(rop.INT_MUL)
         self.emit_operation(op)
-        r = self.getvalue(op.result)
+        r = self.getvalue(op)
         r.intbound.intersect(resbound)
 
     def optimize_INT_LT(self, op):
@@ -345,7 +342,7 @@
     def optimize_ARRAYLEN_GC(self, op):
         self.emit_operation(op)
         array = self.getvalue(op.getarg(0))
-        result = self.getvalue(op.result)
+        result = self.getvalue(op)
         array.make_len_gt(MODE_ARRAY, op.getdescr(), -1)
         array.lenbound.bound.intersect(result.intbound)
         result.intbound = array.lenbound.bound
@@ -353,7 +350,7 @@
     def optimize_STRLEN(self, op):
         self.emit_operation(op)
         array = self.getvalue(op.getarg(0))
-        result = self.getvalue(op.result)
+        result = self.getvalue(op)
         array.make_len_gt(MODE_STR, op.getdescr(), -1)
         array.lenbound.bound.intersect(result.intbound)
         result.intbound = array.lenbound.bound
@@ -361,14 +358,14 @@
     def optimize_UNICODELEN(self, op):
         self.emit_operation(op)
         array = self.getvalue(op.getarg(0))
-        result = self.getvalue(op.result)
+        result = self.getvalue(op)
         array.make_len_gt(MODE_UNICODE, op.getdescr(), -1)
         array.lenbound.bound.intersect(result.intbound)
         result.intbound = array.lenbound.bound
 
     def optimize_STRGETITEM(self, op):
         self.emit_operation(op)
-        v1 = self.getvalue(op.result)
+        v1 = self.getvalue(op)
         v1.intbound.make_ge(IntLowerBound(0))
         v1.intbound.make_lt(IntUpperBound(256))
 
@@ -376,7 +373,7 @@
         self.emit_operation(op)
         descr = op.getdescr()
         if descr.is_integer_bounded():
-            v1 = self.getvalue(op.result)
+            v1 = self.getvalue(op)
             v1.intbound.make_ge(IntLowerBound(descr.get_integer_min()))
             v1.intbound.make_le(IntUpperBound(descr.get_integer_max()))
 
@@ -393,7 +390,7 @@
         self.emit_operation(op)
         descr = op.getdescr()
         if descr and descr.is_item_integer_bounded():
-            v1 = self.getvalue(op.result)
+            v1 = self.getvalue(op)
             v1.intbound.make_ge(IntLowerBound(descr.get_item_integer_min()))
             v1.intbound.make_le(IntUpperBound(descr.get_item_integer_max()))
 
@@ -404,7 +401,7 @@
 
     def optimize_UNICODEGETITEM(self, op):
         self.emit_operation(op)
-        v1 = self.getvalue(op.result)
+        v1 = self.getvalue(op)
         v1.intbound.make_ge(IntLowerBound(0))
 
     def make_int_lt(self, box1, box2):
@@ -430,7 +427,7 @@
         self.make_int_le(box2, box1)
 
     def propagate_bounds_INT_LT(self, op):
-        r = self.getvalue(op.result)
+        r = self.getvalue(op)
         if r.is_constant():
             if r.box.same_constant(CONST_1):
                 self.make_int_lt(op.getarg(0), op.getarg(1))
@@ -438,7 +435,7 @@
                 self.make_int_ge(op.getarg(0), op.getarg(1))
 
     def propagate_bounds_INT_GT(self, op):
-        r = self.getvalue(op.result)
+        r = self.getvalue(op)
         if r.is_constant():
             if r.box.same_constant(CONST_1):
                 self.make_int_gt(op.getarg(0), op.getarg(1))
@@ -446,7 +443,7 @@
                 self.make_int_le(op.getarg(0), op.getarg(1))
 
     def propagate_bounds_INT_LE(self, op):
-        r = self.getvalue(op.result)
+        r = self.getvalue(op)
         if r.is_constant():
             if r.box.same_constant(CONST_1):
                 self.make_int_le(op.getarg(0), op.getarg(1))
@@ -454,7 +451,7 @@
                 self.make_int_gt(op.getarg(0), op.getarg(1))
 
     def propagate_bounds_INT_GE(self, op):
-        r = self.getvalue(op.result)
+        r = self.getvalue(op)
         if r.is_constant():
             if r.box.same_constant(CONST_1):
                 self.make_int_ge(op.getarg(0), op.getarg(1))
@@ -462,7 +459,7 @@
                 self.make_int_lt(op.getarg(0), op.getarg(1))
 
     def propagate_bounds_INT_EQ(self, op):
-        r = self.getvalue(op.result)
+        r = self.getvalue(op)
         if r.is_constant():
             if r.box.same_constant(CONST_1):
                 v1 = self.getvalue(op.getarg(0))
@@ -473,7 +470,7 @@
                     self.propagate_bounds_backward(op.getarg(1))
 
     def propagate_bounds_INT_NE(self, op):
-        r = self.getvalue(op.result)
+        r = self.getvalue(op)
         if r.is_constant():
             if r.box.same_constant(CONST_0):
                 v1 = self.getvalue(op.getarg(0))
@@ -484,7 +481,7 @@
                     self.propagate_bounds_backward(op.getarg(1))
 
     def propagate_bounds_INT_IS_TRUE(self, op):
-        r = self.getvalue(op.result)
+        r = self.getvalue(op)
         if r.is_constant():
             if r.box.same_constant(CONST_1):
                 v1 = self.getvalue(op.getarg(0))
@@ -493,7 +490,7 @@
                     self.propagate_bounds_backward(op.getarg(0))
 
     def propagate_bounds_INT_IS_ZERO(self, op):
-        r = self.getvalue(op.result)
+        r = self.getvalue(op)
         if r.is_constant():
             if r.box.same_constant(CONST_1):
                 v1 = self.getvalue(op.getarg(0))
@@ -507,7 +504,7 @@
     def propagate_bounds_INT_ADD(self, op):
         v1 = self.getvalue(op.getarg(0))
         v2 = self.getvalue(op.getarg(1))
-        r = self.getvalue(op.result)
+        r = self.getvalue(op)
         b = r.intbound.sub_bound(v2.intbound)
         if v1.intbound.intersect(b):
             self.propagate_bounds_backward(op.getarg(0))
@@ -518,7 +515,7 @@
     def propagate_bounds_INT_SUB(self, op):
         v1 = self.getvalue(op.getarg(0))
         v2 = self.getvalue(op.getarg(1))
-        r = self.getvalue(op.result)
+        r = self.getvalue(op)
         b = r.intbound.add_bound(v2.intbound)
         if v1.intbound.intersect(b):
             self.propagate_bounds_backward(op.getarg(0))
@@ -529,7 +526,7 @@
     def propagate_bounds_INT_MUL(self, op):
         v1 = self.getvalue(op.getarg(0))
         v2 = self.getvalue(op.getarg(1))
-        r = self.getvalue(op.result)
+        r = self.getvalue(op)
         b = r.intbound.div_bound(v2.intbound)
         if v1.intbound.intersect(b):
             self.propagate_bounds_backward(op.getarg(0))
@@ -540,7 +537,7 @@
     def propagate_bounds_INT_LSHIFT(self, op):
         v1 = self.getvalue(op.getarg(0))
         v2 = self.getvalue(op.getarg(1))
-        r = self.getvalue(op.result)
+        r = self.getvalue(op)
         b = r.intbound.rshift_bound(v2.intbound)
         if v1.intbound.intersect(b):
             self.propagate_bounds_backward(op.getarg(0))
diff --git a/rpython/jit/metainterp/optimizeopt/optimizer.py b/rpython/jit/metainterp/optimizeopt/optimizer.py
--- a/rpython/jit/metainterp/optimizeopt/optimizer.py
+++ b/rpython/jit/metainterp/optimizeopt/optimizer.py
@@ -273,7 +273,7 @@
 CVAL_ZERO    = ConstantValue(CONST_0)
 CVAL_ZERO_FLOAT = ConstantValue(Const._new(0.0))
 llhelper.CVAL_NULLREF = ConstantValue(llhelper.CONST_NULL)
-REMOVED = AbstractResOp(None)
+REMOVED = AbstractResOp()
 
 
 class Optimization(object):
@@ -377,14 +377,12 @@
         self.interned_ints = {}
         self.resumedata_memo = resume.ResumeDataLoopMemo(metainterp_sd)
         self.bool_boxes = {}
-        self.producer = {}
         self.pendingfields = None # set temporarily to a list, normally by
                                   # heap.py, as we're about to generate a guard
         self.quasi_immutable_deps = None
         self.opaque_pointers = {}
         self.replaces_guard = {}
         self._newoperations = []
-        self.seen_results = {}
         self.optimizer = self
         self.optpure = None
         self.optearlyforce = None
@@ -479,7 +477,6 @@
 
     def clear_newoperations(self):
         self._newoperations = []
-        self.seen_results = {}
 
     def make_equal_to(self, box, value, replace=False):
         assert isinstance(value, OptValue)
@@ -541,7 +538,6 @@
         self.first_optimization.propagate_forward(op)
 
     def propagate_forward(self, op):
-        self.producer[op.result] = op
         dispatch_opt(self, op)
 
     def emit_operation(self, op):
@@ -574,10 +570,6 @@
                 op = self.store_final_boxes_in_guard(op, pendingfields)
         elif op.can_raise():
             self.exception_might_have_happened = True
-        if op.result:
-            if op.result in self.seen_results:
-                raise ValueError, "invalid optimization"
-            self.seen_results[op.result] = None
         self._newoperations.append(op)
 
     def replace_op(self, old_op, new_op):
diff --git a/rpython/jit/metainterp/optimizeopt/pure.py b/rpython/jit/metainterp/optimizeopt/pure.py
--- a/rpython/jit/metainterp/optimizeopt/pure.py
+++ b/rpython/jit/metainterp/optimizeopt/pure.py
@@ -38,7 +38,8 @@
                 return
 
             # did we do the exact same operation already?
-            args = self.optimizer.make_args_key(op)
+            args = self.optimizer.make_args_key(op.getopnum(),
+                                                op.getarglist(), op.getdescr())
             oldop = self.pure_operations.get(args, None)
             if oldop is not None and oldop.getdescr() is op.getdescr():
                 assert oldop.getopnum() == op.getopnum()
@@ -102,8 +103,7 @@
         self.optimizer.optpure = self
 
     def pure(self, opnum, args, result):
-        op = ResOperation(opnum, args, result)
-        key = self.optimizer.make_args_key(op)
+        key = self.optimizer.make_args_key(opnum, args, None)
         if key not in self.pure_operations:
             self.pure_operations[key] = result
 
diff --git a/rpython/jit/metainterp/optimizeopt/rewrite.py b/rpython/jit/metainterp/optimizeopt/rewrite.py
--- a/rpython/jit/metainterp/optimizeopt/rewrite.py
+++ b/rpython/jit/metainterp/optimizeopt/rewrite.py
@@ -119,8 +119,8 @@
         else:
             self.emit_operation(op)
             # Synthesize the reverse ops for optimize_default to reuse
-            self.pure(rop.INT_ADD, [op.result, op.getarg(1)], op.getarg(0))
-            self.pure(rop.INT_SUB, [op.getarg(0), op.result], op.getarg(1))
+            self.pure(rop.INT_ADD, [op, op.getarg(1)], op.getarg(0))
+            self.pure(rop.INT_SUB, [op.getarg(0), op], op.getarg(1))
 
     def optimize_INT_ADD(self, op):
         v1 = self.getvalue(op.getarg(0))
@@ -133,10 +133,10 @@
             self.make_equal_to(op.result, v1)
         else:
             self.emit_operation(op)
-            self.pure(rop.INT_ADD, [op.getarg(1), op.getarg(0)], op.result)
+            self.pure(rop.INT_ADD, [op.getarg(1), op.getarg(0)], op)
             # Synthesize the reverse op for optimize_default to reuse
-            self.pure(rop.INT_SUB, [op.result, op.getarg(1)], op.getarg(0))
-            self.pure(rop.INT_SUB, [op.result, op.getarg(0)], op.getarg(1))
+            self.pure(rop.INT_SUB, [op, op.getarg(1)], op.getarg(0))
+            self.pure(rop.INT_SUB, [op, op.getarg(0)], op.getarg(1))
 
     def optimize_INT_MUL(self, op):
         v1 = self.getvalue(op.getarg(0))
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py
@@ -105,7 +105,7 @@
             loop.operations[-1].setdescr(token)
         expected = convert_old_style_to_targets(self.parse(optops), jump=True)
         self._do_optimize_loop(loop, call_pure_results)
-        print '\n'.join([str(o) for o in loop.operations])
+        #print '\n'.join([str(o) for o in loop.operations])
         self.assert_equal(loop, expected)
 
     def setup_method(self, meth=None):
diff --git a/rpython/jit/metainterp/optimizeopt/util.py b/rpython/jit/metainterp/optimizeopt/util.py
--- a/rpython/jit/metainterp/optimizeopt/util.py
+++ b/rpython/jit/metainterp/optimizeopt/util.py
@@ -157,32 +157,11 @@
             x = op1.getarg(i)
             y = op2.getarg(i)
             assert x.same_box(remap.get(y, y))
-        if op2.result in remap:
-            if op2.result is None:
-                assert op1.result == remap[op2.result]
-            else:
-                assert op1.result.same_box(remap[op2.result])
+        if op2 in remap:
+            assert op1.same_box(remap[op2])
         else:
-            remap[op2.result] = op1.result
-        if op1.getopnum() not in (rop.JUMP, rop.LABEL):      # xxx obscure
+            remap[op2] = op1
+        if op1.getopnum() not in (rop.JUMP, rop.LABEL) and not op1.is_guard():      # xxx obscure
             assert op1.getdescr() == op2.getdescr()
-        if op1.getfailargs() or op2.getfailargs():
-            assert len(op1.getfailargs()) == len(op2.getfailargs())
-            if strict_fail_args:
-                for x, y in zip(op1.getfailargs(), op2.getfailargs()):
-                    if x is None:
-                        assert remap.get(y, y) is None
-                    else:
-                        assert x.same_box(remap.get(y, y))
-            else:
-                fail_args1 = set(op1.getfailargs())
-                fail_args2 = set([remap.get(y, y) for y in op2.getfailargs()])
-                for x in fail_args1:
-                    for y in fail_args2:
-                        if x.same_box(y):
-                            fail_args2.remove(y)
-                            break
-                    else:
-                        assert False
     assert len(oplist1) == len(oplist2)
     return True
diff --git a/rpython/jit/metainterp/resoperation.py b/rpython/jit/metainterp/resoperation.py
--- a/rpython/jit/metainterp/resoperation.py
+++ b/rpython/jit/metainterp/resoperation.py
@@ -5,10 +5,13 @@
     def _get_hash_(self):
         return compute_identity_hash(self)
 
+    def same_box(self, other):
+        return self is other
+
 @specialize.argtype(2)
 def ResOperation(opnum, args, result, descr=None):
     cls = opclasses[opnum]
-    op = cls(result)
+    op = cls()
     op.initarglist(args)
     if descr is not None:
         assert isinstance(op, ResOpWithDescr)
@@ -44,9 +47,6 @@
 
     _attrs_ = ()
 
-    def __init__(self, result):
-        self.result = result
-
     def getopnum(self):
         return self.opnum
 
@@ -115,14 +115,9 @@
             op.pc = self.pc
         return op
 
-    def __repr__(self):
-        try:
-            return self.repr()
-        except NotImplementedError:
-            return object.__repr__(self)
-
-    def repr(self, graytext=False):
+    def repr(self, memo, graytext=False):
         # RPython-friendly version
+        XXX
         if self.result is not None:
             sres = '%s = ' % (self.result,)
         else:
diff --git a/rpython/jit/tool/oparser.py b/rpython/jit/tool/oparser.py
--- a/rpython/jit/tool/oparser.py
+++ b/rpython/jit/tool/oparser.py
@@ -23,7 +23,7 @@
         return 'escape'
 
     def clone(self):
-        op = ESCAPE_OP(0)
+        op = ESCAPE_OP()
         op.initarglist(self.getarglist()[:])
         return op
 
@@ -38,7 +38,7 @@
         return 'force_spill'
 
     def clone(self):
-        op = FORCE_SPILL(0)
+        op = FORCE_SPILL()
         op.initarglist(self.getarglist()[:])
         return op
 
@@ -265,12 +265,12 @@
 
     def create_op(self, opnum, args, descr):
         if opnum == ESCAPE_OP.OPNUM:
-            op = ESCAPE_OP(0)
+            op = ESCAPE_OP()
             op.initarglist(args)
             assert descr is None
             return op
         if opnum == FORCE_SPILL.OPNUM:
-            op = FORCE_SPILL(0)
+            op = FORCE_SPILL()
             op.initarglist(args)
             assert descr is None
             return op

From noreply at buildbot.pypy.org  Fri Nov 14 17:16:03 2014
From: noreply at buildbot.pypy.org (fijal)
Date: Fri, 14 Nov 2014 17:16:03 +0100 (CET)
Subject: [pypy-commit] pypy optresult:  hack hack hack
Message-ID: <20141114161603.83CF91C345F@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: optresult
Changeset: r74521:38b07f93b97a
Date: 2014-11-14 18:15 +0200
http://bitbucket.org/pypy/pypy/changeset/38b07f93b97a/

Log:	hack hack hack

diff --git a/rpython/jit/metainterp/executor.py b/rpython/jit/metainterp/executor.py
--- a/rpython/jit/metainterp/executor.py
+++ b/rpython/jit/metainterp/executor.py
@@ -4,8 +4,10 @@
 from rpython.rtyper.lltypesystem import lltype, rstr
 from rpython.rlib.rarithmetic import ovfcheck, r_longlong, is_valid_int
 from rpython.rlib.unroll import unrolling_iterable
-from rpython.jit.metainterp.history import BoxInt, BoxPtr, BoxFloat, check_descr
+from rpython.rlib.objectmodel import specialize
+from rpython.jit.metainterp.history import check_descr
 from rpython.jit.metainterp.history import INT, REF, FLOAT, VOID, AbstractDescr
+from rpython.jit.metainterp.history import ConstInt, ConstFloat, ConstPtr
 from rpython.jit.metainterp import resoperation
 from rpython.jit.metainterp.resoperation import rop
 from rpython.jit.metainterp.blackhole import BlackholeInterpreter, NULL
@@ -14,6 +16,7 @@
 # ____________________________________________________________
 
 def do_call(cpu, metainterp, argboxes, descr):
+    xxx
     assert metainterp is not None
     # count the number of arguments of the different types
     count_i = count_r = count_f = 0
@@ -141,6 +144,7 @@
         cpu.bh_setarrayitem_raw_i(array, index, itembox.getint(), arraydescr)
 
 def do_getinteriorfield_gc(cpu, _, arraybox, indexbox, descr):
+    xxxx
     array = arraybox.getref_base()
     index = indexbox.getint()
     if descr.is_pointer_field():
@@ -175,6 +179,7 @@
     return cpu.bh_getfield_gc_f(struct, fielddescr)
 
 def do_getfield_raw(cpu, _, structbox, fielddescr):
+    xxxx
     check_descr(fielddescr)
     struct = structbox.getint()
     if fielddescr.is_pointer_field():
@@ -212,6 +217,7 @@
         cpu.bh_raw_store_i(addr, offset, valuebox.getint(), arraydescr)
 
 def do_raw_load(cpu, _, addrbox, offsetbox, arraydescr):
+    xxx
     addr = addrbox.getint()
     offset = offsetbox.getint()
     if arraydescr.is_array_of_pointers():
@@ -228,7 +234,7 @@
     return cpu.bh_new_with_vtable(vtable, descr)
 
 def do_new_with_vtable(cpu, _, clsbox):
-    return BoxPtr(exec_new_with_vtable(cpu, clsbox))
+    return exec_new_with_vtable(cpu, clsbox)
 
 def do_int_add_ovf(cpu, metainterp, box1, box2):
     # the overflow operations can be called without a metainterp, if an
@@ -241,7 +247,7 @@
         assert metainterp is not None
         metainterp.execute_raised(OverflowError(), constant=True)
         z = 0
-    return BoxInt(z)
+    return z
 
 def do_int_sub_ovf(cpu, metainterp, box1, box2):
     a = box1.getint()
@@ -252,7 +258,7 @@
         assert metainterp is not None
         metainterp.execute_raised(OverflowError(), constant=True)
         z = 0
-    return BoxInt(z)
+    return z
 
 def do_int_mul_ovf(cpu, metainterp, box1, box2):
     a = box1.getint()
@@ -263,16 +269,16 @@
         assert metainterp is not None
         metainterp.execute_raised(OverflowError(), constant=True)
         z = 0
-    return BoxInt(z)
+    return z
 
 def do_same_as_i(cpu, _, v):
-    return v
+    return v.getint()
 
 def do_same_as_r(cpu, _, v):
-    return v
+    return v.getref_base()
 
 def do_same_as_f(cpu, _, v):
-    return v
+    return v.getfloatstorage()
 
 def do_copystrcontent(cpu, _, srcbox, dstbox,
                       srcstartbox, dststartbox, lengthbox):
@@ -339,7 +345,7 @@
             # parameters.
             name = 'bhimpl_' + key.lower()
             if hasattr(BlackholeInterpreter, name):
-                func = make_execute_function_with_boxes(
+                func = make_execute_function(
                     key.lower(),
                     getattr(BlackholeInterpreter, name).im_func)
                 if func is not None:
@@ -374,7 +380,7 @@
             raise AssertionError("missing %r" % (key,))
     return execute_by_num_args
 
-def make_execute_function_with_boxes(name, func):
+def make_execute_function(name, func):
     # Make a wrapper for 'func'.  The func is a simple bhimpl_xxx function
     # from the BlackholeInterpreter class.  The wrapper is a new function
     # that receives and returns boxed values.
@@ -454,8 +460,22 @@
 execute_varargs._annspecialcase_ = 'specialize:arg(2)'
 
 
-def execute_nonspec(cpu, metainterp, opnum, argboxes, descr=None):
-    xxxx
+def execute_nonspec_const(cpu, metainterp, opnum, argboxes, descr=None,
+                          type='i'):
+    if type == 'i':
+        return ConstInt(_execute_nonspec(cpu, metainterp, opnum, argboxes,
+                                         descr, 'i'))
+    elif type == 'f':
+        return ConstFloat(_execute_nonspec(cpu, metainterp, opnum, argboxes,
+                                         descr, 'f'))
+    elif type == 'r':
+        return ConstPtr(_execute_nonspec(cpu, metainterp, opnum, argboxes,
+                                         descr, 'r'))
+    else:
+        assert False
+
+ at specialize.arg(5)
+def _execute_nonspec(cpu, metainterp, opnum, argboxes, descr=None, type='i'):
     arity = resoperation.oparity[opnum]
     assert arity == -1 or len(argboxes) == arity
     if resoperation.opwithdescr[opnum]:
diff --git a/rpython/jit/metainterp/history.py b/rpython/jit/metainterp/history.py
--- a/rpython/jit/metainterp/history.py
+++ b/rpython/jit/metainterp/history.py
@@ -202,6 +202,9 @@
     def same_constant(self, other):
         raise NotImplementedError
 
+    def repr(self, memo):
+        return self.repr_rpython()
+
     def __repr__(self):
         return 'Const(%s)' % self._getrepr_()
 
diff --git a/rpython/jit/metainterp/optimizeopt/intbounds.py b/rpython/jit/metainterp/optimizeopt/intbounds.py
--- a/rpython/jit/metainterp/optimizeopt/intbounds.py
+++ b/rpython/jit/metainterp/optimizeopt/intbounds.py
@@ -151,6 +151,7 @@
                 # nonneg % power-of-two ==> nonneg & (power-of-two - 1)
                 arg1 = op.getarg(0)
                 arg2 = ConstInt(val-1)
+                xxx
                 op = op.copy_and_change(rop.INT_AND, args=[arg1, arg2])
         self.emit_operation(op)
         if v2.is_constant():
@@ -197,7 +198,7 @@
         if lastop is not None:
             opnum = lastop.getopnum()
             args = lastop.getarglist()
-            result = lastop.result
+            result = lastop
             # If the INT_xxx_OVF was replaced with INT_xxx or removed
             # completely, then we can kill the GUARD_NO_OVERFLOW.
             if (opnum != rop.INT_ADD_OVF and
@@ -236,13 +237,14 @@
         v1 = self.getvalue(op.getarg(0))
         v2 = self.getvalue(op.getarg(1))
         resbound = v1.intbound.add_bound(v2.intbound)
+        r = self.getvalue(op)
         if resbound.bounded():
             # Transform into INT_ADD.  The following guard will be killed
             # by optimize_GUARD_NO_OVERFLOW; if we see instead an
             # optimize_GUARD_OVERFLOW, then InvalidLoop.
             op = op.copy_and_change(rop.INT_ADD)
         self.emit_operation(op) # emit the op
-        r = self.getvalue(op)
+        r.box = op
         r.intbound.intersect(resbound)
 
     def optimize_INT_SUB_OVF(self, op):
@@ -253,6 +255,7 @@
             return
         resbound = v1.intbound.sub_bound(v2.intbound)
         if resbound.bounded():
+            xxxx
             op = op.copy_and_change(rop.INT_SUB)
         self.emit_operation(op) # emit the op
         r = self.getvalue(op)
@@ -263,6 +266,7 @@
         v2 = self.getvalue(op.getarg(1))
         resbound = v1.intbound.mul_bound(v2.intbound)
         if resbound.bounded():
+            xxx
             op = op.copy_and_change(rop.INT_MUL)
         self.emit_operation(op)
         r = self.getvalue(op)
diff --git a/rpython/jit/metainterp/optimizeopt/optimizer.py b/rpython/jit/metainterp/optimizeopt/optimizer.py
--- a/rpython/jit/metainterp/optimizeopt/optimizer.py
+++ b/rpython/jit/metainterp/optimizeopt/optimizer.py
@@ -1,5 +1,5 @@
 from rpython.jit.metainterp import jitprof, resume, compile
-from rpython.jit.metainterp.executor import execute_nonspec
+from rpython.jit.metainterp.executor import execute_nonspec_const
 from rpython.jit.metainterp.history import BoxInt, BoxFloat, Const, ConstInt, REF
 from rpython.jit.metainterp.optimizeopt.intutils import IntBound, IntUnbounded, \
                                                      ImmutableIntUnbounded, \
@@ -542,7 +542,7 @@
 
     def emit_operation(self, op):
         if op.returns_bool_result():
-            self.bool_boxes[self.getvalue(op.result)] = None
+            self.bool_boxes[self.getvalue(op)] = None
         self._emit_operation(op)
 
     @specialize.argtype(0)
@@ -640,9 +640,9 @@
     def constant_fold(self, op):
         argboxes = [self.get_constant_box(op.getarg(i))
                     for i in range(op.numargs())]
-        resbox = execute_nonspec(self.cpu, None,
-                                 op.getopnum(), argboxes, op.getdescr())
-        return resbox.constbox()
+        return execute_nonspec_const(self.cpu, None,
+                                       op.getopnum(), argboxes,
+                                       op.getdescr(), op.type)
 
     #def optimize_GUARD_NO_OVERFLOW(self, op):
     #    # otherwise the default optimizer will clear fields, which is unwanted
diff --git a/rpython/jit/metainterp/optimizeopt/pure.py b/rpython/jit/metainterp/optimizeopt/pure.py
--- a/rpython/jit/metainterp/optimizeopt/pure.py
+++ b/rpython/jit/metainterp/optimizeopt/pure.py
@@ -34,7 +34,7 @@
                 resbox = self.optimizer.constant_fold(op)
                 # note that INT_xxx_OVF is not done from here, and the
                 # overflows in the INT_xxx operations are ignored
-                self.optimizer.make_constant(op.result, resbox)
+                self.optimizer.make_constant(op, resbox)
                 return
 
             # did we do the exact same operation already?
@@ -43,8 +43,7 @@
             oldop = self.pure_operations.get(args, None)
             if oldop is not None and oldop.getdescr() is op.getdescr():
                 assert oldop.getopnum() == op.getopnum()
-                self.optimizer.make_equal_to(op.result, self.getvalue(oldop.result),
-                                   True)
+                self.optimizer.make_equal_to(op, self.getvalue(oldop), True)
                 return
             else:
                 self.pure_operations[args] = op
@@ -53,7 +52,7 @@
         # otherwise, the operation remains
         self.emit_operation(op)
         if op.returns_bool_result():
-            self.optimizer.bool_boxes[self.getvalue(op.result)] = None
+            self.optimizer.bool_boxes[self.getvalue(op)] = None
         if nextop:
             self.emit_operation(nextop)
 
@@ -62,7 +61,7 @@
         result = self._can_optimize_call_pure(op)
         if result is not None:
             # this removes a CALL_PURE with all constant arguments.
-            self.make_constant(op.result, result)
+            self.make_constant(op, result)
             self.last_emitted_operation = REMOVED
             return
 
@@ -74,7 +73,7 @@
             assert oldop.getopnum() == op.getopnum()
             # this removes a CALL_PURE that has the same (non-constant)
             # arguments as a previous CALL_PURE.
-            self.make_equal_to(op.result, self.getvalue(oldop.result))
+            self.make_equal_to(op, self.getvalue(oldop))
             self.last_emitted_operation = REMOVED
             return
         else:
@@ -83,8 +82,7 @@
 
         # replace CALL_PURE with just CALL
         args = op.getarglist()
-        self.emit_operation(ResOperation(rop.CALL, args, op.result,
-                                         op.getdescr()))
+        self.emit_operation(ResOperation(rop.CALL, args, op.getdescr()))
     optimize_CALL_PURE_R = optimize_CALL_PURE_I
     optimize_CALL_PURE_F = optimize_CALL_PURE_I
     optimize_CALL_PURE_N = optimize_CALL_PURE_I
diff --git a/rpython/jit/metainterp/optimizeopt/rewrite.py b/rpython/jit/metainterp/optimizeopt/rewrite.py
--- a/rpython/jit/metainterp/optimizeopt/rewrite.py
+++ b/rpython/jit/metainterp/optimizeopt/rewrite.py
@@ -38,10 +38,10 @@
             value = self.getvalue(oldop.result)
             if value.is_constant():
                 if value.box.same_constant(CONST_1):
-                    self.make_constant(op.result, CONST_0)
+                    self.make_constant(op, CONST_0)
                     return True
                 elif value.box.same_constant(CONST_0):
-                    self.make_constant(op.result, CONST_1)
+                    self.make_constant(op, CONST_1)
                     return True
 
         return False
@@ -79,19 +79,19 @@
         v1 = self.getvalue(op.getarg(0))
         v2 = self.getvalue(op.getarg(1))
         if v1.is_null() or v2.is_null():
-            self.make_constant_int(op.result, 0)
+            self.make_constant_int(op, 0)
             return
         elif v2.is_constant():
             val = v2.box.getint()
             if val == -1 or v1.intbound.lower >= 0 \
                 and v1.intbound.upper <= val & ~(val + 1):
-                self.make_equal_to(op.result, v1)
+                self.make_equal_to(op, v1)
                 return
         elif v1.is_constant():
             val = v1.box.getint()
             if val == -1 or v2.intbound.lower >= 0 \
                 and v2.intbound.upper <= val & ~(val + 1):
-                self.make_equal_to(op.result, v2)
+                self.make_equal_to(op, v2)
                 return
 
         self.emit_operation(op)
@@ -115,7 +115,7 @@
             op = op.copy_and_change(rop.INT_NEG, args=[v2.box])
             self.emit_operation(op)
         elif v1 is v2:
-            self.make_constant_int(op.result, 0)
+            self.make_constant_int(op, 0)
         else:
             self.emit_operation(op)
             # Synthesize the reverse ops for optimize_default to reuse
@@ -149,7 +149,7 @@
             self.make_equal_to(op.result, v1)
         elif (v1.is_constant() and v1.box.getint() == 0) or \
              (v2.is_constant() and v2.box.getint() == 0):
-            self.make_constant_int(op.result, 0)
+            self.make_constant_int(op, 0)
         else:
             for lhs, rhs in [(v1, v2), (v2, v1)]:
                 if lhs.is_constant():
@@ -177,7 +177,7 @@
         if v2.is_constant() and v2.box.getint() == 0:
             self.make_equal_to(op.result, v1)
         elif v1.is_constant() and v1.box.getint() == 0:
-            self.make_constant_int(op.result, 0)
+            self.make_constant_int(op, 0)
         else:
             self.emit_operation(op)
 
@@ -188,7 +188,7 @@
         if v2.is_constant() and v2.box.getint() == 0:
             self.make_equal_to(op.result, v1)
         elif v1.is_constant() and v1.box.getint() == 0:
-            self.make_constant_int(op.result, 0)
+            self.make_constant_int(op, 0)
         else:
             self.emit_operation(op)
 
@@ -418,9 +418,9 @@
     def _optimize_nullness(self, op, box, expect_nonnull):
         value = self.getvalue(box)
         if value.is_nonnull():
-            self.make_constant_int(op.result, expect_nonnull)
+            self.make_constant_int(op, expect_nonnull)
         elif value.is_null():
-            self.make_constant_int(op.result, not expect_nonnull)
+            self.make_constant_int(op, not expect_nonnull)
         else:
             self.emit_operation(op)
 
@@ -439,17 +439,17 @@
         if value0.is_virtual():
             if value1.is_virtual():
                 intres = (value0 is value1) ^ expect_isnot
-                self.make_constant_int(op.result, intres)
+                self.make_constant_int(op, intres)
             else:
-                self.make_constant_int(op.result, expect_isnot)
+                self.make_constant_int(op, expect_isnot)
         elif value1.is_virtual():
-            self.make_constant_int(op.result, expect_isnot)
+            self.make_constant_int(op, expect_isnot)
         elif value1.is_null():
             self._optimize_nullness(op, op.getarg(0), expect_isnot)
         elif value0.is_null():
             self._optimize_nullness(op, op.getarg(1), expect_isnot)
         elif value0 is value1:
-            self.make_constant_int(op.result, not expect_isnot)
+            self.make_constant_int(op, not expect_isnot)
         else:
             if instance:
                 cls0 = value0.get_constant_class(self.optimizer.cpu)
@@ -458,7 +458,7 @@
                     if cls1 is not None and not cls0.same_constant(cls1):
                         # cannot be the same object, as we know that their
                         # class is different
-                        self.make_constant_int(op.result, expect_isnot)
+                        self.make_constant_int(op, expect_isnot)
                         return
             self.emit_operation(op)
 
@@ -539,7 +539,7 @@
         # Note that it's also done in pure.py.  For now we need both...
         result = self._can_optimize_call_pure(op)
         if result is not None:
-            self.make_constant(op.result, result)
+            self.make_constant(op, result)
             self.last_emitted_operation = REMOVED
             return
         self.emit_operation(op)
@@ -565,7 +565,7 @@
             self.make_equal_to(op.result, v1)
             return
         elif v1.is_constant() and v1.box.getint() == 0:
-            self.make_constant_int(op.result, 0)
+            self.make_constant_int(op, 0)
             return
         if v1.intbound.known_ge(IntBound(0, 0)) and v2.is_constant():
             val = v2.box.getint()
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py
@@ -19,7 +19,7 @@
     opt = optimizeopt.Optimizer(FakeMetaInterpStaticData(LLtypeMixin.cpu),
                                 None)
     fdescr = ResumeGuardDescr()
-    op = ResOperation(rop.GUARD_TRUE, ['dummy'], None, descr=fdescr)
+    op = ResOperation(rop.GUARD_TRUE, ['dummy'], descr=fdescr)
     # setup rd data
     fi0 = resume.FrameInfo(None, "code0", 11)
     fdescr.rd_frame_info_list = resume.FrameInfo(fi0, "code1", 33)
@@ -99,7 +99,7 @@
     def optimize_loop(self, ops, optops, call_pure_results=None):
         loop = self.parse(ops)
         token = JitCellToken()
-        loop.operations = [ResOperation(rop.LABEL, loop.inputargs, None, descr=TargetToken(token))] + \
+        loop.operations = [ResOperation(rop.LABEL, loop.inputargs, descr=TargetToken(token))] + \
                           loop.operations
         if loop.operations[-1].getopnum() == rop.JUMP:
             loop.operations[-1].setdescr(token)
@@ -232,7 +232,7 @@
     def test_remove_guard_class_constant(self):
         ops = """
         [i0]
-        p0 = same_as(ConstPtr(myptr))
+        p0 = same_as_r(ConstPtr(myptr))
         guard_class(p0, ConstClass(node_vtable)) []
         jump(i0)
         """
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_util.py b/rpython/jit/metainterp/optimizeopt/test/test_util.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_util.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_util.py
@@ -410,9 +410,9 @@
         preamble.inputargs = inputargs
 
         token = JitCellToken()
-        preamble.operations = [ResOperation(rop.LABEL, inputargs, None, descr=TargetToken(token))] + \
+        preamble.operations = [ResOperation(rop.LABEL, inputargs, descr=TargetToken(token))] + \
                               operations +  \
-                              [ResOperation(rop.LABEL, jump_args, None, descr=token)]
+                              [ResOperation(rop.LABEL, jump_args, descr=token)]
         self._do_optimize_loop(preamble, call_pure_results)
 
         assert preamble.operations[-1].getopnum() == rop.LABEL
@@ -451,11 +451,11 @@
 def convert_old_style_to_targets(loop, jump):
     newloop = TreeLoop(loop.name)
     newloop.inputargs = loop.inputargs
-    newloop.operations = [ResOperation(rop.LABEL, loop.inputargs, None, descr=FakeDescr())] + \
+    newloop.operations = [ResOperation(rop.LABEL, loop.inputargs, descr=FakeDescr())] + \
                       loop.operations
     if not jump:
         assert newloop.operations[-1].getopnum() == rop.JUMP
-        newloop.operations[-1] = ResOperation(rop.LABEL, newloop.operations[-1].getarglist(), None, descr=FakeDescr())
+        newloop.operations[-1] = ResOperation(rop.LABEL, newloop.operations[-1].getarglist(), descr=FakeDescr())
     return newloop
 
 # ____________________________________________________________
diff --git a/rpython/jit/metainterp/optimizeopt/util.py b/rpython/jit/metainterp/optimizeopt/util.py
--- a/rpython/jit/metainterp/optimizeopt/util.py
+++ b/rpython/jit/metainterp/optimizeopt/util.py
@@ -134,10 +134,17 @@
     width = totwidth / 2 - 1
     print ' Comparing lists '.center(totwidth, '-')
     text_right = text_right or 'expected'
+    memo = {}
     print '%s| %s' % ('optimized'.center(width), text_right.center(width))
     for op1, op2 in itertools.izip_longest(oplist1, oplist2, fillvalue=''):
-        txt1 = str(op1)
-        txt2 = str(op2)
+        if op1:
+            txt1 = op1.repr(memo)
+        else:
+            txt1 = ''
+        if op2:
+            txt2 = op2.repr(memo)
+        else:
+            txt2 = ''
         while txt1 or txt2:
             part1 = txt1[:width]
             part2 = txt2[:width]
diff --git a/rpython/jit/metainterp/optimizeopt/virtualize.py b/rpython/jit/metainterp/optimizeopt/virtualize.py
--- a/rpython/jit/metainterp/optimizeopt/virtualize.py
+++ b/rpython/jit/metainterp/optimizeopt/virtualize.py
@@ -15,15 +15,13 @@
 
 class AbstractVirtualValue(optimizer.OptValue):
     _attrs_ = ('keybox', 'source_op', '_cached_vinfo')
-    box = None
     level = optimizer.LEVEL_NONNULL
     is_about_raw = False
+    box = None
     _cached_vinfo = None
 
-    def __init__(self, keybox, source_op=None):
-        self.keybox = keybox   # only used as a key in dictionaries
-        self.source_op = source_op  # the NEW_WITH_VTABLE/NEW_ARRAY operation
-                                    # that builds this box
+    def __init__(self, source_op):
+        self.source_op = source_op
 
     def is_forced_virtual(self):
         return self.box is not None
@@ -35,7 +33,7 @@
 
     def force_box(self, optforce):
         if self.box is None:
-            optforce.forget_numberings(self.keybox)
+            optforce.forget_numberings(self.source_op)
             self._really_force(optforce)
         return self.box
 
@@ -75,8 +73,8 @@
 class AbstractVirtualStructValue(AbstractVirtualValue):
     _attrs_ = ('_fields', 'cpu', '_cached_sorted_fields')
 
-    def __init__(self, cpu, keybox, source_op=None):
-        AbstractVirtualValue.__init__(self, keybox, source_op)
+    def __init__(self, cpu, source_op):
+        AbstractVirtualValue.__init__(self, source_op)
         self.cpu = cpu
         self._fields = {}
         self._cached_sorted_fields = None
@@ -151,7 +149,7 @@
             # keep self._fields, because it's all immutable anyway
         else:
             optforce.emit_operation(op)
-            self.box = box = op.result
+            self.box = box = op
             #
             iteritems = self._fields.iteritems()
             if not we_are_translated(): #random order is fine, except for tests
@@ -200,8 +198,8 @@
 class VirtualValue(AbstractVirtualStructValue):
     level = optimizer.LEVEL_KNOWNCLASS
 
-    def __init__(self, cpu, known_class, keybox, source_op=None):
-        AbstractVirtualStructValue.__init__(self, cpu, keybox, source_op)
+    def __init__(self, cpu, known_class, source_op):
+        AbstractVirtualStructValue.__init__(self, cpu, source_op)
         assert isinstance(known_class, Const)
         self.known_class = known_class
 
@@ -522,9 +520,9 @@
 
     _last_guard_not_forced_2 = None
 
-    def make_virtual(self, known_class, box, source_op=None):
-        vvalue = VirtualValue(self.optimizer.cpu, known_class, box, source_op)
-        self.make_equal_to(box, vvalue)
+    def make_virtual(self, known_class, source_op):
+        vvalue = VirtualValue(self.optimizer.cpu, known_class, source_op)
+        self.make_equal_to(source_op, vvalue)
         return vvalue
 
     def make_varray(self, arraydescr, size, box, source_op=None,
@@ -702,10 +700,10 @@
             self.emit_operation(op)
 
     def optimize_NEW_WITH_VTABLE(self, op):
-        self.make_virtual(op.getarg(0), op.result, op)
+        self.make_virtual(op.getarg(0), op)
 
     def optimize_NEW(self, op):
-        self.make_vstruct(op.getdescr(), op.result, op)
+        self.make_vstruct(op.getdescr(), op)
 
     def optimize_NEW_ARRAY(self, op):
         sizebox = self.get_constant_box(op.getarg(0))
diff --git a/rpython/jit/metainterp/resoperation.py b/rpython/jit/metainterp/resoperation.py
--- a/rpython/jit/metainterp/resoperation.py
+++ b/rpython/jit/metainterp/resoperation.py
@@ -1,5 +1,6 @@
 from rpython.rlib.objectmodel import we_are_translated, specialize
 from rpython.rlib.objectmodel import compute_identity_hash
+from rpython.rtyper.lltypesystem import lltype, llmemory
 
 class AbstractValue(object):
     def _get_hash_(self):
@@ -8,8 +9,10 @@
     def same_box(self, other):
         return self is other
 
- at specialize.argtype(2)
-def ResOperation(opnum, args, result, descr=None):
+    def repr_short(self, memo):
+        return self.repr(memo)
+
+def ResOperation(opnum, args, descr=None):
     cls = opclasses[opnum]
     op = cls()
     op.initarglist(args)
@@ -20,16 +23,6 @@
         elif op.is_guard():
             assert not descr.final_descr
         op.setdescr(descr)
-    if isinstance(result, int):
-        op._resint = result
-    elif isinstance(result, float):
-        op._resfloat = result
-    elif result is None:
-        pass
-    else:
-        from rpython.rtyper.lltypesystem import lltype, llmemory
-        assert lltype.typeOf(result) == llmemory.GCREF
-        op._resref = result
     return op
 
 
@@ -97,13 +90,25 @@
         "shallow copy: the returned operation is meant to be used in place of self"
         if args is None:
             args = self.getarglist()
-        if result is None:
-            result = self.result
         if descr is None:
             descr = self.getdescr()
-        newop = ResOperation(opnum, args, result, descr)
+        newop = ResOperation(opnum, args, descr)
+        newop.copy_value_from(self)
         return newop
 
+    @specialize.argtype(1)
+    def setvalue(self, value):
+        if isinstance(value, int):
+            self._resint = value
+        elif isinstance(value, float):
+            self._resfloat = value
+        elif value is None:
+            pass
+        else:
+            assert lltype.typeOf(value) == llmemory.GCREF
+            self._resref = value
+
+
     def clone(self):
         args = self.getarglist()
         descr = self.getdescr()
@@ -117,9 +122,15 @@
 
     def repr(self, memo, graytext=False):
         # RPython-friendly version
-        XXX
-        if self.result is not None:
-            sres = '%s = ' % (self.result,)
+        if self.type != 'v':
+            try:
+                sres = '%s = ' % memo[self]
+            except KeyError:
+                name = self.type + str(len(memo))
+                memo[self] = name
+                sres = name + ' = '
+        #if self.result is not None:
+        #    sres = '%s = ' % (self.result,)
         else:
             sres = ''
         if self.name:
@@ -132,12 +143,15 @@
         descr = self.getdescr()
         if descr is None or we_are_translated():
             return '%s%s%s(%s)' % (prefix, sres, self.getopname(),
-                                   ', '.join([str(a) for a in args]))
+                                   ', '.join([a.repr_short(memo) for a in args]))
         else:
             return '%s%s%s(%s)' % (prefix, sres, self.getopname(),
-                                   ', '.join([str(a) for a in args] +
+                                   ', '.join([a.repr_short(memo) for a in args] +
                                              ['descr=%r' % descr]))
 
+    def repr_short(self, memo):
+        return memo[self]
+
     def getopname(self):
         try:
             return opname[self.getopnum()].lower()
@@ -274,43 +288,67 @@
 
     type = 'i'
 
+    _resint = 0
+
     def getint(self):
         return self._resint
 
     def setint(self, intval):
         self._resint = intval
 
+    def copy_value_from(self, other):
+        self.setint(other.getint())
+
 class FloatOp(object):
     _mixin_ = True
 
     type = 'f'
 
+    _resfloat = 0.0
+
     def getfloatstorage(self):
         return self._resfloat
 
     def setfloatstorage(self, floatval):
         self._resfloat = floatval
 
+    def copy_value_from(self, other):
+        self.setfloatstorage(other.getfloatstorage())
+
 class RefOp(object):
     _mixin_ = True
 
     type = 'r'
 
+    _resref = lltype.nullptr(llmemory.GCREF.TO)
+
     def getref_base(self):
         return self._resref
 
     def setref_base(self, refval):
         self._resref = refval
 
-class InputArgInt(IntOp, AbstractValue):
+    def copy_value_from(self, other):
+        self.setref_base(other.getref_base())
+
+class AbstractInputArg(AbstractValue):    
+    def repr(self, memo):
+        try:
+            return memo[self]
+        except KeyError:
+            name = self.type + str(len(memo))
+            memo[self] = name
+            return name
+        
+class InputArgInt(IntOp, AbstractInputArg):
     def __init__(self, intval):
-        self.setint(intval)
+        self.setint(intval)            
 
-class InputArgFloat(FloatOp, AbstractValue):
+class InputArgFloat(FloatOp, AbstractInputArg):
     def __init__(self, f):
         self.setfloatstorage(f)
 
-class InputArgRef(RefOp, AbstractValue):
+class InputArgRef(RefOp, AbstractInputArg):
     def __init__(self, r):
         self.setref_base(r)
 
diff --git a/rpython/jit/tool/oparser.py b/rpython/jit/tool/oparser.py
--- a/rpython/jit/tool/oparser.py
+++ b/rpython/jit/tool/oparser.py
@@ -275,18 +275,7 @@
             assert descr is None
             return op
         else:
-            tp = optypes[opnum]
-            if tp == 'i':
-                result = 0
-            elif tp == 'r':
-                from rpython.rtyper.lltypesystem import lltype, llmemory
-                result = lltype.nullptr(llmemory.GCREF.TO)
-            elif tp == 'f':
-                result = 0.0
-            else:
-                assert tp == 'n'
-                result = None
-            return ResOperation(opnum, args, result, descr)
+            return ResOperation(opnum, args, descr)
 
     def parse_result_op(self, line):
         res, op = line.split("=", 1)

From noreply at buildbot.pypy.org  Fri Nov 14 17:18:53 2014
From: noreply at buildbot.pypy.org (fijal)
Date: Fri, 14 Nov 2014 17:18:53 +0100 (CET)
Subject: [pypy-commit] pypy optresult: more hacking
Message-ID: <20141114161853.85E751D281E@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: optresult
Changeset: r74522:555089bcfdc6
Date: 2014-11-14 18:18 +0200
http://bitbucket.org/pypy/pypy/changeset/555089bcfdc6/

Log:	more hacking

diff --git a/rpython/jit/metainterp/optimizeopt/intbounds.py b/rpython/jit/metainterp/optimizeopt/intbounds.py
--- a/rpython/jit/metainterp/optimizeopt/intbounds.py
+++ b/rpython/jit/metainterp/optimizeopt/intbounds.py
@@ -73,9 +73,9 @@
         v2 = self.getvalue(op.getarg(1))
         if v1 is v2:
             if op.getopnum() == rop.INT_OR:
-                self.make_equal_to(op.result, v1)
+                self.make_equal_to(op, v1)
             else:
-                self.make_constant_int(op.result, 0)
+                self.make_constant_int(op, 0)
             return
         self.emit_operation(op)
         if v1.intbound.known_ge(IntBound(0, 0)) and \
@@ -179,7 +179,7 @@
         # b.has_lower
         if b.has_lower and b.has_upper:
             # Synthesize the reverse op for optimize_default to reuse
-            self.pure(rop.INT_RSHIFT, [op.result, op.getarg(1)], op.getarg(0))
+            self.pure(rop.INT_RSHIFT, [op, op.getarg(1)], op.getarg(0))
 
     def optimize_INT_RSHIFT(self, op):
         v1 = self.getvalue(op.getarg(0))
@@ -187,7 +187,7 @@
         b = v1.intbound.rshift_bound(v2.intbound)
         if b.has_lower and b.has_upper and b.lower == b.upper:
             # constant result (likely 0, for rshifts that kill all bits)
-            self.make_constant_int(op.result, b.lower)
+            self.make_constant_int(op, b.lower)
         else:
             self.emit_operation(op)
             r = self.getvalue(op)
@@ -251,7 +251,7 @@
         v1 = self.getvalue(op.getarg(0))
         v2 = self.getvalue(op.getarg(1))
         if v1 is v2:
-            self.make_constant_int(op.result, 0)
+            self.make_constant_int(op, 0)
             return
         resbound = v1.intbound.sub_bound(v2.intbound)
         if resbound.bounded():
@@ -276,9 +276,9 @@
         v1 = self.getvalue(op.getarg(0))
         v2 = self.getvalue(op.getarg(1))
         if v1.intbound.known_lt(v2.intbound):
-            self.make_constant_int(op.result, 1)
+            self.make_constant_int(op, 1)
         elif v1.intbound.known_ge(v2.intbound) or v1 is v2:
-            self.make_constant_int(op.result, 0)
+            self.make_constant_int(op, 0)
         else:
             self.emit_operation(op)
 
@@ -286,9 +286,9 @@
         v1 = self.getvalue(op.getarg(0))
         v2 = self.getvalue(op.getarg(1))
         if v1.intbound.known_gt(v2.intbound):
-            self.make_constant_int(op.result, 1)
+            self.make_constant_int(op, 1)
         elif v1.intbound.known_le(v2.intbound) or v1 is v2:
-            self.make_constant_int(op.result, 0)
+            self.make_constant_int(op, 0)
         else:
             self.emit_operation(op)
 
@@ -296,9 +296,9 @@
         v1 = self.getvalue(op.getarg(0))
         v2 = self.getvalue(op.getarg(1))
         if v1.intbound.known_le(v2.intbound) or v1 is v2:
-            self.make_constant_int(op.result, 1)
+            self.make_constant_int(op, 1)
         elif v1.intbound.known_gt(v2.intbound):
-            self.make_constant_int(op.result, 0)
+            self.make_constant_int(op, 0)
         else:
             self.emit_operation(op)
 
@@ -306,9 +306,9 @@
         v1 = self.getvalue(op.getarg(0))
         v2 = self.getvalue(op.getarg(1))
         if v1.intbound.known_ge(v2.intbound) or v1 is v2:
-            self.make_constant_int(op.result, 1)
+            self.make_constant_int(op, 1)
         elif v1.intbound.known_lt(v2.intbound):
-            self.make_constant_int(op.result, 0)
+            self.make_constant_int(op, 0)
         else:
             self.emit_operation(op)
 
@@ -316,11 +316,11 @@
         v1 = self.getvalue(op.getarg(0))
         v2 = self.getvalue(op.getarg(1))
         if v1.intbound.known_gt(v2.intbound):
-            self.make_constant_int(op.result, 0)
+            self.make_constant_int(op, 0)
         elif v1.intbound.known_lt(v2.intbound):
-            self.make_constant_int(op.result, 0)
+            self.make_constant_int(op, 0)
         elif v1 is v2:
-            self.make_constant_int(op.result, 1)
+            self.make_constant_int(op, 1)
         else:
             self.emit_operation(op)
 
@@ -328,18 +328,18 @@
         v1 = self.getvalue(op.getarg(0))
         v2 = self.getvalue(op.getarg(1))
         if v1.intbound.known_gt(v2.intbound):
-            self.make_constant_int(op.result, 1)
+            self.make_constant_int(op, 1)
         elif v1.intbound.known_lt(v2.intbound):
-            self.make_constant_int(op.result, 1)
+            self.make_constant_int(op, 1)
         elif v1 is v2:
-            self.make_constant_int(op.result, 0)
+            self.make_constant_int(op, 0)
         else:
             self.emit_operation(op)
 
     def optimize_INT_FORCE_GE_ZERO(self, op):
         value = self.getvalue(op.getarg(0))
         if value.intbound.known_ge(IntBound(0, 0)):
-            self.make_equal_to(op.result, value)
+            self.make_equal_to(op, value)
         else:
             self.emit_operation(op)
 
diff --git a/rpython/jit/metainterp/optimizeopt/rewrite.py b/rpython/jit/metainterp/optimizeopt/rewrite.py
--- a/rpython/jit/metainterp/optimizeopt/rewrite.py
+++ b/rpython/jit/metainterp/optimizeopt/rewrite.py
@@ -26,7 +26,8 @@
 
     def propagate_forward(self, op):
         if op.boolinverse != -1 or op.boolreflex != -1:
-            args = self.optimizer.make_args_key(op)
+            args = self.optimizer.make_args_key(op.getopnum(),
+                                                op.getarglist(), op.getdescr())
             if self.find_rewritable_bool(op, args):
                 return
 
@@ -35,7 +36,7 @@
     def try_boolinvers(self, op, targs):
         oldop = self.get_pure_result(targs)
         if oldop is not None and oldop.getdescr() is op.getdescr():
-            value = self.getvalue(oldop.result)
+            value = self.getvalue(oldop)
             if value.is_constant():
                 if value.box.same_constant(CONST_1):
                     self.make_constant(op, CONST_0)
@@ -50,26 +51,26 @@
     def find_rewritable_bool(self, op, args):
         oldopnum = op.boolinverse
         if oldopnum != -1:
-            targs = self.optimizer.make_args_key(ResOperation(oldopnum, [args[0], args[1]],
-                                                              None))
+            targs = self.optimizer.make_args_key(oldopnum, [args[0], args[1]],
+                                                 None)
             if self.try_boolinvers(op, targs):
                 return True
 
         oldopnum = op.boolreflex # FIXME: add INT_ADD, INT_MUL
         if oldopnum != -1:
-            targs = self.optimizer.make_args_key(ResOperation(oldopnum, [args[1], args[0]],
-                                                              None))
+            targs = self.optimizer.make_args_key(oldopnum, [args[1], args[0]],
+                                                 None)
             oldop = self.get_pure_result(targs)
             if oldop is not None and oldop.getdescr() is op.getdescr():
-                self.make_equal_to(op.result, self.getvalue(oldop.result))
+                self.make_equal_to(op, self.getvalue(oldop))
                 return True
 
         if op.boolreflex == -1:
             return False
         oldopnum = opclasses[op.boolreflex].boolinverse
         if oldopnum != -1:
-            targs = self.optimizer.make_args_key(
-                ResOperation(oldopnum, [args[1], args[0]], None))
+            targs = self.optimizer.make_args_key(oldopnum, [args[1], args[0]],
+                                                 None)
             if self.try_boolinvers(op, targs):
                 return True
 
@@ -100,9 +101,9 @@
         v1 = self.getvalue(op.getarg(0))
         v2 = self.getvalue(op.getarg(1))
         if v1.is_null():
-            self.make_equal_to(op.result, v2)
+            self.make_equal_to(op, v2)
         elif v2.is_null():
-            self.make_equal_to(op.result, v1)
+            self.make_equal_to(op, v1)
         else:
             self.emit_operation(op)
 
@@ -110,7 +111,7 @@
         v1 = self.getvalue(op.getarg(0))
         v2 = self.getvalue(op.getarg(1))
         if v2.is_constant() and v2.box.getint() == 0:
-            self.make_equal_to(op.result, v1)
+            self.make_equal_to(op, v1)
         elif v1.is_constant() and v1.box.getint() == 0:
             op = op.copy_and_change(rop.INT_NEG, args=[v2.box])
             self.emit_operation(op)
@@ -128,9 +129,9 @@
 
         # If one side of the op is 0 the result is the other side.
         if v1.is_constant() and v1.box.getint() == 0:
-            self.make_equal_to(op.result, v2)
+            self.make_equal_to(op, v2)
         elif v2.is_constant() and v2.box.getint() == 0:
-            self.make_equal_to(op.result, v1)
+            self.make_equal_to(op, v1)
         else:
             self.emit_operation(op)
             self.pure(rop.INT_ADD, [op.getarg(1), op.getarg(0)], op)
@@ -144,9 +145,9 @@
 
         # If one side of the op is 1 the result is the other side.
         if v1.is_constant() and v1.box.getint() == 1:
-            self.make_equal_to(op.result, v2)
+            self.make_equal_to(op, v2)
         elif v2.is_constant() and v2.box.getint() == 1:
-            self.make_equal_to(op.result, v1)
+            self.make_equal_to(op, v1)
         elif (v1.is_constant() and v1.box.getint() == 0) or \
              (v2.is_constant() and v2.box.getint() == 0):
             self.make_constant_int(op, 0)
@@ -166,7 +167,7 @@
         v2 = self.getvalue(op.getarg(1))
 
         if v2.is_constant() and v2.box.getint() == 1:
-            self.make_equal_to(op.result, v1)
+            self.make_equal_to(op, v1)
         else:
             self.emit_operation(op)
 
@@ -175,7 +176,7 @@
         v2 = self.getvalue(op.getarg(1))
 
         if v2.is_constant() and v2.box.getint() == 0:
-            self.make_equal_to(op.result, v1)
+            self.make_equal_to(op, v1)
         elif v1.is_constant() and v1.box.getint() == 0:
             self.make_constant_int(op, 0)
         else:
@@ -186,7 +187,7 @@
         v2 = self.getvalue(op.getarg(1))
 
         if v2.is_constant() and v2.box.getint() == 0:
-            self.make_equal_to(op.result, v1)
+            self.make_equal_to(op, v1)
         elif v1.is_constant() and v1.box.getint() == 0:
             self.make_constant_int(op, 0)
         else:
@@ -197,9 +198,9 @@
         v2 = self.getvalue(op.getarg(1))
 
         if v1.is_constant() and v1.box.getint() == 0:
-            self.make_equal_to(op.result, v2)
+            self.make_equal_to(op, v2)
         elif v2.is_constant() and v2.box.getint() == 0:
-            self.make_equal_to(op.result, v1)
+            self.make_equal_to(op, v1)
         else:
             self.emit_operation(op)
 

From noreply at buildbot.pypy.org  Fri Nov 14 17:52:54 2014
From: noreply at buildbot.pypy.org (fijal)
Date: Fri, 14 Nov 2014 17:52:54 +0100 (CET)
Subject: [pypy-commit] pypy optresult: whack until a third of
 test_optimizebasic passes, nothing to see there, really
Message-ID: <20141114165254.754241C32A5@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: optresult
Changeset: r74523:2f3a6006bba2
Date: 2014-11-14 18:52 +0200
http://bitbucket.org/pypy/pypy/changeset/2f3a6006bba2/

Log:	whack until a third of test_optimizebasic passes, nothing to see
	there, really

diff --git a/rpython/jit/metainterp/optimizeopt/heap.py b/rpython/jit/metainterp/optimizeopt/heap.py
--- a/rpython/jit/metainterp/optimizeopt/heap.py
+++ b/rpython/jit/metainterp/optimizeopt/heap.py
@@ -98,7 +98,7 @@
             self._lazy_setfield = None
             if optheap.postponed_op:
                 for a in op.getarglist():
-                    if a is optheap.postponed_op.result:
+                    if a is optheap.postponed_op:
                         optheap.emit_postponed_op()
                         break
             optheap.next_optimization.propagate_forward(op)
@@ -162,6 +162,7 @@
                                          result, op.getdescr())
                     shortboxes.add_potential(getop, synthetic=True)
                 elif op.result is not None:
+                    xxxx
                     shortboxes.add_potential(op)
 
 class BogusPureField(JitException):
@@ -317,10 +318,10 @@
         try:
             res_v = d[args]
         except KeyError:
-            d[args] = self.getvalue(op.result)
+            d[args] = self.getvalue(op)
             return False
         else:
-            self.make_equal_to(op.result, res_v)
+            self.make_equal_to(op, res_v)
             self.last_emitted_operation = REMOVED
             return True
 
@@ -436,13 +437,13 @@
         cf = self.field_cache(op.getdescr())
         fieldvalue = cf.getfield_from_cache(self, structvalue)
         if fieldvalue is not None:
-            self.make_equal_to(op.result, fieldvalue)
+            self.make_equal_to(op, fieldvalue)
             return
         # default case: produce the operation
         structvalue.ensure_nonnull()
         self.emit_operation(op)
         # then remember the result of reading the field
-        fieldvalue = self.getvalue(op.result)
+        fieldvalue = self.getvalue(op)
         cf.remember_field_value(structvalue, fieldvalue, op)
     optimize_GETFIELD_GC_R = optimize_GETFIELD_GC_I
     optimize_GETFIELD_GC_F = optimize_GETFIELD_GC_I
@@ -452,7 +453,7 @@
         cf = self.field_cache(op.getdescr())
         fieldvalue = cf.getfield_from_cache(self, structvalue)
         if fieldvalue is not None:
-            self.make_equal_to(op.result, fieldvalue)
+            self.make_equal_to(op, fieldvalue)
             return
         # default case: produce the operation
         structvalue.ensure_nonnull()
@@ -461,7 +462,8 @@
     optimize_GETFIELD_GC_PURE_F = optimize_GETFIELD_GC_PURE_I
 
     def optimize_SETFIELD_GC(self, op):
-        if self.has_pure_result(rop.GETFIELD_GC_PURE, [op.getarg(0)],
+        opnum = self.optimizer.getfield_pure_for_descr(op.getdescr())
+        if self.has_pure_result(opnum, [op.getarg(0)],
                                 op.getdescr()):
             os.write(2, '[bogus _immutable_field_ declaration: %s]\n' %
                      (op.getdescr().repr_of_descr()))
@@ -480,7 +482,7 @@
             cf = self.arrayitem_cache(op.getdescr(), indexvalue.box.getint())
             fieldvalue = cf.getfield_from_cache(self, arrayvalue)
             if fieldvalue is not None:
-                self.make_equal_to(op.result, fieldvalue)
+                self.make_equal_to(op, fieldvalue)
                 return
         else:
             # variable index, so make sure the lazy setarrayitems are done
@@ -490,7 +492,7 @@
         self.emit_operation(op)
         # the remember the result of reading the array item
         if cf is not None:
-            fieldvalue = self.getvalue(op.result)
+            fieldvalue = self.getvalue(op)
             cf.remember_field_value(arrayvalue, fieldvalue, op)
     optimize_GETARRAYITEM_GC_R = optimize_GETARRAYITEM_GC_I
     optimize_GETARRAYITEM_GC_F = optimize_GETARRAYITEM_GC_I
@@ -505,7 +507,7 @@
             cf = self.arrayitem_cache(op.getdescr(), indexvalue.box.getint())
             fieldvalue = cf.getfield_from_cache(self, arrayvalue)
             if fieldvalue is not None:
-                self.make_equal_to(op.result, fieldvalue)
+                self.make_equal_to(op, fieldvalue)
                 return
         else:
             # variable index, so make sure the lazy setarrayitems are done
@@ -518,8 +520,8 @@
     optimize_GETARRAYITEM_GC_PURE_F = optimize_GETARRAYITEM_GC_PURE_I
 
     def optimize_SETARRAYITEM_GC(self, op):
-        if self.has_pure_result(rop.GETARRAYITEM_GC_PURE, [op.getarg(0),
-                                                           op.getarg(1)],
+        opnum = self.optimizer.getarrayitem_pure_for_descr(op.getdescr())
+        if self.has_pure_result(opnum, [op.getarg(0), op.getarg(1)],
                                 op.getdescr()):
             os.write(2, '[bogus immutable array declaration: %s]\n' %
                      (op.getdescr().repr_of_descr()))
diff --git a/rpython/jit/metainterp/optimizeopt/optimizer.py b/rpython/jit/metainterp/optimizeopt/optimizer.py
--- a/rpython/jit/metainterp/optimizeopt/optimizer.py
+++ b/rpython/jit/metainterp/optimizeopt/optimizer.py
@@ -331,6 +331,31 @@
             return self.optimizer.optpure.get_pure_result(key)
         return None
 
+    def getfield_pure_for_descr(self, descr):
+        if descr.is_pointer_field():
+            return rop.GETFIELD_GC_PURE_R
+        elif descr.is_float_field():
+            return rop.GETFIELD_GC_PURE_F
+        return rop.GETFIELD_GC_PURE_I
+
+    def getarrayitem_pure_for_descr(self, descr):
+        if descr.is_array_of_pointers():
+            return rop.GETARRAYITEM_GC_PURE_R
+        elif descr.is_array_of_floats():
+            return rop.GETARRAYITEM_GC_PURE_F
+        return rop.GETARRAYITEM_GC_PURE_I
+
+    def call_for_descr(self, descr):
+        tp = descr.get_result_type()
+        if tp == 'i':
+            return rop.CALL_I
+        elif tp == 'r':
+            return rop.CALL_R
+        elif tp == 'f':
+            return rop.CALL_F
+        assert tp == 'v'
+        return rop.CALL_N
+
     def setup(self):
         pass
 
@@ -603,6 +628,7 @@
                 # This is done after the operation is emitted to let
                 # store_final_boxes_in_guard set the guard_opnum field of the
                 # descr to the original rop.GUARD_VALUE.
+                v = self.getvalue(op)
                 constvalue = op.getarg(1).getint()
                 if constvalue == 0:
                     opnum = rop.GUARD_FALSE
@@ -610,8 +636,9 @@
                     opnum = rop.GUARD_TRUE
                 else:
                     raise AssertionError("uh?")
-                newop = ResOperation(opnum, [op.getarg(0)], op.result, descr)
+                newop = ResOperation(opnum, [op.getarg(0)], descr)
                 newop.setfailargs(op.getfailargs())
+                v.box = newop
                 return newop
             else:
                 # a real GUARD_VALUE.  Make it use one counter per value.
diff --git a/rpython/jit/metainterp/optimizeopt/pure.py b/rpython/jit/metainterp/optimizeopt/pure.py
--- a/rpython/jit/metainterp/optimizeopt/pure.py
+++ b/rpython/jit/metainterp/optimizeopt/pure.py
@@ -67,7 +67,8 @@
 
         # Step 2: check if all arguments are the same as a previous
         # CALL_PURE.
-        args = self.optimizer.make_args_key(op)
+        args = self.optimizer.make_args_key(op.getopnum(), op.getarglist(),
+                                            op.getdescr())
         oldop = self.pure_operations.get(args, None)
         if oldop is not None and oldop.getdescr() is op.getdescr():
             assert oldop.getopnum() == op.getopnum()
@@ -82,7 +83,10 @@
 
         # replace CALL_PURE with just CALL
         args = op.getarglist()
-        self.emit_operation(ResOperation(rop.CALL, args, op.getdescr()))
+        opnum = self.optimizer.call_for_descr(op.getdescr())
+        newop = ResOperation(opnum, args, op.getdescr())
+        self.getvalue(op).box = newop
+        self.emit_operation(newop)
     optimize_CALL_PURE_R = optimize_CALL_PURE_I
     optimize_CALL_PURE_F = optimize_CALL_PURE_I
     optimize_CALL_PURE_N = optimize_CALL_PURE_I
@@ -106,8 +110,7 @@
             self.pure_operations[key] = result
 
     def has_pure_result(self, opnum, args, descr):
-        op = ResOperation(opnum, args, None, descr)
-        key = self.optimizer.make_args_key(op)
+        key = self.optimizer.make_args_key(opnum, args, descr)
         op = self.pure_operations.get(key, None)
         if op is None:
             return False
diff --git a/rpython/jit/metainterp/optimizeopt/rewrite.py b/rpython/jit/metainterp/optimizeopt/rewrite.py
--- a/rpython/jit/metainterp/optimizeopt/rewrite.py
+++ b/rpython/jit/metainterp/optimizeopt/rewrite.py
@@ -224,7 +224,7 @@
                     ))
                     return
         self.emit_operation(op)
-        self.pure(rop.FLOAT_MUL, [arg2, arg1], op.result)
+        self.pure(rop.FLOAT_MUL, [arg2, arg1], op)
 
     def optimize_FLOAT_TRUEDIV(self, op):
         arg1 = op.getarg(0)
@@ -392,15 +392,16 @@
 
         resvalue = self.loop_invariant_results.get(key, None)
         if resvalue is not None:
-            self.make_equal_to(op.result, resvalue)
+            self.make_equal_to(op, resvalue)
             self.last_emitted_operation = REMOVED
             return
         # change the op to be a normal call, from the backend's point of view
         # there is no reason to have a separate operation for this
         self.loop_invariant_producer[key] = op
-        op = op.copy_and_change(rop.CALL)
-        self.emit_operation(op)
-        resvalue = self.getvalue(op.result)
+        newop = op.copy_and_change(self.optimizer.call_for_descr(op.getdescr()))
+        self.emit_operation(newop)
+        resvalue = self.getvalue(op)
+        resvalue.box = newop
         self.loop_invariant_results[key] = resvalue
     optimize_CALL_LOOPINVARIANT_R = optimize_CALL_LOOPINVARIANT_I
     optimize_CALL_LOOPINVARIANT_F = optimize_CALL_LOOPINVARIANT_I
@@ -427,7 +428,7 @@
 
     def optimize_INT_IS_TRUE(self, op):
         if self.getvalue(op.getarg(0)) in self.optimizer.bool_boxes:
-            self.make_equal_to(op.result, self.getvalue(op.getarg(0)))
+            self.make_equal_to(op, self.getvalue(op.getarg(0)))
             return
         self._optimize_nullness(op, op.getarg(0), True)
 
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py
@@ -445,7 +445,7 @@
         pv = new_with_vtable(ConstClass(node_vtable))
         setfield_gc(pv, p0, descr=valuedescr)
         guard_nonnull(p0) []
-        p1 = getfield_gc(pv, descr=valuedescr)
+        p1 = getfield_gc_r(pv, descr=valuedescr)
         guard_nonnull(p1) []
         jump(p0)
         """
@@ -589,7 +589,7 @@
     def test_p123_simple(self):
         ops = """
         [i1, p2, p3]
-        i3 = getfield_gc(p3, descr=valuedescr)
+        i3 = getfield_gc_i(p3, descr=valuedescr)
         escape(i3)
         p1 = new_with_vtable(ConstClass(node_vtable))
         setfield_gc(p1, i1, descr=valuedescr)
@@ -601,7 +601,7 @@
     def test_p123_nested(self):
         ops = """
         [i1, p2, p3]
-        i3 = getfield_gc(p3, descr=valuedescr)
+        i3 = getfield_gc_i(p3, descr=valuedescr)
         escape(i3)
         p1 = new_with_vtable(ConstClass(node_vtable))
         p1sub = new_with_vtable(ConstClass(node_vtable2))
@@ -617,8 +617,8 @@
     def test_p123_anti_nested(self):
         ops = """
         [i1, p2, p3]
-        p3sub = getfield_gc(p3, descr=nextdescr)
-        i3 = getfield_gc(p3sub, descr=valuedescr)
+        p3sub = getfield_gc_r(p3, descr=nextdescr)
+        i3 = getfield_gc_i(p3sub, descr=valuedescr)
         escape(i3)
         p1 = new_with_vtable(ConstClass(node_vtable))
         p2sub = new_with_vtable(ConstClass(node_vtable2))
@@ -635,7 +635,7 @@
     def test_keep_guard_no_exception(self):
         ops = """
         [i1]
-        i2 = call(i1, descr=nonwritedescr)
+        i2 = call_i(i1, descr=nonwritedescr)
         guard_no_exception() [i1, i2]
         jump(i2)
         """
@@ -644,13 +644,13 @@
     def test_keep_guard_no_exception_with_call_pure_that_is_not_folded(self):
         ops = """
         [i1]
-        i2 = call_pure(123456, i1, descr=nonwritedescr)
+        i2 = call_pure_i(123456, i1, descr=nonwritedescr)
         guard_no_exception() [i1, i2]
         jump(i2)
         """
         expected = """
         [i1]
-        i2 = call(123456, i1, descr=nonwritedescr)
+        i2 = call_i(123456, i1, descr=nonwritedescr)
         guard_no_exception() [i1, i2]
         jump(i2)
         """
@@ -661,8 +661,8 @@
         call_pure_results = {tuple(arg_consts): ConstInt(5)}
         ops = """
         [i1]
-        i3 = same_as(81)
-        i2 = call_pure(123456, i3, descr=nonwritedescr)
+        i3 = same_as_i(81)
+        i2 = call_pure_i(123456, i3, descr=nonwritedescr)
         guard_no_exception() [i1, i2]
         jump(i2)
         """
@@ -675,15 +675,15 @@
     def test_remove_guard_no_exception_with_duplicated_call_pure(self):
         ops = """
         [i1]
-        i2 = call_pure(123456, i1, descr=nonwritedescr)
+        i2 = call_pure_i(123456, i1, descr=nonwritedescr)
         guard_no_exception() [i1, i2]
-        i3 = call_pure(123456, i1, descr=nonwritedescr)
+        i3 = call_pure_i(123456, i1, descr=nonwritedescr)
         guard_no_exception() [i1, i2, i3]
         jump(i3)
         """
         expected = """
         [i1]
-        i2 = call(123456, i1, descr=nonwritedescr)
+        i2 = call_i(123456, i1, descr=nonwritedescr)
         guard_no_exception() [i1, i2]
         jump(i2)
         """
@@ -694,20 +694,20 @@
     def test_call_loopinvariant(self):
         ops = """
         [i1]
-        i2 = call_loopinvariant(1, i1, descr=nonwritedescr)
+        i2 = call_loopinvariant_i(1, i1, descr=nonwritedescr)
         guard_no_exception() []
         guard_value(i2, 1) []
-        i3 = call_loopinvariant(1, i1, descr=nonwritedescr)
+        i3 = call_loopinvariant_i(1, i1, descr=nonwritedescr)
         guard_no_exception() []
         guard_value(i3, 1) []
-        i4 = call_loopinvariant(1, i1, descr=nonwritedescr)
+        i4 = call_loopinvariant_i(1, i1, descr=nonwritedescr)
         guard_no_exception() []
         guard_value(i4, 1) []
         jump(i1)
         """
         expected = """
         [i1]
-        i2 = call(1, i1, descr=nonwritedescr)
+        i2 = call_i(1, i1, descr=nonwritedescr)
         guard_no_exception() []
         guard_value(i2, 1) []
         jump(i1)
@@ -847,7 +847,7 @@
         [i]
         p1 = new_with_vtable(ConstClass(node_vtable))
         setfield_gc(p1, i, descr=valuedescr)
-        i0 = getfield_gc(p1, descr=valuedescr)
+        i0 = getfield_gc_i(p1, descr=valuedescr)
         i1 = int_add(i0, 1)
         jump(i1)
         """
@@ -912,7 +912,7 @@
         [i0]
         p0 = new_with_vtable(ConstClass(node_vtable))
         setfield_gc(p0, NULL, descr=nextdescr)
-        p2 = getfield_gc(p0, descr=nextdescr)
+        p2 = getfield_gc_r(p0, descr=nextdescr)
         i1 = ptr_eq(p2, NULL)
         jump(i1)
         """
@@ -927,7 +927,7 @@
         [i0]
         p0 = new_with_vtable(ConstClass(node_vtable))
         setfield_gc(p0, ConstPtr(myptr), descr=nextdescr)
-        p2 = getfield_gc(p0, descr=nextdescr)
+        p2 = getfield_gc_r(p0, descr=nextdescr)
         i1 = ptr_eq(p2, NULL)
         jump(i1)
         """
@@ -945,11 +945,11 @@
         setinteriorfield_gc(p0, 0, f0, descr=complexrealdescr)
         setinteriorfield_gc(p0, 1, f3, descr=compleximagdescr)
         setinteriorfield_gc(p0, 1, f2, descr=complexrealdescr)
-        f4 = getinteriorfield_gc(p0, 0, descr=complexrealdescr)
-        f5 = getinteriorfield_gc(p0, 1, descr=complexrealdescr)
+        f4 = getinteriorfield_gc_f(p0, 0, descr=complexrealdescr)
+        f5 = getinteriorfield_gc_f(p0, 1, descr=complexrealdescr)
         f6 = float_mul(f4, f5)
-        f7 = getinteriorfield_gc(p0, 0, descr=compleximagdescr)
-        f8 = getinteriorfield_gc(p0, 1, descr=compleximagdescr)
+        f7 = getinteriorfield_gc_f(p0, 0, descr=compleximagdescr)
+        f8 = getinteriorfield_gc_f(p0, 1, descr=compleximagdescr)
         f9 = float_mul(f7, f8)
         f10 = float_add(f6, f9)
         finish(f10)
@@ -969,8 +969,8 @@
         p0 = new_array_clear(1, descr=complexarraydescr)
         setinteriorfield_gc(p0, 0, f0, descr=complexrealdescr)
         setinteriorfield_gc(p0, 0, f1, descr=compleximagdescr)
-        f2 = getinteriorfield_gc(p0, 0, descr=complexrealdescr)
-        f3 = getinteriorfield_gc(p0, 0, descr=compleximagdescr)
+        f2 = getinteriorfield_gc_f(p0, 0, descr=complexrealdescr)
+        f3 = getinteriorfield_gc_f(p0, 0, descr=compleximagdescr)
         f4 = float_mul(f2, f3)
         i0 = escape(f4, p0)
         finish(i0)
@@ -1004,7 +1004,7 @@
         [i]
         p1 = new_with_vtable(ConstClass(node_vtable))
         setfield_gc(p1, i, descr=valuedescr)
-        i0 = getfield_gc(p1, descr=valuedescr)
+        i0 = getfield_gc_i(p1, descr=valuedescr)
         i1 = int_add(i0, 1)
         escape(p1)
         escape(p1)
@@ -1024,7 +1024,7 @@
     def test_nonvirtual_2(self):
         ops = """
         [i, p0]
-        i0 = getfield_gc(p0, descr=valuedescr)
+        i0 = getfield_gc_i(p0, descr=valuedescr)
         escape(p0)
         i1 = int_add(i0, i)
         p1 = new_with_vtable(ConstClass(node_vtable))
@@ -1039,9 +1039,9 @@
         [i]
         p1 = new_with_vtable(ConstClass(node_vtable))
         setfield_gc(p1, i, descr=valuedescr)
-        i1 = getfield_gc(p1, descr=valuedescr)
+        i1 = getfield_gc_i(p1, descr=valuedescr)
         escape(p1)
-        i2 = getfield_gc(p1, descr=valuedescr)
+        i2 = getfield_gc_i(p1, descr=valuedescr)
         i3 = int_add(i1, i2)
         jump(i3)
         """
@@ -1050,7 +1050,7 @@
         p1 = new_with_vtable(ConstClass(node_vtable))
         setfield_gc(p1, i, descr=valuedescr)
         escape(p1)
-        i2 = getfield_gc(p1, descr=valuedescr)
+        i2 = getfield_gc_i(p1, descr=valuedescr)
         i3 = int_add(i, i2)
         jump(i3)
         """
@@ -1061,10 +1061,10 @@
         [i]
         p1 = new_with_vtable(ConstClass(node_vtable))
         setfield_gc(p1, i, descr=valuedescr)
-        i1 = getfield_gc(p1, descr=valuedescr)
+        i1 = getfield_gc_i(p1, descr=valuedescr)
         setfield_gc(p1, 0, descr=valuedescr)
         escape(p1)
-        i2 = getfield_gc(p1, descr=valuedescr)
+        i2 = getfield_gc_i(p1, descr=valuedescr)
         jump(i2)
         """
         expected = """
@@ -1072,7 +1072,7 @@
         p1 = new_with_vtable(ConstClass(node_vtable))
         setfield_gc(p1, 0, descr=valuedescr)
         escape(p1)
-        i2 = getfield_gc(p1, descr=valuedescr)
+        i2 = getfield_gc_i(p1, descr=valuedescr)
         jump(i2)
         """
         self.optimize_loop(ops, expected)
@@ -1082,7 +1082,7 @@
         [i]
         p1 = new_with_vtable(ConstClass(node_vtable))
         setfield_gc(p1, i, descr=valuedescr)
-        i1 = getfield_gc_pure(p1, descr=valuedescr)
+        i1 = getfield_gc_pure_i(p1, descr=valuedescr)
         jump(i1)
         """
         expected = """
@@ -1094,7 +1094,7 @@
     def test_getfield_gc_pure_2(self):
         ops = """
         [i]
-        i1 = getfield_gc_pure(ConstPtr(myptr), descr=valuedescr)
+        i1 = getfield_gc_pure_i(ConstPtr(myptr), descr=valuedescr)
         jump(i1)
         """
         expected = """
@@ -1107,7 +1107,7 @@
     def test_getfield_gc_nonpure_2(self):
         ops = """
         [i]
-        i1 = getfield_gc(ConstPtr(myptr), descr=valuedescr)
+        i1 = getfield_gc_i(ConstPtr(myptr), descr=valuedescr)
         jump(i1)
         """
         expected = ops
@@ -1121,7 +1121,7 @@
         guard_value(i3, 3) []
         setarrayitem_gc(p1, 1, i1, descr=arraydescr)
         setarrayitem_gc(p1, 0, 25, descr=arraydescr)
-        i2 = getarrayitem_gc(p1, 1, descr=arraydescr)
+        i2 = getarrayitem_gc_i(p1, 1, descr=arraydescr)
         jump(i2)
         """
         expected = """
@@ -1135,7 +1135,7 @@
         [i1]
         p1 = new_array(2, descr=arraydescr)
         setarrayitem_gc(p1, 0, 25, descr=arraydescr)
-        i2 = getarrayitem_gc(p1, 0, descr=arraydescr)
+        i2 = getarrayitem_gc_i(p1, 0, descr=arraydescr)
         jump(i2)
         """
         expected = """
@@ -1152,7 +1152,7 @@
         guard_value(i3, 3) []
         setarrayitem_gc(p1, 1, f1, descr=floatarraydescr)
         setarrayitem_gc(p1, 0, 3.5, descr=floatarraydescr)
-        f2 = getarrayitem_gc(p1, 1, descr=floatarraydescr)
+        f2 = getarrayitem_gc_f(p1, 1, descr=floatarraydescr)
         jump(f2)
         """
         expected = """
@@ -1220,7 +1220,7 @@
     def test_p123_array(self):
         ops = """
         [i1, p2, p3]
-        i3 = getarrayitem_gc(p3, 0, descr=arraydescr)
+        i3 = getarrayitem_gc_i(p3, 0, descr=arraydescr)
         escape(i3)
         p1 = new_array(1, descr=arraydescr)
         setarrayitem_gc(p1, 0, i1, descr=arraydescr)
@@ -1234,7 +1234,7 @@
         []
         p2 = new_with_vtable(ConstClass(node_vtable))
         setfield_gc(p2, 3, descr=valuedescr)
-        i1 = getfield_gc(p2, descr=valuedescr)    # i1 = const 3
+        i1 = getfield_gc_i(p2, descr=valuedescr)    # i1 = const 3
         p1 = new_array(i1, descr=arraydescr)
         escape(p1)
         i2 = arraylen_gc(p1)
@@ -1271,7 +1271,7 @@
     def test_p123_vstruct(self):
         ops = """
         [i1, p2, p3]
-        i3 = getfield_gc(p3, descr=adescr)
+        i3 = getfield_gc_i(p3, descr=adescr)
         escape(i3)
         p1 = new(descr=ssize)
         setfield_gc(p1, i1, descr=adescr)
@@ -1283,10 +1283,10 @@
     def test_duplicate_getfield_1(self):
         ops = """
         [p1, p2]
-        i1 = getfield_gc(p1, descr=valuedescr)
-        i2 = getfield_gc(p2, descr=valuedescr)
-        i3 = getfield_gc(p1, descr=valuedescr)
-        i4 = getfield_gc(p2, descr=valuedescr)
+        i1 = getfield_gc_i(p1, descr=valuedescr)
+        i2 = getfield_gc_i(p2, descr=valuedescr)
+        i3 = getfield_gc_i(p1, descr=valuedescr)
+        i4 = getfield_gc_i(p2, descr=valuedescr)
         escape(i1)
         escape(i2)
         escape(i3)
@@ -1295,8 +1295,8 @@
         """
         expected = """
         [p1, p2]
-        i1 = getfield_gc(p1, descr=valuedescr)
-        i2 = getfield_gc(p2, descr=valuedescr)
+        i1 = getfield_gc_i(p1, descr=valuedescr)
+        i2 = getfield_gc_i(p2, descr=valuedescr)
         escape(i1)
         escape(i2)
         escape(i1)
@@ -1309,7 +1309,7 @@
         ops = """
         [p1, i1]
         setfield_gc(p1, i1, descr=valuedescr)
-        i2 = getfield_gc(p1, descr=valuedescr)
+        i2 = getfield_gc_i(p1, descr=valuedescr)
         escape(i2)
         jump(p1, i1)
         """
@@ -1326,7 +1326,7 @@
         [p1, p2, i1]
         setfield_gc(p1, i1, descr=valuedescr)
         setfield_gc(p2, p1, descr=nextdescr)
-        i2 = getfield_gc(p1, descr=valuedescr)
+        i2 = getfield_gc_i(p1, descr=valuedescr)
         escape(i2)
         jump(p1, p2, i1)
         """
@@ -1344,7 +1344,7 @@
         [p1, p2, i1, i2]
         setfield_gc(p1, i1, descr=valuedescr)
         setfield_gc(p2, i2, descr=valuedescr)
-        i3 = getfield_gc(p1, descr=valuedescr)
+        i3 = getfield_gc_i(p1, descr=valuedescr)
         escape(i3)
         jump(p1, p2, i1, i3)
         """
@@ -1353,16 +1353,16 @@
     def test_duplicate_getfield_mergepoint_has_no_side_effects(self):
         ops = """
         [p1]
-        i1 = getfield_gc(p1, descr=valuedescr)
+        i1 = getfield_gc_i(p1, descr=valuedescr)
         debug_merge_point(15, 0)
-        i2 = getfield_gc(p1, descr=valuedescr)
+        i2 = getfield_gc_i(p1, descr=valuedescr)
         escape(i1)
         escape(i2)
         jump(p1)
         """
         expected = """
         [p1]
-        i1 = getfield_gc(p1, descr=valuedescr)
+        i1 = getfield_gc_i(p1, descr=valuedescr)
         debug_merge_point(15, 0)
         escape(i1)
         escape(i1)
@@ -1373,17 +1373,17 @@
     def test_duplicate_getfield_ovf_op_does_not_clear(self):
         ops = """
         [p1]
-        i1 = getfield_gc(p1, descr=valuedescr)
+        i1 = getfield_gc_i(p1, descr=valuedescr)
         i2 = int_add_ovf(i1, 14)
         guard_no_overflow() []
-        i3 = getfield_gc(p1, descr=valuedescr)
+        i3 = getfield_gc_i(p1, descr=valuedescr)
         escape(i2)
         escape(i3)
         jump(p1)
         """
         expected = """
         [p1]
-        i1 = getfield_gc(p1, descr=valuedescr)
+        i1 = getfield_gc_i(p1, descr=valuedescr)
         i2 = int_add_ovf(i1, 14)
         guard_no_overflow() []
         escape(i2)
@@ -1395,16 +1395,16 @@
     def test_duplicate_getfield_setarrayitem_does_not_clear(self):
         ops = """
         [p1, p2]
-        i1 = getfield_gc(p1, descr=valuedescr)
+        i1 = getfield_gc_i(p1, descr=valuedescr)
         setarrayitem_gc(p2, 0, p1, descr=arraydescr2)
-        i3 = getfield_gc(p1, descr=valuedescr)
+        i3 = getfield_gc_i(p1, descr=valuedescr)
         escape(i1)
         escape(i3)
         jump(p1, p2)
         """
         expected = """
         [p1, p2]
-        i1 = getfield_gc(p1, descr=valuedescr)
+        i1 = getfield_gc_i(p1, descr=valuedescr)
         setarrayitem_gc(p2, 0, p1, descr=arraydescr2)
         escape(i1)
         escape(i1)
@@ -1415,15 +1415,15 @@
     def test_duplicate_getfield_constant(self):
         ops = """
         []
-        i1 = getfield_gc(ConstPtr(myptr), descr=valuedescr)
-        i2 = getfield_gc(ConstPtr(myptr), descr=valuedescr)
+        i1 = getfield_gc_i(ConstPtr(myptr), descr=valuedescr)
+        i2 = getfield_gc_i(ConstPtr(myptr), descr=valuedescr)
         escape(i1)
         escape(i2)
         jump()
         """
         expected = """
         []
-        i1 = getfield_gc(ConstPtr(myptr), descr=valuedescr)
+        i1 = getfield_gc_i(ConstPtr(myptr), descr=valuedescr)
         escape(i1)
         escape(i1)
         jump()
@@ -1434,15 +1434,15 @@
         ops = """
         [p1]
         guard_value(p1, ConstPtr(myptr)) []
-        i1 = getfield_gc(p1, descr=valuedescr)
-        i2 = getfield_gc(ConstPtr(myptr), descr=valuedescr)
+        i1 = getfield_gc_i(p1, descr=valuedescr)
+        i2 = getfield_gc_i(ConstPtr(myptr), descr=valuedescr)
         escape(i1)
         escape(i2)
         jump(p1)
         """
         expected = """
         []
-        i1 = getfield_gc(ConstPtr(myptr), descr=valuedescr)
+        i1 = getfield_gc_i(ConstPtr(myptr), descr=valuedescr)
         escape(i1)
         escape(i1)
         jump()
@@ -1453,9 +1453,9 @@
     def test_duplicate_getfield_sideeffects_1(self):
         ops = """
         [p1]
-        i1 = getfield_gc(p1, descr=valuedescr)
+        i1 = getfield_gc_i(p1, descr=valuedescr)
         escape()
-        i2 = getfield_gc(p1, descr=valuedescr)
+        i2 = getfield_gc_i(p1, descr=valuedescr)
         escape(i1)
         escape(i2)
         jump(p1)
@@ -1467,7 +1467,7 @@
         [p1, i1]
         setfield_gc(p1, i1, descr=valuedescr)
         escape()
-        i2 = getfield_gc(p1, descr=valuedescr)
+        i2 = getfield_gc_i(p1, descr=valuedescr)
         escape(i2)
         jump(p1, i1)
         """
@@ -1491,7 +1491,7 @@
         ops = """
         [p1, i1, i3]
         setfield_gc(p1, i1, descr=valuedescr)
-        i2 = getfield_gc(p1, descr=valuedescr)
+        i2 = getfield_gc_i(p1, descr=valuedescr)
         setfield_gc(p1, i3, descr=valuedescr)
         escape(i2)
         jump(p1, i1, i3)
@@ -1508,7 +1508,7 @@
         ops = """
         [p1, p2, i1, i3]
         setfield_gc(p1, i1, descr=valuedescr)
-        i2 = getfield_gc(p2, descr=valuedescr)
+        i2 = getfield_gc_i(p2, descr=valuedescr)
         setfield_gc(p1, i3, descr=valuedescr)
         escape(i2)
         jump(p1, p2, i1, i3)
@@ -1523,8 +1523,8 @@
         setfield_gc(p1, i1, descr=valuedescr)
         #
         # some operations on which the above setfield_gc cannot have effect
-        i3 = getarrayitem_gc_pure(p3, 1, descr=arraydescr)
-        i4 = getarrayitem_gc(p3, i3, descr=arraydescr)
+        i3 = getarrayitem_gc_pure_i(p3, 1, descr=arraydescr)
+        i4 = getarrayitem_gc_i(p3, i3, descr=arraydescr)
         i5 = int_add(i3, i4)
         setarrayitem_gc(p3, 0, i5, descr=arraydescr)
         setfield_gc(p1, i4, descr=nextdescr)
@@ -1535,8 +1535,8 @@
         expected = """
         [p1, i1, i2, p3]
         #
-        i3 = getarrayitem_gc_pure(p3, 1, descr=arraydescr)
-        i4 = getarrayitem_gc(p3, i3, descr=arraydescr)
+        i3 = getarrayitem_gc_pure_i(p3, 1, descr=arraydescr)
+        i4 = getarrayitem_gc_i(p3, i3, descr=arraydescr)
         i5 = int_add(i3, i4)
         #
         setfield_gc(p1, i2, descr=valuedescr)
@@ -1553,8 +1553,8 @@
         setfield_gc(p1, i1, descr=valuedescr)
         setfield_gc(p0, p1, descr=nextdescr)
         setfield_raw(i1, i1, descr=valuedescr)    # random op with side-effects
-        p2 = getfield_gc(p0, descr=nextdescr)
-        i2 = getfield_gc(p2, descr=valuedescr)
+        p2 = getfield_gc_r(p0, descr=nextdescr)
+        i2 = getfield_gc_i(p2, descr=valuedescr)
         setfield_gc(p0, NULL, descr=nextdescr)
         escape(i2)
         jump(p0, i1)
@@ -1685,10 +1685,10 @@
     def test_duplicate_getarrayitem_1(self):
         ops = """
         [p1]
-        p2 = getarrayitem_gc(p1, 0, descr=arraydescr2)
-        p3 = getarrayitem_gc(p1, 1, descr=arraydescr2)
-        p4 = getarrayitem_gc(p1, 0, descr=arraydescr2)
-        p5 = getarrayitem_gc(p1, 1, descr=arraydescr2)
+        p2 = getarrayitem_gc_r(p1, 0, descr=arraydescr2)
+        p3 = getarrayitem_gc_r(p1, 1, descr=arraydescr2)
+        p4 = getarrayitem_gc_r(p1, 0, descr=arraydescr2)
+        p5 = getarrayitem_gc_r(p1, 1, descr=arraydescr2)
         escape(p2)
         escape(p3)
         escape(p4)
@@ -1697,8 +1697,8 @@
         """
         expected = """
         [p1]
-        p2 = getarrayitem_gc(p1, 0, descr=arraydescr2)
-        p3 = getarrayitem_gc(p1, 1, descr=arraydescr2)
+        p2 = getarrayitem_gc_r(p1, 0, descr=arraydescr2)
+        p3 = getarrayitem_gc_r(p1, 1, descr=arraydescr2)
         escape(p2)
         escape(p3)
         escape(p2)
@@ -1711,7 +1711,7 @@
         ops = """
         [p1, p2]
         setarrayitem_gc(p1, 0, p2, descr=arraydescr2)
-        p3 = getarrayitem_gc(p1, 0, descr=arraydescr2)
+        p3 = getarrayitem_gc_r(p1, 0, descr=arraydescr2)
         escape(p3)
         jump(p1, p3)
         """
@@ -1752,9 +1752,9 @@
         setarrayitem_gc(p1, i1, p2, descr=arraydescr2)
         setarrayitem_gc(p1, 0, p3, descr=arraydescr2)
         setarrayitem_gc(p1, 1, p4, descr=arraydescr2)
-        p5 = getarrayitem_gc(p1, i1, descr=arraydescr2)
-        p6 = getarrayitem_gc(p1, 0, descr=arraydescr2)
-        p7 = getarrayitem_gc(p1, 1, descr=arraydescr2)
+        p5 = getarrayitem_gc_r(p1, i1, descr=arraydescr2)
+        p6 = getarrayitem_gc_r(p1, 0, descr=arraydescr2)
+        p7 = getarrayitem_gc_r(p1, 1, descr=arraydescr2)
         escape(p5)
         escape(p6)
         escape(p7)
@@ -1765,7 +1765,7 @@
         setarrayitem_gc(p1, i1, p2, descr=arraydescr2)
         setarrayitem_gc(p1, 0, p3, descr=arraydescr2)
         setarrayitem_gc(p1, 1, p4, descr=arraydescr2)
-        p5 = getarrayitem_gc(p1, i1, descr=arraydescr2)
+        p5 = getarrayitem_gc_r(p1, i1, descr=arraydescr2)
         escape(p5)
         escape(p3)
         escape(p4)
@@ -1776,9 +1776,9 @@
     def test_getarrayitem_pure_does_not_invalidate(self):
         ops = """
         [p1, p2]
-        p3 = getarrayitem_gc(p1, 0, descr=arraydescr2)
-        i4 = getfield_gc_pure(ConstPtr(myptr), descr=valuedescr)
-        p5 = getarrayitem_gc(p1, 0, descr=arraydescr2)
+        p3 = getarrayitem_gc_r(p1, 0, descr=arraydescr2)
+        i4 = getfield_gc_pure_i(ConstPtr(myptr), descr=valuedescr)
+        p5 = getarrayitem_gc_r(p1, 0, descr=arraydescr2)
         escape(p3)
         escape(i4)
         escape(p5)
@@ -1786,7 +1786,7 @@
         """
         expected = """
         [p1, p2]
-        p3 = getarrayitem_gc(p1, 0, descr=arraydescr2)
+        p3 = getarrayitem_gc_r(p1, 0, descr=arraydescr2)
         escape(p3)
         escape(5)
         escape(p3)
@@ -1799,8 +1799,8 @@
         [p1, p2, p3, p4, i1]
         setarrayitem_gc(p1, 0, p3, descr=arraydescr2)
         setarrayitem_gc(p2, 1, p4, descr=arraydescr2)
-        p5 = getarrayitem_gc(p1, 0, descr=arraydescr2)
-        p6 = getarrayitem_gc(p2, 1, descr=arraydescr2)
+        p5 = getarrayitem_gc_r(p1, 0, descr=arraydescr2)
+        p6 = getarrayitem_gc_r(p2, 1, descr=arraydescr2)
         escape(p5)
         escape(p6)
         jump(p1, p2, p3, p4, i1)
@@ -1821,7 +1821,7 @@
         setarrayitem_gc(p0, 0, i0, descr=arraydescr)
         i6 = int_add(i0, 1)
         setarrayitem_gc(p0, i1, i6, descr=arraydescr)
-        i10 = getarrayitem_gc(p0, 0, descr=arraydescr)
+        i10 = getarrayitem_gc_i(p0, 0, descr=arraydescr)
         i11 = int_add(i10, i0)
         jump(p0, i11, i1)
         """
@@ -1830,7 +1830,7 @@
         i6 = int_add(i0, 1)
         setarrayitem_gc(p0, 0, i0, descr=arraydescr)
         setarrayitem_gc(p0, i1, i6, descr=arraydescr)
-        i10 = getarrayitem_gc(p0, 0, descr=arraydescr)
+        i10 = getarrayitem_gc_i(p0, 0, descr=arraydescr)
         i11 = int_add(i10, i0)
         jump(p0, i11, i1)
         """
@@ -1839,19 +1839,19 @@
     def test_duplicate_getarrayitem_after_setarrayitem_bug2(self):
         ops = """
         [p0, i0, i1]
-        i2 = getarrayitem_gc(p0, 0, descr=arraydescr)
+        i2 = getarrayitem_gc_i(p0, 0, descr=arraydescr)
         i6 = int_add(i0, 1)
         setarrayitem_gc(p0, i1, i6, descr=arraydescr)
-        i10 = getarrayitem_gc(p0, 0, descr=arraydescr)
+        i10 = getarrayitem_gc_i(p0, 0, descr=arraydescr)
         i11 = int_add(i10, i2)
         jump(p0, i11, i1)
         """
         expected = """
         [p0, i0, i1]
-        i2 = getarrayitem_gc(p0, 0, descr=arraydescr)
+        i2 = getarrayitem_gc_i(p0, 0, descr=arraydescr)
         i6 = int_add(i0, 1)
         setarrayitem_gc(p0, i1, i6, descr=arraydescr)
-        i10 = getarrayitem_gc(p0, 0, descr=arraydescr)
+        i10 = getarrayitem_gc_i(p0, 0, descr=arraydescr)
         i11 = int_add(i10, i2)
         jump(p0, i11, i1)
         """
@@ -1860,7 +1860,7 @@
     def test_bug_1(self):
         ops = """
         [i0, p1]
-        p4 = getfield_gc(p1, descr=nextdescr)
+        p4 = getfield_gc_r(p1, descr=nextdescr)
         guard_nonnull(p4) []
         escape(p4)
         #
@@ -1910,10 +1910,10 @@
         [p1]
         guard_nonnull(p1) []
         guard_class(p1, ConstClass(node_vtable2)) []
-        p2 = getfield_gc(p1, descr=nextdescr)
+        p2 = getfield_gc_r(p1, descr=nextdescr)
         guard_nonnull(12) []
         guard_class(p2, ConstClass(node_vtable)) []
-        p3 = getfield_gc(p1, descr=otherdescr)
+        p3 = getfield_gc_r(p1, descr=otherdescr)
         guard_nonnull(12) []
         guard_class(p3, ConstClass(node_vtable)) []
         setfield_gc(p3, p2, descr=otherdescr)
@@ -1943,10 +1943,10 @@
         [p1]
         guard_nonnull(p1) []
         guard_class(p1, ConstClass(node_vtable2)) []
-        p2 = getfield_gc(p1, descr=nextdescr)
+        p2 = getfield_gc_r(p1, descr=nextdescr)
         guard_nonnull(12) []
         guard_class(p2, ConstClass(node_vtable)) []
-        p3 = getfield_gc(p1, descr=otherdescr)
+        p3 = getfield_gc_r(p1, descr=otherdescr)
         guard_nonnull(12) []
         guard_class(p3, ConstClass(node_vtable)) []
         p1a = new_with_vtable(ConstClass(node_vtable2))
@@ -1999,7 +1999,7 @@
     def test_invalid_loop_3(self):
         ops = """
         [p1]
-        p2 = getfield_gc(p1, descr=nextdescr)
+        p2 = getfield_gc_r(p1, descr=nextdescr)
         guard_isnull(p2) []
         #
         p3 = new_with_vtable(ConstClass(node_vtable))
@@ -2100,8 +2100,8 @@
     def test_oois_of_itself(self):
         ops = """
         [p0]
-        p1 = getfield_gc(p0, descr=nextdescr)
-        p2 = getfield_gc(p0, descr=nextdescr)
+        p1 = getfield_gc_r(p0, descr=nextdescr)
+        p2 = getfield_gc_r(p0, descr=nextdescr)
         i1 = ptr_eq(p1, p2)
         guard_true(i1) []
         i2 = ptr_ne(p1, p2)
@@ -2110,7 +2110,7 @@
         """
         expected = """
         [p0]
-        p1 = getfield_gc(p0, descr=nextdescr)
+        p1 = getfield_gc_r(p0, descr=nextdescr)
         jump(p0)
         """
         self.optimize_loop(ops, expected)
@@ -2501,7 +2501,7 @@
         # first rename i3 into i4
         p1 = new_with_vtable(ConstClass(node_vtable))
         setfield_gc(p1, i3, descr=valuedescr)
-        i4 = getfield_gc(p1, descr=valuedescr)
+        i4 = getfield_gc_i(p1, descr=valuedescr)
         #
         i2 = int_add(10, 5)
         guard_true(i1, descr=fdescr) [i2, i4]
@@ -2565,7 +2565,7 @@
             [i1, i2, i3]
             p1 = new_with_vtable(ConstClass(node_vtable))
             setfield_gc(p1, i3, descr=valuedescr)
-            i4 = getfield_gc(p1, descr=valuedescr)   # copy of i3
+            i4 = getfield_gc_i(p1, descr=valuedescr)   # copy of i3
             p2 = new_with_vtable(ConstClass(node_vtable))
             setfield_gc(p1, i2, descr=valuedescr)
             setfield_gc(p1, p2, descr=nextdescr)
@@ -2654,8 +2654,8 @@
         setfield_gc(p2, i1, descr=adescr)
         setfield_gc(p2, p1, descr=bdescr)
         guard_true(i1, descr=fdescr) [p2]
-        i3 = getfield_gc(p2, descr=adescr)
-        p3 = getfield_gc(p2, descr=bdescr)
+        i3 = getfield_gc_i(p2, descr=adescr)
+        p3 = getfield_gc_r(p2, descr=bdescr)
         jump(i3, p3)
         """
         expected = """
@@ -2672,7 +2672,7 @@
         ops = """
         [i1, p1a, i2]
         p6s = getarrayitem_gc(p1a, 0, descr=arraydescr2)
-        p7v = getfield_gc(p6s, descr=bdescr)
+        p7v = getfield_gc_r(p6s, descr=bdescr)
         p5s = new(descr=ssize)
         setfield_gc(p5s, i2, descr=adescr)
         setfield_gc(p5s, p7v, descr=bdescr)
@@ -2761,16 +2761,16 @@
     def test_residual_call_does_not_invalidate_caches(self):
         ops = """
         [p1, p2]
-        i1 = getfield_gc(p1, descr=valuedescr)
+        i1 = getfield_gc_i(p1, descr=valuedescr)
         i2 = call(i1, descr=nonwritedescr)
-        i3 = getfield_gc(p1, descr=valuedescr)
+        i3 = getfield_gc_i(p1, descr=valuedescr)
         escape(i1)
         escape(i3)
         jump(p1, p2)
         """
         expected = """
         [p1, p2]
-        i1 = getfield_gc(p1, descr=valuedescr)
+        i1 = getfield_gc_i(p1, descr=valuedescr)
         i2 = call(i1, descr=nonwritedescr)
         escape(i1)
         escape(i1)
@@ -2781,11 +2781,11 @@
     def test_residual_call_invalidate_some_caches(self):
         ops = """
         [p1, p2]
-        i1 = getfield_gc(p1, descr=adescr)
-        i2 = getfield_gc(p1, descr=bdescr)
+        i1 = getfield_gc_i(p1, descr=adescr)
+        i2 = getfield_gc_i(p1, descr=bdescr)
         i3 = call(i1, descr=writeadescr)
-        i4 = getfield_gc(p1, descr=adescr)
-        i5 = getfield_gc(p1, descr=bdescr)
+        i4 = getfield_gc_i(p1, descr=adescr)
+        i5 = getfield_gc_i(p1, descr=bdescr)
         escape(i1)
         escape(i2)
         escape(i4)
@@ -2794,10 +2794,10 @@
         """
         expected = """
         [p1, p2]
-        i1 = getfield_gc(p1, descr=adescr)
-        i2 = getfield_gc(p1, descr=bdescr)
+        i1 = getfield_gc_i(p1, descr=adescr)
+        i2 = getfield_gc_i(p1, descr=bdescr)
         i3 = call(i1, descr=writeadescr)
-        i4 = getfield_gc(p1, descr=adescr)
+        i4 = getfield_gc_i(p1, descr=adescr)
         escape(i1)
         escape(i2)
         escape(i4)
@@ -3708,7 +3708,7 @@
     def test_framestackdepth_overhead(self):
         ops = """
         [p0, i22]
-        i1 = getfield_gc(p0, descr=valuedescr)
+        i1 = getfield_gc_i(p0, descr=valuedescr)
         i2 = int_gt(i1, i22)
         guard_false(i2) []
         i3 = int_add(i1, 1)
@@ -3725,7 +3725,7 @@
         """
         expected = """
         [p0, i22]
-        i1 = getfield_gc(p0, descr=valuedescr)
+        i1 = getfield_gc_i(p0, descr=valuedescr)
         i2 = int_gt(i1, i22)
         guard_false(i2) []
         i3 = int_add(i1, 1)
@@ -5006,7 +5006,7 @@
         escape(p1)
         setfield_gc(p0, i0, descr=adescr)
         setfield_gc(p1, i1, descr=adescr)
-        i2 = getfield_gc(p0, descr=adescr)
+        i2 = getfield_gc_i(p0, descr=adescr)
         jump(i2, i2)
         """
         expected = """
@@ -5248,35 +5248,35 @@
     def test_getfield_cmp_above_bounds(self):
         ops = """
         [p0]
-        i0 = getfield_gc(p0, descr=chardescr)
+        i0 = getfield_gc_i(p0, descr=chardescr)
         i1 = int_lt(i0, 256)
         guard_true(i1) []
         """
 
         expected = """
         [p0]
-        i0 = getfield_gc(p0, descr=chardescr)
+        i0 = getfield_gc_i(p0, descr=chardescr)
         """
         self.optimize_loop(ops, expected)
 
     def test_getfield_cmp_below_bounds(self):
         ops = """
         [p0]
-        i0 = getfield_gc(p0, descr=chardescr)
+        i0 = getfield_gc_i(p0, descr=chardescr)
         i1 = int_gt(i0, -1)
         guard_true(i1) []
         """
 
         expected = """
         [p0]
-        i0 = getfield_gc(p0, descr=chardescr)
+        i0 = getfield_gc_i(p0, descr=chardescr)
         """
         self.optimize_loop(ops, expected)
 
     def test_getfield_cmp_in_bounds(self):
         ops = """
         [p0]
-        i0 = getfield_gc(p0, descr=chardescr)
+        i0 = getfield_gc_i(p0, descr=chardescr)
         i1 = int_gt(i0, 0)
         guard_true(i1) []
         i2 = int_lt(i0, 255)
diff --git a/rpython/jit/metainterp/optimizeopt/virtualize.py b/rpython/jit/metainterp/optimizeopt/virtualize.py
--- a/rpython/jit/metainterp/optimizeopt/virtualize.py
+++ b/rpython/jit/metainterp/optimizeopt/virtualize.py
@@ -28,7 +28,7 @@
 
     def get_key_box(self):
         if self.box is None:
-            return self.keybox
+            return self.source_op
         return self.box
 
     def force_box(self, optforce):
@@ -157,8 +157,7 @@
                 iteritems.sort(key=lambda (x, y): x.sort_key())
             for ofs, value in iteritems:
                 subbox = value.force_box(optforce)
-                op = ResOperation(rop.SETFIELD_GC, [box, subbox], None,
-                                  descr=ofs)
+                op = ResOperation(rop.SETFIELD_GC, [box, subbox], descr=ofs)
                 optforce.emit_operation(op)
 
     def _get_field_descr_list(self):
@@ -220,8 +219,8 @@
 
 class VStructValue(AbstractVirtualStructValue):
 
-    def __init__(self, cpu, structdescr, keybox, source_op=None):
-        AbstractVirtualStructValue.__init__(self, cpu, keybox, source_op)
+    def __init__(self, cpu, structdescr, source_op):
+        AbstractVirtualStructValue.__init__(self, cpu, source_op)
         self.structdescr = structdescr
 
     @specialize.argtype(1)
@@ -265,9 +264,9 @@
 
 class VArrayValue(AbstractVArrayValue):
 
-    def __init__(self, arraydescr, constvalue, size, keybox, source_op=None,
+    def __init__(self, arraydescr, constvalue, size, source_op,
                  clear=False):
-        AbstractVirtualValue.__init__(self, keybox, source_op)
+        AbstractVirtualValue.__init__(self, source_op)
         self.arraydescr = arraydescr
         self.constvalue = constvalue
         if clear:
@@ -332,7 +331,7 @@
         # * if source_op is NEW_ARRAY, emit NEW_ARRAY_CLEAR if it's
         #   followed by setting most items to zero anyway
         optforce.emit_operation(self.source_op)
-        self.box = box = self.source_op.result
+        self.box = box = self.source_op
         for index in range(len(self._items)):
             subvalue = self._items[index]
             if subvalue is None:
@@ -342,7 +341,7 @@
                     continue
             subbox = subvalue.force_box(optforce)
             op = ResOperation(rop.SETARRAYITEM_GC,
-                              [box, ConstInt(index), subbox], None,
+                              [box, ConstInt(index), subbox],
                                descr=self.arraydescr)
             optforce.emit_operation(op)
 
@@ -352,8 +351,8 @@
 
 
 class VArrayStructValue(AbstractVirtualValue):
-    def __init__(self, arraydescr, size, keybox, source_op=None):
-        AbstractVirtualValue.__init__(self, keybox, source_op)
+    def __init__(self, arraydescr, size, source_op):
+        AbstractVirtualValue.__init__(self, source_op)
         self.arraydescr = arraydescr
         self._items = [{} for _ in xrange(size)]
 
@@ -372,7 +371,7 @@
         if not we_are_translated():
             self.source_op.name = 'FORCE ' + self.source_op.name
         optforce.emit_operation(self.source_op)
-        self.box = box = self.source_op.result
+        self.box = box = self.source_op
         for index in range(len(self._items)):
             iteritems = self._items[index].iteritems()
             # random order is fine, except for tests
@@ -382,7 +381,7 @@
             for descr, value in iteritems:
                 subbox = value.force_box(optforce)
                 op = ResOperation(rop.SETINTERIORFIELD_GC,
-                    [box, ConstInt(index), subbox], None, descr=descr
+                    [box, ConstInt(index), subbox], descr=descr
                 )
                 optforce.emit_operation(op)
 
@@ -460,7 +459,7 @@
         if not we_are_translated():
             op.name = 'FORCE ' + self.source_op.name
         optforce.emit_operation(self.source_op)
-        self.box = self.source_op.result
+        self.box = self.source_op
         for i in range(len(self.buffer.offsets)):
             # write the value
             offset = self.buffer.offsets[i]
@@ -495,7 +494,7 @@
         assert op is not None
         if not we_are_translated():
             op.name = 'FORCE ' + self.source_op.name
-        self.box = self.source_op.result
+        self.box = self.source_op
         self.rawbuffer_value.force_box(optforce)
         optforce.emit_operation(op)
 
@@ -525,21 +524,20 @@
         self.make_equal_to(source_op, vvalue)
         return vvalue
 
-    def make_varray(self, arraydescr, size, box, source_op=None,
-                    clear=False):
+    def make_varray(self, arraydescr, size, source_op, clear=False):
         if arraydescr.is_array_of_structs():
             assert clear
-            vvalue = VArrayStructValue(arraydescr, size, box, source_op)
+            vvalue = VArrayStructValue(arraydescr, size, source_op)
         else:
             constvalue = self.new_const_item(arraydescr)
-            vvalue = VArrayValue(arraydescr, constvalue, size, box, source_op,
+            vvalue = VArrayValue(arraydescr, constvalue, size, source_op,
                                  clear=clear)
-        self.make_equal_to(box, vvalue)
+        self.make_equal_to(source_op, vvalue)
         return vvalue
 
-    def make_vstruct(self, structdescr, box, source_op=None):
-        vvalue = VStructValue(self.optimizer.cpu, structdescr, box, source_op)
-        self.make_equal_to(box, vvalue)
+    def make_vstruct(self, structdescr, source_op):
+        vvalue = VStructValue(self.optimizer.cpu, structdescr, source_op)
+        self.make_equal_to(source_op, vvalue)
         return vvalue
 
     def make_virtual_raw_memory(self, size, box, source_op):
@@ -607,8 +605,8 @@
         # Replace the VIRTUAL_REF operation with a virtual structure of type
         # 'jit_virtual_ref'.  The jit_virtual_ref structure may be forced soon,
         # but the point is that doing so does not force the original structure.
-        op = ResOperation(rop.NEW_WITH_VTABLE, [c_cls], op.result)
-        vrefvalue = self.make_virtual(c_cls, op.result, op)
+        op = ResOperation(rop.NEW_WITH_VTABLE, [c_cls], op)
+        vrefvalue = self.make_virtual(c_cls, op)
         tokenbox = BoxPtr()
         self.emit_operation(ResOperation(rop.FORCE_TOKEN, [], tokenbox))
         vrefvalue.setfield(descr_virtual_token, self.getvalue(tokenbox))
@@ -657,7 +655,7 @@
                     not tokenvalue.box.nonnull()):
                 forcedvalue = vref.getfield(vrefinfo.descr_forced, None)
                 if forcedvalue is not None and not forcedvalue.is_null():
-                    self.make_equal_to(op.result, forcedvalue)
+                    self.make_equal_to(op, forcedvalue)
                     self.last_emitted_operation = REMOVED
                     return True
         return False
@@ -670,14 +668,14 @@
         if value.is_forced_virtual() and op.is_always_pure():
             fieldvalue = value.getfield(op.getdescr(), None)
             if fieldvalue is not None:
-                self.make_equal_to(op.result, fieldvalue)
+                self.make_equal_to(op, fieldvalue)
                 return
         if value.is_virtual():
             assert isinstance(value, AbstractVirtualValue)
             fieldvalue = value.getfield(op.getdescr(), None)
             if fieldvalue is None:
                 fieldvalue = self.optimizer.new_const(op.getdescr())
-            self.make_equal_to(op.result, fieldvalue)
+            self.make_equal_to(op, fieldvalue)
         else:
             value.ensure_nonnull()
             self.emit_operation(op)
@@ -708,14 +706,14 @@
     def optimize_NEW_ARRAY(self, op):
         sizebox = self.get_constant_box(op.getarg(0))
         if sizebox is not None:
-            self.make_varray(op.getdescr(), sizebox.getint(), op.result, op)
+            self.make_varray(op.getdescr(), sizebox.getint(), op)
         else:
             self.emit_operation(op)
 
     def optimize_NEW_ARRAY_CLEAR(self, op):
         sizebox = self.get_constant_box(op.getarg(0))
         if sizebox is not None:
-            self.make_varray(op.getdescr(), sizebox.getint(), op.result, op,
+            self.make_varray(op.getdescr(), sizebox.getint(), op,
                              clear=True)
         else:
             self.emit_operation(op)        
@@ -740,7 +738,7 @@
         if sizebox is None:
             self.emit_operation(op)
             return
-        self.make_virtual_raw_memory(sizebox.getint(), op.result, op)
+        self.make_virtual_raw_memory(sizebox.getint(), op)
         self.last_emitted_operation = REMOVED
 
     def do_RAW_FREE(self, op):
@@ -758,19 +756,19 @@
             # translation occurs without any VRawXxxValue instance around
             if value.is_about_raw:
                 if isinstance(value, VRawBufferValue):
-                    self.make_virtual_raw_slice(value, offset, op.result, op)
+                    self.make_virtual_raw_slice(value, offset, op)
                     return
                 elif isinstance(value, VRawSliceValue):
                     offset = offset + value.offset
                     self.make_virtual_raw_slice(value.rawbuffer_value, offset,
-                                                op.result, op)
+                                                op)
                     return
         self.emit_operation(op)
 
     def optimize_ARRAYLEN_GC(self, op):
         value = self.getvalue(op.getarg(0))
         if value.is_virtual():
-            self.make_constant_int(op.result, value.getlength())
+            self.make_constant_int(op, value.getlength())
         else:
             value.ensure_nonnull()
             self.emit_operation(op)
@@ -784,7 +782,7 @@
                 itemvalue = value.getitem(indexbox.getint())
                 if itemvalue is None:   # reading uninitialized array items?
                     itemvalue = value.constvalue     # bah, just return 0
-                self.make_equal_to(op.result, itemvalue)
+                self.make_equal_to(op, itemvalue)
                 return
         value.ensure_nonnull()
         self.emit_operation(op)
@@ -826,7 +824,7 @@
                 except InvalidRawOperation:
                     pass
                 else:
-                    self.make_equal_to(op.result, itemvalue)
+                    self.make_equal_to(op, itemvalue)
                     return
         value.ensure_nonnull()
         self.emit_operation(op)
@@ -865,7 +863,7 @@
                 except InvalidRawOperation:
                     pass
                 else:
-                    self.make_equal_to(op.result, itemvalue)
+                    self.make_equal_to(op, itemvalue)
                     return
         value.ensure_nonnull()
         self.emit_operation(op)
@@ -897,7 +895,7 @@
                 )
                 if fieldvalue is None:
                     fieldvalue = self.new_const(descr)
-                self.make_equal_to(op.result, fieldvalue)
+                self.make_equal_to(op, fieldvalue)
                 return
         value.ensure_nonnull()
         self.emit_operation(op)
diff --git a/rpython/jit/tool/oparser.py b/rpython/jit/tool/oparser.py
--- a/rpython/jit/tool/oparser.py
+++ b/rpython/jit/tool/oparser.py
@@ -16,6 +16,8 @@
 
     OPNUM = -123
 
+    type = 'i'
+
     def getopnum(self):
         return self.OPNUM
 

From noreply at buildbot.pypy.org  Fri Nov 14 19:06:34 2014
From: noreply at buildbot.pypy.org (rlamy)
Date: Fri, 14 Nov 2014 19:06:34 +0100 (CET)
Subject: [pypy-commit] pypy ssa-flow: transform_dead_op_vars() is compatible
	with SSA
Message-ID: <20141114180634.A48221D288E@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy 
Branch: ssa-flow
Changeset: r74524:c18905e4bc4f
Date: 2014-11-14 04:05 +0000
http://bitbucket.org/pypy/pypy/changeset/c18905e4bc4f/

Log:	transform_dead_op_vars() is compatible with SSA

diff --git a/rpython/flowspace/test/test_objspace.py b/rpython/flowspace/test/test_objspace.py
--- a/rpython/flowspace/test/test_objspace.py
+++ b/rpython/flowspace/test/test_objspace.py
@@ -1291,6 +1291,17 @@
         assert isinstance(link.args[0], Constant)
         assert link.args[0].value == 5
 
+    def test_remove_dead_ops(self):
+        def f():
+            a = [1]
+            b = (a, a)
+            c = type(b)
+        graph = self.codetest(f)
+        simplify_graph(graph)
+        assert graph.startblock.operations == []
+        [link] = graph.startblock.exits
+        assert link.target is graph.returnblock
+
 
 DATA = {'x': 5,
         'y': 6}
diff --git a/rpython/translator/simplify.py b/rpython/translator/simplify.py
--- a/rpython/translator/simplify.py
+++ b/rpython/translator/simplify.py
@@ -1002,9 +1002,10 @@
     eliminate_empty_blocks,
     remove_assertion_errors,
     remove_trivial_links,
-    SSA_to_SSI,
     coalesce_bool,
     transform_dead_op_vars,
+    eliminate_empty_blocks,
+    SSA_to_SSI,
     remove_identical_vars,
     transform_ovfcheck,
     simplify_exceptions,

From noreply at buildbot.pypy.org  Fri Nov 14 19:06:36 2014
From: noreply at buildbot.pypy.org (rlamy)
Date: Fri, 14 Nov 2014 19:06:36 +0100 (CET)
Subject: [pypy-commit] pypy ssa-flow: transform_dead_op_vars earlier
Message-ID: <20141114180636.0A6611D288E@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy 
Branch: ssa-flow
Changeset: r74525:59a2022d0bd0
Date: 2014-11-14 04:27 +0000
http://bitbucket.org/pypy/pypy/changeset/59a2022d0bd0/

Log:	transform_dead_op_vars earlier

diff --git a/rpython/translator/simplify.py b/rpython/translator/simplify.py
--- a/rpython/translator/simplify.py
+++ b/rpython/translator/simplify.py
@@ -999,12 +999,11 @@
 # ____ all passes & simplify_graph
 
 all_passes = [
+    transform_dead_op_vars,
     eliminate_empty_blocks,
     remove_assertion_errors,
     remove_trivial_links,
     coalesce_bool,
-    transform_dead_op_vars,
-    eliminate_empty_blocks,
     SSA_to_SSI,
     remove_identical_vars,
     transform_ovfcheck,

From noreply at buildbot.pypy.org  Fri Nov 14 19:06:37 2014
From: noreply at buildbot.pypy.org (rlamy)
Date: Fri, 14 Nov 2014 19:06:37 +0100 (CET)
Subject: [pypy-commit] pypy ssa-flow: optimise remove_trivial_links()
Message-ID: <20141114180637.3CF541D288E@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy 
Branch: ssa-flow
Changeset: r74526:5ad8e04f2740
Date: 2014-11-14 04:48 +0000
http://bitbucket.org/pypy/pypy/changeset/5ad8e04f2740/

Log:	optimise remove_trivial_links()

diff --git a/rpython/flowspace/model.py b/rpython/flowspace/model.py
--- a/rpython/flowspace/model.py
+++ b/rpython/flowspace/model.py
@@ -206,8 +206,6 @@
         return uniqueitems([w for w in result if isinstance(w, Constant)])
 
     def renamevariables(self, mapping):
-        for a in mapping:
-            assert isinstance(a, Variable), a
         self.inputargs = [mapping.get(a, a) for a in self.inputargs]
         for op in self.operations:
             op.args = [mapping.get(a, a) for a in op.args]
diff --git a/rpython/translator/simplify.py b/rpython/translator/simplify.py
--- a/rpython/translator/simplify.py
+++ b/rpython/translator/simplify.py
@@ -270,10 +270,15 @@
                 while vprev in renaming:
                     vprev = renaming[vprev]
                 renaming[vtarg] = vprev
-            target.renamevariables(renaming)
-            source.operations.extend(target.operations)
-            source.exitswitch = newexitswitch = target.exitswitch
-            exits = target.exits
+            def rename(v):
+                return renaming.get(v, v)
+            for op in target.operations:
+                source.operations.append(op.replace(renaming))
+            source.exitswitch = newexitswitch = rename(target.exitswitch)
+            exits = []
+            for exit in target.exits:
+                newexit = exit.copy(rename)
+                exits.append(newexit)
             source.recloseblock(*exits)
             if isinstance(newexitswitch, Constant) and newexitswitch != c_last_exception:
                 exits = replace_exitswitch_by_constant(source, newexitswitch)

From noreply at buildbot.pypy.org  Fri Nov 14 22:30:07 2014
From: noreply at buildbot.pypy.org (rlamy)
Date: Fri, 14 Nov 2014 22:30:07 +0100 (CET)
Subject: [pypy-commit] pypy ssa-flow: begin adapting remove_identical_vars()
	to SSA
Message-ID: <20141114213007.87ED71C32A5@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy 
Branch: ssa-flow
Changeset: r74527:2467d09a6d1e
Date: 2014-11-14 21:29 +0000
http://bitbucket.org/pypy/pypy/changeset/2467d09a6d1e/

Log:	begin adapting remove_identical_vars() to SSA

diff --git a/rpython/translator/simplify.py b/rpython/translator/simplify.py
--- a/rpython/translator/simplify.py
+++ b/rpython/translator/simplify.py
@@ -7,14 +7,15 @@
 import py
 from collections import defaultdict
 
+from rpython.tool.algo.unionfind import UnionFind
 from rpython.flowspace.model import (Variable, Constant,
                                      c_last_exception, checkgraph, mkentrymap)
 from rpython.flowspace.operation import OverflowingOperation, op
 from rpython.rlib import rarithmetic
 from rpython.translator import unsimplify
-from rpython.translator.backendopt import ssa
 from rpython.rtyper.lltypesystem import lloperation, lltype
-from rpython.translator.backendopt.ssa import SSA_to_SSI
+from rpython.translator.backendopt.ssa import (
+        SSA_to_SSI, DataFlowFamilyBuilder)
 
 def get_graph(arg, translator):
     if isinstance(arg, Variable):
@@ -550,6 +551,66 @@
             if block.inputargs[i] not in read_vars:
                 del block.inputargs[i]
 
+class Representative(object):
+    def __init__(self, var):
+        self.rep = var
+
+    def absorb(self, other):
+        pass
+
+def all_equal(lst):
+    first = lst[0]
+    return all(first == x for x in lst[1:])
+
+def remove_identical_vars_SSA(graph):
+    """When the same variable is passed multiple times into the next block,
+    pass it only once.  This enables further optimizations by the annotator,
+    which otherwise doesn't realize that tests performed on one of the copies
+    of the variable also affect the other."""
+    uf = UnionFind(Representative)
+    entrymap = mkentrymap(graph)
+    del entrymap[graph.startblock]
+    entrymap.pop(graph.returnblock, None)
+    entrymap.pop(graph.exceptblock, None)
+    inputs = {}
+    for block, links in entrymap.items():
+        phis = zip(block.inputargs, zip(*[link.args for link in links]))
+        inputs[block] = phis
+
+    def trivial_phis(block):
+        phis = inputs[block]
+        to_remove = []
+        for i, (input, phi_args) in enumerate(phis):
+            new_args = [uf.find_rep(arg) for arg in phi_args]
+            if all_equal(new_args):
+                uf.union(new_args[0], input)
+                to_remove.append(i)
+        for i in reversed(to_remove):
+            del phis[i]
+        return bool(to_remove)
+
+    progress = True
+    while progress:
+        progress = False
+        for block in inputs:
+            if trivial_phis(block):
+                progress = True
+
+    renaming = {key: uf[key].rep for key in uf}
+    for block, links in entrymap.items():
+        if inputs[block]:
+            new_inputs, new_args = zip(*inputs[block])
+            new_args = map(list, zip(*new_args))
+        else:
+            new_inputs = []
+            new_args = [[] for _ in links]
+        block.inputargs = new_inputs
+        assert len(links) == len(new_args)
+        for link, args in zip(links, new_args):
+            link.args = args
+    for block in graph.iterblocks():
+        block.renamevariables(renaming)
+
 def remove_identical_vars(graph):
     """When the same variable is passed multiple times into the next block,
     pass it only once.  This enables further optimizations by the annotator,
@@ -1009,8 +1070,8 @@
     remove_assertion_errors,
     remove_trivial_links,
     coalesce_bool,
+    remove_identical_vars_SSA,
     SSA_to_SSI,
-    remove_identical_vars,
     transform_ovfcheck,
     simplify_exceptions,
     transform_xxxitem,

From noreply at buildbot.pypy.org  Sat Nov 15 01:43:50 2014
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Sat, 15 Nov 2014 01:43:50 +0100 (CET)
Subject: [pypy-commit] pypy default: Optimize mmap slicing with step != 1
Message-ID: <20141115004350.723301C093D@cobra.cs.uni-duesseldorf.de>

Author: Alex Gaynor 
Branch: 
Changeset: r74528:9e48410d1e31
Date: 2014-11-14 16:43 -0800
http://bitbucket.org/pypy/pypy/changeset/9e48410d1e31/

Log:	Optimize mmap slicing with step != 1

diff --git a/pypy/module/mmap/interp_mmap.py b/pypy/module/mmap/interp_mmap.py
--- a/pypy/module/mmap/interp_mmap.py
+++ b/pypy/module/mmap/interp_mmap.py
@@ -5,6 +5,7 @@
 from rpython.rlib import rmmap, rarithmetic
 from rpython.rlib.buffer import Buffer
 from rpython.rlib.rmmap import RValueError, RTypeError, RMMapError
+from rpython.rlib.rstring import StringBuilder
 
 if rmmap.HAVE_LARGEFILE_SUPPORT:
     OFF_T = rarithmetic.r_longlong
@@ -163,17 +164,18 @@
         self.check_valid()
 
         space = self.space
-        start, stop, step = space.decode_index(w_index, self.mmap.size)
+        start, stop, step, length = space.decode_index4(w_index, self.mmap.size)
         if step == 0:  # index only
             return space.wrap(self.mmap.getitem(start))
         elif step == 1:
             if stop - start < 0:
                 return space.wrap("")
-            return space.wrap(self.mmap.getslice(start, stop - start))
+            return space.wrap(self.mmap.getslice(start, length))
         else:
-            res = "".join([self.mmap.getitem(i)
-                           for i in range(start, stop, step)])
-            return space.wrap(res)
+            b = StringBuilder(length)
+            for i in range(start, stop, step):
+                b.append(self.mmap.getitem(i))
+            return space.wrap(b.build())
 
     def descr_setitem(self, w_index, w_value):
         space = self.space

From noreply at buildbot.pypy.org  Sat Nov 15 02:39:53 2014
From: noreply at buildbot.pypy.org (rlamy)
Date: Sat, 15 Nov 2014 02:39:53 +0100 (CET)
Subject: [pypy-commit] pypy ssa-flow: fix remove_identical_vars_SSA()
Message-ID: <20141115013953.78D501C0548@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy 
Branch: ssa-flow
Changeset: r74529:18a5e66199ad
Date: 2014-11-15 01:39 +0000
http://bitbucket.org/pypy/pypy/changeset/18a5e66199ad/

Log:	fix remove_identical_vars_SSA()

diff --git a/rpython/flowspace/test/test_objspace.py b/rpython/flowspace/test/test_objspace.py
--- a/rpython/flowspace/test/test_objspace.py
+++ b/rpython/flowspace/test/test_objspace.py
@@ -1302,6 +1302,16 @@
         [link] = graph.startblock.exits
         assert link.target is graph.returnblock
 
+    def test_not_combine(self):
+        def f(n):
+            t = not n
+            if not n:
+                t += 1
+            return t
+        graph = self.codetest(f)
+        simplify_graph(graph)
+        assert self.all_operations(graph) == {'bool': 1, 'inplace_add': 1}
+
 
 DATA = {'x': 5,
         'y': 6}
diff --git a/rpython/translator/simplify.py b/rpython/translator/simplify.py
--- a/rpython/translator/simplify.py
+++ b/rpython/translator/simplify.py
@@ -562,6 +562,9 @@
     first = lst[0]
     return all(first == x for x in lst[1:])
 
+def isspecialvar(v):
+    return isinstance(v, Variable) and v._name in ('last_exception_', 'last_exc_value_')
+
 def remove_identical_vars_SSA(graph):
     """When the same variable is passed multiple times into the next block,
     pass it only once.  This enables further optimizations by the annotator,
@@ -577,14 +580,22 @@
         phis = zip(block.inputargs, zip(*[link.args for link in links]))
         inputs[block] = phis
 
-    def trivial_phis(block):
+    def simplify_phis(block):
         phis = inputs[block]
         to_remove = []
+        unique_phis = {}
         for i, (input, phi_args) in enumerate(phis):
             new_args = [uf.find_rep(arg) for arg in phi_args]
-            if all_equal(new_args):
+            if all_equal(new_args) and not isspecialvar(new_args[0]):
                 uf.union(new_args[0], input)
                 to_remove.append(i)
+            else:
+                t = tuple(new_args)
+                if t in unique_phis:
+                    uf.union(unique_phis[t], input)
+                    to_remove.append(i)
+                else:
+                    unique_phis[t] = input
         for i in reversed(to_remove):
             del phis[i]
         return bool(to_remove)
@@ -593,7 +604,7 @@
     while progress:
         progress = False
         for block in inputs:
-            if trivial_phis(block):
+            if simplify_phis(block):
                 progress = True
 
     renaming = {key: uf[key].rep for key in uf}
@@ -637,7 +648,7 @@
     #    when for all possible incoming paths they would get twice the same
     #    value (this is really the purpose of remove_identical_vars()).
     #
-    builder = ssa.DataFlowFamilyBuilder(graph)
+    builder = DataFlowFamilyBuilder(graph)
     variable_families = builder.get_variable_families()  # vertical removal
     while True:
         if not builder.merge_identical_phi_nodes():    # horizontal removal
@@ -732,7 +743,7 @@
     # NB. this assumes RPythonicity: we can only iterate over something
     # that has a len(), and this len() cannot change as long as we are
     # using the iterator.
-    builder = ssa.DataFlowFamilyBuilder(graph)
+    builder = DataFlowFamilyBuilder(graph)
     variable_families = builder.get_variable_families()
     c_append = Constant('append')
     newlist_v = {}

From noreply at buildbot.pypy.org  Sat Nov 15 03:59:32 2014
From: noreply at buildbot.pypy.org (rlamy)
Date: Sat, 15 Nov 2014 03:59:32 +0100 (CET)
Subject: [pypy-commit] pypy ssa-flow: UnionFind: ensure meaningful result
 when absorb() isn't commutative
Message-ID: <20141115025932.B10951D2D57@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy 
Branch: ssa-flow
Changeset: r74530:4ee0a64a020b
Date: 2014-11-15 02:59 +0000
http://bitbucket.org/pypy/pypy/changeset/4ee0a64a020b/

Log:	UnionFind: ensure meaningful result when absorb() isn't commutative

diff --git a/rpython/tool/algo/test/test_unionfind.py b/rpython/tool/algo/test/test_unionfind.py
--- a/rpython/tool/algo/test/test_unionfind.py
+++ b/rpython/tool/algo/test/test_unionfind.py
@@ -18,6 +18,17 @@
     uf.find(2)
     for i in xrange(2, 20, 2):
         uf.union(i, 2)
-    assert len(state) == 2 # we have exactly 2 partitions
+    assert len(state) == 2  # we have exactly 2 partitions
 
+def test_asymmetric_absorb():
+    class Info(object):
+        def __init__(self, obj):
+            self.values = [obj]
 
+        def absorb(self, other):
+            self.values += other.values
+
+    uf = UnionFind(Info)
+    uf.union(2, 3)
+    uf.union(1, 2)
+    assert uf[1].values == uf[2].values == uf[3].values == [1, 2, 3]
diff --git a/rpython/tool/algo/unionfind.py b/rpython/tool/algo/unionfind.py
--- a/rpython/tool/algo/unionfind.py
+++ b/rpython/tool/algo/unionfind.py
@@ -72,19 +72,16 @@
         if rep1 is rep2:
             return new1 or new2, rep1, info1
 
-        w1 = self.weight[rep1]
-        w2 = self.weight[rep2]
-
-        w = w1 + w2
-
-        if w1 < w2:
-            rep1, rep2, info1, info2, = rep2, rep1, info2, info1
-
         if info1 is not None:
             info1.absorb(info2)
 
+        w1 = self.weight[rep1]
+        w2 = self.weight[rep2]
+        w = w1 + w2
+        if w1 < w2:
+            rep1, rep2 = rep2, rep1
+
         self.link_to_parent[rep2] = rep1
-
         del self.weight[rep2]
         del self.root_info[rep2]
 

From noreply at buildbot.pypy.org  Sat Nov 15 06:53:21 2014
From: noreply at buildbot.pypy.org (rlamy)
Date: Sat, 15 Nov 2014 06:53:21 +0100 (CET)
Subject: [pypy-commit] pypy ssa-flow: coalesce_bool() sort of requires SSI,
	actually
Message-ID: <20141115055321.C91E61C32A5@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy 
Branch: ssa-flow
Changeset: r74531:198df09019ba
Date: 2014-11-15 05:16 +0000
http://bitbucket.org/pypy/pypy/changeset/198df09019ba/

Log:	coalesce_bool() sort of requires SSI, actually

diff --git a/rpython/translator/simplify.py b/rpython/translator/simplify.py
--- a/rpython/translator/simplify.py
+++ b/rpython/translator/simplify.py
@@ -1080,9 +1080,9 @@
     eliminate_empty_blocks,
     remove_assertion_errors,
     remove_trivial_links,
-    coalesce_bool,
     remove_identical_vars_SSA,
     SSA_to_SSI,
+    coalesce_bool,
     transform_ovfcheck,
     simplify_exceptions,
     transform_xxxitem,

From noreply at buildbot.pypy.org  Sat Nov 15 06:53:23 2014
From: noreply at buildbot.pypy.org (rlamy)
Date: Sat, 15 Nov 2014 06:53:23 +0100 (CET)
Subject: [pypy-commit] pypy ssa-flow: remove_trivial_links() does not need
 to rename anything if done after remove_identical_vars()
Message-ID: <20141115055323.6728D1C32A5@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy 
Branch: ssa-flow
Changeset: r74532:bd556294573d
Date: 2014-11-15 05:53 +0000
http://bitbucket.org/pypy/pypy/changeset/bd556294573d/

Log:	remove_trivial_links() does not need to rename anything if done
	after remove_identical_vars()

diff --git a/rpython/translator/simplify.py b/rpython/translator/simplify.py
--- a/rpython/translator/simplify.py
+++ b/rpython/translator/simplify.py
@@ -250,42 +250,32 @@
 def remove_trivial_links(graph):
     """Remove trivial links by merging their source and target blocks
 
-    A link is trivial if it only renames variables, is the single exit of its
+    A link is trivial if it has no arguments, is the single exit of its
     source and the single parent of its target.
     """
     entrymap = mkentrymap(graph)
     block = graph.startblock
     seen = set([block])
     stack = list(block.exits)
-    renaming = {}
     while stack:
         link = stack.pop()
         if link.target in seen:
             continue
         source = link.prevblock
         target = link.target
-        if (source.exitswitch is None and len(entrymap[target]) == 1 and
+        if (not link.args and source.exitswitch is None and
+                len(entrymap[target]) == 1 and
                 target.exits):  # stop at the returnblock
             assert len(source.exits) == 1
-            for vprev, vtarg in zip(link.args, target.inputargs):
-                while vprev in renaming:
-                    vprev = renaming[vprev]
-                renaming[vtarg] = vprev
             def rename(v):
                 return renaming.get(v, v)
-            for op in target.operations:
-                source.operations.append(op.replace(renaming))
-            source.exitswitch = newexitswitch = rename(target.exitswitch)
-            exits = []
-            for exit in target.exits:
-                newexit = exit.copy(rename)
-                exits.append(newexit)
-            source.recloseblock(*exits)
+            source.operations.extend(target.operations)
+            source.exitswitch = newexitswitch = target.exitswitch
+            source.recloseblock(*target.exits)
             if isinstance(newexitswitch, Constant) and newexitswitch != c_last_exception:
                 exits = replace_exitswitch_by_constant(source, newexitswitch)
-            stack.extend(exits)
+            stack.extend(source.exits)
         else:
-            target.renamevariables(renaming)
             seen.add(target)
             stack.extend(target.exits)
 
@@ -1079,8 +1069,8 @@
     transform_dead_op_vars,
     eliminate_empty_blocks,
     remove_assertion_errors,
+    remove_identical_vars_SSA,
     remove_trivial_links,
-    remove_identical_vars_SSA,
     SSA_to_SSI,
     coalesce_bool,
     transform_ovfcheck,

From noreply at buildbot.pypy.org  Sun Nov 16 00:21:59 2014
From: noreply at buildbot.pypy.org (rlamy)
Date: Sun, 16 Nov 2014 00:21:59 +0100 (CET)
Subject: [pypy-commit] pypy ssa-flow: fix test
Message-ID: <20141115232200.09D181D2304@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy 
Branch: ssa-flow
Changeset: r74533:76bc6682de8d
Date: 2014-11-15 22:06 +0000
http://bitbucket.org/pypy/pypy/changeset/76bc6682de8d/

Log:	fix test

diff --git a/rpython/translator/backendopt/test/test_escape.py b/rpython/translator/backendopt/test/test_escape.py
--- a/rpython/translator/backendopt/test/test_escape.py
+++ b/rpython/translator/backendopt/test/test_escape.py
@@ -111,7 +111,10 @@
         a.x = 12
         return a1.x
     t, adi, graph = build_adi(fn6, [int])
-    avar = graph.startblock.exits[0].target.inputargs[1]
+    graph.show()
+    op = graph.startblock.exits[0].target.operations[0]
+    assert op.opname == 'setfield'
+    avar = op.args[0]
     state = adi.getstate(avar)
     assert len(state.creation_points) == 2
     for crep in state.creation_points:
@@ -179,7 +182,7 @@
     t, adi, graph = build_adi(f, [])
     g_graph = graphof(t, g)
     a0var = graph.startblock.operations[0].result
-    b0var = graph.startblock.operations[3].result 
+    b0var = graph.startblock.operations[3].result
     a0state = adi.getstate(a0var)
     b0state = adi.getstate(b0var)
     a0crep, = a0state.creation_points

From noreply at buildbot.pypy.org  Sun Nov 16 00:22:01 2014
From: noreply at buildbot.pypy.org (rlamy)
Date: Sun, 16 Nov 2014 00:22:01 +0100 (CET)
Subject: [pypy-commit] pypy ssa-flow: make constant-folding exit switches a
	separate pass
Message-ID: <20141115232201.525DB1D2304@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy 
Branch: ssa-flow
Changeset: r74534:c0b305b29bd0
Date: 2014-11-15 23:21 +0000
http://bitbucket.org/pypy/pypy/changeset/c0b305b29bd0/

Log:	make constant-folding exit switches a separate pass

diff --git a/rpython/translator/simplify.py b/rpython/translator/simplify.py
--- a/rpython/translator/simplify.py
+++ b/rpython/translator/simplify.py
@@ -247,6 +247,30 @@
             seen.append(case)
         block.recloseblock(*exits)
 
+def constfold_exitswitch(graph):
+    """Remove trivial links by merging their source and target blocks
+
+    A link is trivial if it has no arguments, is the single exit of its
+    source and the single parent of its target.
+    """
+    block = graph.startblock
+    seen = set([block])
+    stack = list(block.exits)
+    while stack:
+        link = stack.pop()
+        target = link.target
+        if target in seen:
+            continue
+        source = link.prevblock
+        switch = source.exitswitch
+        if (isinstance(switch, Constant) and switch != c_last_exception):
+            exits = replace_exitswitch_by_constant(source, switch)
+            stack.extend(exits)
+        else:
+            seen.add(target)
+            stack.extend(target.exits)
+
+
 def remove_trivial_links(graph):
     """Remove trivial links by merging their source and target blocks
 
@@ -267,13 +291,9 @@
                 len(entrymap[target]) == 1 and
                 target.exits):  # stop at the returnblock
             assert len(source.exits) == 1
-            def rename(v):
-                return renaming.get(v, v)
             source.operations.extend(target.operations)
             source.exitswitch = newexitswitch = target.exitswitch
             source.recloseblock(*target.exits)
-            if isinstance(newexitswitch, Constant) and newexitswitch != c_last_exception:
-                exits = replace_exitswitch_by_constant(source, newexitswitch)
             stack.extend(source.exits)
         else:
             seen.add(target)
@@ -1070,6 +1090,7 @@
     eliminate_empty_blocks,
     remove_assertion_errors,
     remove_identical_vars_SSA,
+    constfold_exitswitch,
     remove_trivial_links,
     SSA_to_SSI,
     coalesce_bool,

From noreply at buildbot.pypy.org  Sun Nov 16 05:27:15 2014
From: noreply at buildbot.pypy.org (rlamy)
Date: Sun, 16 Nov 2014 05:27:15 +0100 (CET)
Subject: [pypy-commit] pypy ssa-flow: SSA_to_SSI() isn't needed in
	cleanup_graph()
Message-ID: <20141116042715.CC8BD1D2487@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy 
Branch: ssa-flow
Changeset: r74535:da73f13a1ce8
Date: 2014-11-16 03:51 +0000
http://bitbucket.org/pypy/pypy/changeset/da73f13a1ce8/

Log:	SSA_to_SSI() isn't needed in cleanup_graph()

diff --git a/rpython/translator/simplify.py b/rpython/translator/simplify.py
--- a/rpython/translator/simplify.py
+++ b/rpython/translator/simplify.py
@@ -1111,7 +1111,6 @@
 def cleanup_graph(graph):
     checkgraph(graph)
     eliminate_empty_blocks(graph)
-    SSA_to_SSI(graph)
     join_blocks(graph)
     remove_identical_vars(graph)
     checkgraph(graph)

From noreply at buildbot.pypy.org  Sun Nov 16 05:27:17 2014
From: noreply at buildbot.pypy.org (rlamy)
Date: Sun, 16 Nov 2014 05:27:17 +0100 (CET)
Subject: [pypy-commit] pypy ssa-flow: fix 2 tests: replace expected results
 with equivalent forms
Message-ID: <20141116042717.096E81D2487@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy 
Branch: ssa-flow
Changeset: r74536:a1d79d171f8f
Date: 2014-11-16 04:15 +0000
http://bitbucket.org/pypy/pypy/changeset/a1d79d171f8f/

Log:	fix 2 tests: replace expected results with equivalent forms

diff --git a/rpython/jit/codewriter/test/test_flatten.py b/rpython/jit/codewriter/test/test_flatten.py
--- a/rpython/jit/codewriter/test/test_flatten.py
+++ b/rpython/jit/codewriter/test/test_flatten.py
@@ -197,10 +197,10 @@
             int_gt %i2, $0 -> %i4
             goto_if_not %i4, L2
             -live- L2
-            int_copy %i2 -> %i5
-            int_copy %i3 -> %i6
-            int_add %i6, %i5 -> %i7
-            int_sub %i5, $1 -> %i8
+            int_copy %i3 -> %i5
+            int_copy %i2 -> %i6
+            int_add %i5, %i6 -> %i7
+            int_sub %i6, $1 -> %i8
             int_copy %i8 -> %i2
             int_copy %i7 -> %i3
             goto L1
@@ -221,10 +221,10 @@
             L1:
             goto_if_not_int_gt %i2, $0, L2
             -live- L2
-            int_copy %i2 -> %i4
-            int_copy %i3 -> %i5
-            int_add %i5, %i4 -> %i6
-            int_sub %i4, $1 -> %i7
+            int_copy %i3 -> %i4
+            int_copy %i2 -> %i5
+            int_add %i4, %i5 -> %i6
+            int_sub %i5, $1 -> %i7
             int_copy %i7 -> %i2
             int_copy %i6 -> %i3
             goto L1

From noreply at buildbot.pypy.org  Sun Nov 16 05:27:18 2014
From: noreply at buildbot.pypy.org (rlamy)
Date: Sun, 16 Nov 2014 05:27:18 +0100 (CET)
Subject: [pypy-commit] pypy ssa-flow: rm debugging cruft
Message-ID: <20141116042718.43A9F1D2487@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy 
Branch: ssa-flow
Changeset: r74537:a99ad1783a8d
Date: 2014-11-16 04:16 +0000
http://bitbucket.org/pypy/pypy/changeset/a99ad1783a8d/

Log:	rm debugging cruft

diff --git a/rpython/translator/backendopt/test/test_escape.py b/rpython/translator/backendopt/test/test_escape.py
--- a/rpython/translator/backendopt/test/test_escape.py
+++ b/rpython/translator/backendopt/test/test_escape.py
@@ -111,7 +111,6 @@
         a.x = 12
         return a1.x
     t, adi, graph = build_adi(fn6, [int])
-    graph.show()
     op = graph.startblock.exits[0].target.operations[0]
     assert op.opname == 'setfield'
     avar = op.args[0]

From noreply at buildbot.pypy.org  Sun Nov 16 19:22:14 2014
From: noreply at buildbot.pypy.org (rlamy)
Date: Sun, 16 Nov 2014 19:22:14 +0100 (CET)
Subject: [pypy-commit] pypy ssa-flow: make variable ordering deterministic
	in SSA_to_SSI()
Message-ID: <20141116182214.B8B411D2304@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy 
Branch: ssa-flow
Changeset: r74538:b754a290822e
Date: 2014-11-16 18:22 +0000
http://bitbucket.org/pypy/pypy/changeset/b754a290822e/

Log:	make variable ordering deterministic in SSA_to_SSI()

diff --git a/rpython/jit/codewriter/test/test_flatten.py b/rpython/jit/codewriter/test/test_flatten.py
--- a/rpython/jit/codewriter/test/test_flatten.py
+++ b/rpython/jit/codewriter/test/test_flatten.py
@@ -197,10 +197,10 @@
             int_gt %i2, $0 -> %i4
             goto_if_not %i4, L2
             -live- L2
-            int_copy %i3 -> %i5
-            int_copy %i2 -> %i6
-            int_add %i5, %i6 -> %i7
-            int_sub %i6, $1 -> %i8
+            int_copy %i2 -> %i5
+            int_copy %i3 -> %i6
+            int_add %i6, %i5 -> %i7
+            int_sub %i5, $1 -> %i8
             int_copy %i8 -> %i2
             int_copy %i7 -> %i3
             goto L1
@@ -221,10 +221,10 @@
             L1:
             goto_if_not_int_gt %i2, $0, L2
             -live- L2
-            int_copy %i3 -> %i4
-            int_copy %i2 -> %i5
-            int_add %i4, %i5 -> %i6
-            int_sub %i5, $1 -> %i7
+            int_copy %i2 -> %i4
+            int_copy %i3 -> %i5
+            int_add %i5, %i4 -> %i6
+            int_sub %i4, $1 -> %i7
             int_copy %i7 -> %i2
             int_copy %i6 -> %i3
             goto L1
diff --git a/rpython/translator/backendopt/ssa.py b/rpython/translator/backendopt/ssa.py
--- a/rpython/translator/backendopt/ssa.py
+++ b/rpython/translator/backendopt/ssa.py
@@ -145,17 +145,26 @@
     del builder
 
     pending = []     # list of (block, var-used-but-not-defined)
-    for block in entrymap:
+    for block in graph.iterblocks():
+        if block not in entrymap:
+            continue
         variables_created = variables_created_in(block)
-        variables_used = set()
+        seen = set(variables_created)
+        variables_used = []
+        def record_used_var(v):
+            if v not in seen:
+                variables_used.append(v)
+                seen.add(v)
         for op in block.operations:
-            variables_used.update(op.args)
-        variables_used.add(block.exitswitch)
+            for arg in op.args:
+                record_used_var(arg)
+        record_used_var(block.exitswitch)
         for link in block.exits:
-            variables_used.update(link.args)
+            for arg in link.args:
+                record_used_var(arg)
 
         for v in variables_used:
-            if (isinstance(v, Variable) and v not in variables_created and
+            if (isinstance(v, Variable) and
                     v._name not in ('last_exception_', 'last_exc_value_')):
                 pending.append((block, v))
 

From noreply at buildbot.pypy.org  Sun Nov 16 20:39:24 2014
From: noreply at buildbot.pypy.org (rlamy)
Date: Sun, 16 Nov 2014 20:39:24 +0100 (CET)
Subject: [pypy-commit] pypy ssa-flow: close branch before merging
Message-ID: <20141116193924.F358F1C0F1B@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy 
Branch: ssa-flow
Changeset: r74539:bb5be160c110
Date: 2014-11-16 19:36 +0000
http://bitbucket.org/pypy/pypy/changeset/bb5be160c110/

Log:	close branch before merging


From noreply at buildbot.pypy.org  Sun Nov 16 20:39:26 2014
From: noreply at buildbot.pypy.org (rlamy)
Date: Sun, 16 Nov 2014 20:39:26 +0100 (CET)
Subject: [pypy-commit] pypy default: merge branch ssa-flow
Message-ID: <20141116193926.A1B2F1C0F1B@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy 
Branch: 
Changeset: r74540:a41db2745c80
Date: 2014-11-16 19:39 +0000
http://bitbucket.org/pypy/pypy/changeset/a41db2745c80/

Log:	merge branch ssa-flow

diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -47,3 +47,7 @@
 .. branch kill-rctime
 
 Rename pypy/module/rctime to pypy/module/time, since it contains the implementation of the 'time' module.
+
+.. branch: ssa-flow
+
+Use SSA form for flow graphs inside build_flow() and part of simplify_graph()
diff --git a/rpython/annotator/test/test_annrpython.py b/rpython/annotator/test/test_annrpython.py
--- a/rpython/annotator/test/test_annrpython.py
+++ b/rpython/annotator/test/test_annrpython.py
@@ -14,7 +14,6 @@
 from rpython.rlib.rarithmetic import r_uint, base_int, r_longlong, r_ulonglong
 from rpython.rlib.rarithmetic import r_singlefloat
 from rpython.rlib import objectmodel
-from rpython.flowspace.objspace import build_flow
 from rpython.flowspace.flowcontext import FlowingError
 from rpython.flowspace.operation import op
 
@@ -53,17 +52,6 @@
                 self.translator.view()
             return s
 
-    def make_fun(self, func):
-        import inspect
-        try:
-            func = func.im_func
-        except AttributeError:
-            pass
-        name = func.func_name
-        funcgraph = build_flow(func)
-        funcgraph.source = inspect.getsource(func)
-        return funcgraph
-
     def test_simple_func(self):
         """
         one test source:
diff --git a/rpython/flowspace/bytecode.py b/rpython/flowspace/bytecode.py
--- a/rpython/flowspace/bytecode.py
+++ b/rpython/flowspace/bytecode.py
@@ -5,7 +5,6 @@
 from opcode import EXTENDED_ARG, HAVE_ARGUMENT
 import opcode
 from rpython.flowspace.argument import Signature
-from rpython.flowspace.flowcontext import BytecodeCorruption
 
 CO_GENERATOR = 0x0020
 CO_VARARGS = 0x0004
@@ -27,6 +26,11 @@
         kwargname = None
     return Signature(argnames, varargname, kwargname)
 
+
+class BytecodeCorruption(Exception):
+    pass
+
+
 class HostCode(object):
     """
     A wrapper around a native code object of the host interpreter
diff --git a/rpython/flowspace/flowcontext.py b/rpython/flowspace/flowcontext.py
--- a/rpython/flowspace/flowcontext.py
+++ b/rpython/flowspace/flowcontext.py
@@ -7,6 +7,7 @@
 import __builtin__
 
 from rpython.tool.error import source_lines
+from rpython.translator.simplify import eliminate_empty_blocks
 from rpython.rlib import rstackovf
 from rpython.flowspace.argument import CallSpec
 from rpython.flowspace.model import (Constant, Variable, Block, Link,
@@ -16,6 +17,7 @@
 from rpython.flowspace.specialcase import (rpython_print_item,
     rpython_print_newline)
 from rpython.flowspace.operation import op
+from rpython.flowspace.bytecode import BytecodeCorruption
 
 w_None = const(None)
 
@@ -30,11 +32,10 @@
         msg += source_lines(self.ctx.graph, None, offset=self.ctx.last_instr)
         return "\n".join(msg)
 
+
 class StopFlowing(Exception):
     pass
 
-class BytecodeCorruption(Exception):
-    pass
 
 class SpamBlock(Block):
     def __init__(self, framestate):
@@ -79,35 +80,10 @@
         self.last_exception = last_exception
 
 def fixeggblocks(graph):
-    varnames = graph.func.func_code.co_varnames
     for block in graph.iterblocks():
         if isinstance(block, SpamBlock):
-            for name, w_value in zip(varnames, block.framestate.mergeable):
-                if isinstance(w_value, Variable):
-                    w_value.rename(name)
             del block.framestate     # memory saver
 
-    # EggBlocks reuse the variables of their previous block,
-    # which is deemed not acceptable for simplicity of the operations
-    # that will be performed later on the flow graph.
-    for link in list(graph.iterlinks()):
-        block = link.target
-        if isinstance(block, EggBlock):
-            if (not block.operations and len(block.exits) == 1 and
-                link.args == block.inputargs):   # not renamed
-                # if the variables are not renamed across this link
-                # (common case for EggBlocks) then it's easy enough to
-                # get rid of the empty EggBlock.
-                link2 = block.exits[0]
-                link.args = list(link2.args)
-                link.target = link2.target
-                assert link2.exitcase is None
-            else:
-                mapping = {}
-                for a in block.inputargs:
-                    mapping[a] = Variable(a)
-                block.renamevariables(mapping)
-
 # ____________________________________________________________
 
 class Recorder(object):
@@ -132,12 +108,11 @@
 
     def guessbool(self, ctx, w_condition):
         block = self.crnt_block
-        vars = block.getvariables()
         links = []
         for case in [False, True]:
-            egg = EggBlock(vars, block, case)
+            egg = EggBlock([], block, case)
             ctx.pendingblocks.append(egg)
-            link = Link(vars, egg, case)
+            link = Link([], egg, case)
             links.append(link)
 
         block.exitswitch = w_condition
@@ -154,16 +129,16 @@
         links = []
         for case in [None] + list(cases):
             if case is not None:
-                assert block.operations[-1].result is bvars[-1]
-                vars = bvars[:-1]
-                vars2 = bvars[:-1]
                 if case is Exception:
                     last_exc = Variable('last_exception')
                 else:
                     last_exc = Constant(case)
                 last_exc_value = Variable('last_exc_value')
-                vars.extend([last_exc, last_exc_value])
-                vars2.extend([Variable(), Variable()])
+                vars = [last_exc, last_exc_value]
+                vars2 = [Variable(), Variable()]
+            else:
+                vars = []
+                vars2 = []
             egg = EggBlock(vars2, block, case)
             ctx.pendingblocks.append(egg)
             link = Link(vars, egg, case)
@@ -498,33 +473,43 @@
         candidates = self.joinpoints.setdefault(next_instr, [])
         for block in candidates:
             newstate = block.framestate.union(currentstate)
-            if newstate is None:
-                continue
-            elif newstate == block.framestate:
-                outputargs = currentstate.getoutputargs(newstate)
-                currentblock.closeblock(Link(outputargs, block))
-                return
-            else:
+            if newstate is not None:
                 break
         else:
             newstate = currentstate.copy()
-            block = None
+            newblock = SpamBlock(newstate)
+            # unconditionally link the current block to the newblock
+            outputargs = currentstate.getoutputargs(newstate)
+            link = Link(outputargs, newblock)
+            currentblock.closeblock(link)
+            candidates.insert(0, newblock)
+            self.pendingblocks.append(newblock)
+            return
+
+        if newstate.matches(block.framestate):
+            outputargs = currentstate.getoutputargs(newstate)
+            currentblock.closeblock(Link(outputargs, block))
+            return
 
         newblock = SpamBlock(newstate)
+        varnames = self.pycode.co_varnames
+        for name, w_value in zip(varnames, newstate.mergeable):
+            if isinstance(w_value, Variable):
+                w_value.rename(name)
         # unconditionally link the current block to the newblock
         outputargs = currentstate.getoutputargs(newstate)
         link = Link(outputargs, newblock)
         currentblock.closeblock(link)
 
-        if block is not None:
-            # to simplify the graph, we patch the old block to point
-            # directly at the new block which is its generalization
-            block.dead = True
-            block.operations = ()
-            block.exitswitch = None
-            outputargs = block.framestate.getoutputargs(newstate)
-            block.recloseblock(Link(outputargs, newblock))
-            candidates.remove(block)
+        # to simplify the graph, we patch the old block to point
+        # directly at the new block which is its generalization
+        block.dead = True
+        block.operations = ()
+        block.exitswitch = None
+        outputargs = block.framestate.getoutputargs(newstate)
+        block.recloseblock(Link(outputargs, newblock))
+        candidates.remove(block)
+
         candidates.insert(0, newblock)
         self.pendingblocks.append(newblock)
 
@@ -936,6 +921,8 @@
         w_newvalue = self.popvalue()
         assert w_newvalue is not None
         self.locals_stack_w[varindex] = w_newvalue
+        if isinstance(w_newvalue, Variable):
+            w_newvalue.rename(self.getlocalvarname(varindex))
 
     def STORE_GLOBAL(self, nameindex):
         varname = self.getname_u(nameindex)
diff --git a/rpython/flowspace/framestate.py b/rpython/flowspace/framestate.py
--- a/rpython/flowspace/framestate.py
+++ b/rpython/flowspace/framestate.py
@@ -13,20 +13,18 @@
         newstate = []
         for w in self.mergeable:
             if isinstance(w, Variable):
-                w = Variable()
+                w = Variable(w)
             newstate.append(w)
         return FrameState(newstate, self.blocklist, self.next_instr)
 
     def getvariables(self):
         return [w for w in self.mergeable if isinstance(w, Variable)]
 
-    def __eq__(self, other):
-        """Two states are equal
-        if they only use different Variables at the same place"""
+    def matches(self, other):
+        """Two states match if they only differ by using different Variables
+        at the same place"""
         # safety check, don't try to compare states with different
         # nonmergeable states
-        assert isinstance(other, FrameState)
-        assert len(self.mergeable) == len(other.mergeable)
         assert self.blocklist == other.blocklist
         assert self.next_instr == other.next_instr
         for w1, w2 in zip(self.mergeable, other.mergeable):
@@ -35,9 +33,6 @@
                 return False
         return True
 
-    def __ne__(self, other):
-        return not (self == other)
-
     def union(self, other):
         """Compute a state that is at least as general as both self and other.
            A state 'a' is more general than a state 'b' if all Variables in 'b'
@@ -66,14 +61,14 @@
 
 def union(w1, w2):
     "Union of two variables or constants."
+    if w1 == w2:
+        return w1
     if w1 is None or w2 is None:
         return None  # if w1 or w2 is an undefined local, we "kill" the value
                      # coming from the other path and return an undefined local
     if isinstance(w1, Variable) or isinstance(w2, Variable):
         return Variable()  # new fresh Variable
     if isinstance(w1, Constant) and isinstance(w2, Constant):
-        if w1 == w2:
-            return w1
         # FlowSignal represent stack unrollers in the stack.
         # They should not be merged because they will be unwrapped.
         # This is needed for try:except: and try:finally:, though
diff --git a/rpython/flowspace/model.py b/rpython/flowspace/model.py
--- a/rpython/flowspace/model.py
+++ b/rpython/flowspace/model.py
@@ -206,8 +206,6 @@
         return uniqueitems([w for w in result if isinstance(w, Constant)])
 
     def renamevariables(self, mapping):
-        for a in mapping:
-            assert isinstance(a, Variable), a
         self.inputargs = [mapping.get(a, a) for a in self.inputargs]
         for op in self.operations:
             op.args = [mapping.get(a, a) for a in op.args]
diff --git a/rpython/flowspace/objspace.py b/rpython/flowspace/objspace.py
--- a/rpython/flowspace/objspace.py
+++ b/rpython/flowspace/objspace.py
@@ -30,7 +30,7 @@
 
 def build_flow(func):
     """
-    Create the flow graph for the function.
+    Create the flow graph (in SSA form) for the function.
     """
     _assert_rpythonic(func)
     if (isgeneratorfunction(func) and
@@ -41,7 +41,6 @@
     ctx = FlowContext(graph, code)
     ctx.build_flow()
     fixeggblocks(graph)
-    checkgraph(graph)
     if code.is_generator:
         tweak_generator_graph(graph)
     return graph
diff --git a/rpython/flowspace/pygraph.py b/rpython/flowspace/pygraph.py
--- a/rpython/flowspace/pygraph.py
+++ b/rpython/flowspace/pygraph.py
@@ -13,7 +13,7 @@
         from rpython.flowspace.flowcontext import SpamBlock
         data = [None] * code.co_nlocals
         for i in range(code.formalargcount):
-            data[i] = Variable()
+            data[i] = Variable(code.co_varnames[i])
         state = FrameState(data + [Constant(None), Constant(None)], [], 0)
         initialblock = SpamBlock(state)
         super(PyGraph, self).__init__(self._sanitize_funcname(func), initialblock)
diff --git a/rpython/flowspace/test/test_framestate.py b/rpython/flowspace/test/test_framestate.py
--- a/rpython/flowspace/test/test_framestate.py
+++ b/rpython/flowspace/test/test_framestate.py
@@ -26,41 +26,41 @@
         ctx = self.get_context(self.func_simple)
         fs1 = ctx.getstate(0)
         fs2 = ctx.getstate(0)
-        assert fs1 == fs2
+        assert fs1.matches(fs2)
 
     def test_neq_hacked_framestate(self):
         ctx = self.get_context(self.func_simple)
         fs1 = ctx.getstate(0)
         ctx.locals_stack_w[ctx.pycode.co_nlocals-1] = Variable()
         fs2 = ctx.getstate(0)
-        assert fs1 != fs2
+        assert not fs1.matches(fs2)
 
     def test_union_on_equal_framestates(self):
         ctx = self.get_context(self.func_simple)
         fs1 = ctx.getstate(0)
         fs2 = ctx.getstate(0)
-        assert fs1.union(fs2) == fs1
+        assert fs1.union(fs2).matches(fs1)
 
     def test_union_on_hacked_framestates(self):
         ctx = self.get_context(self.func_simple)
         fs1 = ctx.getstate(0)
         ctx.locals_stack_w[ctx.pycode.co_nlocals-1] = Variable()
         fs2 = ctx.getstate(0)
-        assert fs1.union(fs2) == fs2  # fs2 is more general
-        assert fs2.union(fs1) == fs2  # fs2 is more general
+        assert fs1.union(fs2).matches(fs2)  # fs2 is more general
+        assert fs2.union(fs1).matches(fs2)  # fs2 is more general
 
     def test_restore_frame(self):
         ctx = self.get_context(self.func_simple)
         fs1 = ctx.getstate(0)
         ctx.locals_stack_w[ctx.pycode.co_nlocals-1] = Variable()
         ctx.setstate(fs1)
-        assert fs1 == ctx.getstate(0)
+        assert fs1.matches(ctx.getstate(0))
 
     def test_copy(self):
         ctx = self.get_context(self.func_simple)
         fs1 = ctx.getstate(0)
         fs2 = fs1.copy()
-        assert fs1 == fs2
+        assert fs1.matches(fs2)
 
     def test_getvariables(self):
         ctx = self.get_context(self.func_simple)
diff --git a/rpython/flowspace/test/test_generator.py b/rpython/flowspace/test/test_generator.py
--- a/rpython/flowspace/test/test_generator.py
+++ b/rpython/flowspace/test/test_generator.py
@@ -67,7 +67,7 @@
             yield n
             yield n
         #
-        graph = build_flow(func)
+        graph = make_generator_entry_graph(func)
         if option.view:
             graph.show()
         block = graph.startblock
diff --git a/rpython/flowspace/test/test_objspace.py b/rpython/flowspace/test/test_objspace.py
--- a/rpython/flowspace/test/test_objspace.py
+++ b/rpython/flowspace/test/test_objspace.py
@@ -635,6 +635,7 @@
 
     def test_highly_branching_example(self):
         x = self.codetest(self.highly_branching_example)
+        simplify_graph(x)
         # roughly 20 blocks + 30 links
         assert len(list(x.iterblocks())) + len(list(x.iterlinks())) < 60
 
@@ -1290,6 +1291,27 @@
         assert isinstance(link.args[0], Constant)
         assert link.args[0].value == 5
 
+    def test_remove_dead_ops(self):
+        def f():
+            a = [1]
+            b = (a, a)
+            c = type(b)
+        graph = self.codetest(f)
+        simplify_graph(graph)
+        assert graph.startblock.operations == []
+        [link] = graph.startblock.exits
+        assert link.target is graph.returnblock
+
+    def test_not_combine(self):
+        def f(n):
+            t = not n
+            if not n:
+                t += 1
+            return t
+        graph = self.codetest(f)
+        simplify_graph(graph)
+        assert self.all_operations(graph) == {'bool': 1, 'inplace_add': 1}
+
 
 DATA = {'x': 5,
         'y': 6}
diff --git a/rpython/tool/algo/test/test_unionfind.py b/rpython/tool/algo/test/test_unionfind.py
--- a/rpython/tool/algo/test/test_unionfind.py
+++ b/rpython/tool/algo/test/test_unionfind.py
@@ -18,6 +18,17 @@
     uf.find(2)
     for i in xrange(2, 20, 2):
         uf.union(i, 2)
-    assert len(state) == 2 # we have exactly 2 partitions
+    assert len(state) == 2  # we have exactly 2 partitions
 
+def test_asymmetric_absorb():
+    class Info(object):
+        def __init__(self, obj):
+            self.values = [obj]
 
+        def absorb(self, other):
+            self.values += other.values
+
+    uf = UnionFind(Info)
+    uf.union(2, 3)
+    uf.union(1, 2)
+    assert uf[1].values == uf[2].values == uf[3].values == [1, 2, 3]
diff --git a/rpython/tool/algo/unionfind.py b/rpython/tool/algo/unionfind.py
--- a/rpython/tool/algo/unionfind.py
+++ b/rpython/tool/algo/unionfind.py
@@ -72,19 +72,16 @@
         if rep1 is rep2:
             return new1 or new2, rep1, info1
 
-        w1 = self.weight[rep1]
-        w2 = self.weight[rep2]
-
-        w = w1 + w2
-
-        if w1 < w2:
-            rep1, rep2, info1, info2, = rep2, rep1, info2, info1
-
         if info1 is not None:
             info1.absorb(info2)
 
+        w1 = self.weight[rep1]
+        w2 = self.weight[rep2]
+        w = w1 + w2
+        if w1 < w2:
+            rep1, rep2 = rep2, rep1
+
         self.link_to_parent[rep2] = rep1
-
         del self.weight[rep2]
         del self.root_info[rep2]
 
diff --git a/rpython/translator/backendopt/ssa.py b/rpython/translator/backendopt/ssa.py
--- a/rpython/translator/backendopt/ssa.py
+++ b/rpython/translator/backendopt/ssa.py
@@ -16,7 +16,9 @@
         # [Block, blockvar, linkvar, linkvar, linkvar...]
         opportunities = []
         opportunities_with_const = []
-        for block, links in mkinsideentrymap(graph).items():
+        entrymap = mkentrymap(graph)
+        del entrymap[graph.startblock]
+        for block, links in entrymap.items():
             assert links
             for n, inputvar in enumerate(block.inputargs):
                 vars = [block, inputvar]
@@ -123,63 +125,48 @@
 
 # ____________________________________________________________
 
-def mkinsideentrymap(graph_or_blocks):
-    # graph_or_blocks can be a full FunctionGraph, or a mapping
-    # {block: reachable-from-outside-flag}.
-    if isinstance(graph_or_blocks, dict):
-        blocks = graph_or_blocks
-        entrymap = {}
-        for block in blocks:
-            for link in block.exits:
-                if link.target in blocks and not blocks[link.target]:
-                    entrymap.setdefault(link.target, []).append(link)
-        return entrymap
-    else:
-        graph = graph_or_blocks
-        entrymap = mkentrymap(graph)
-        del entrymap[graph.startblock]
-        return entrymap
-
 def variables_created_in(block):
-    result = {}
-    for v in block.inputargs:
-        result[v] = True
+    result = set(block.inputargs)
     for op in block.operations:
-        result[op.result] = True
+        result.add(op.result)
     return result
 
 
-def SSA_to_SSI(graph_or_blocks, annotator=None):
+def SSA_to_SSI(graph, annotator=None):
     """Turn a number of blocks belonging to a flow graph into valid (i.e. SSI)
     form, assuming that they are only in SSA form (i.e. they can use each
     other's variables directly, without having to pass and rename them along
     links).
-
-    'graph_or_blocks' can be a graph, or just a dict that lists some blocks
-    from a graph, as follows: {block: reachable-from-outside-flag}.
     """
-    entrymap = mkinsideentrymap(graph_or_blocks)
-    builder = DataFlowFamilyBuilder(graph_or_blocks)
+    entrymap = mkentrymap(graph)
+    del entrymap[graph.startblock]
+    builder = DataFlowFamilyBuilder(graph)
     variable_families = builder.get_variable_families()
     del builder
 
     pending = []     # list of (block, var-used-but-not-defined)
-
-    for block in entrymap:
+    for block in graph.iterblocks():
+        if block not in entrymap:
+            continue
         variables_created = variables_created_in(block)
-        variables_used = {}
+        seen = set(variables_created)
+        variables_used = []
+        def record_used_var(v):
+            if v not in seen:
+                variables_used.append(v)
+                seen.add(v)
         for op in block.operations:
-            for v in op.args:
-                variables_used[v] = True
-        variables_used[block.exitswitch] = True
+            for arg in op.args:
+                record_used_var(arg)
+        record_used_var(block.exitswitch)
         for link in block.exits:
-            for v in link.args:
-                variables_used[v] = True
+            for arg in link.args:
+                record_used_var(arg)
 
         for v in variables_used:
-            if isinstance(v, Variable):
-                if v not in variables_created:
-                    pending.append((block, v))
+            if (isinstance(v, Variable) and
+                    v._name not in ('last_exception_', 'last_exc_value_')):
+                pending.append((block, v))
 
     while pending:
         block, v = pending.pop()
@@ -190,8 +177,7 @@
         for w in variables_created:
             w_rep = variable_families.find_rep(w)
             if v_rep is w_rep:
-                # 'w' is in the same family as 'v', so we can simply
-                # reuse its value for 'v'
+                # 'w' is in the same family as 'v', so we can reuse it
                 block.renamevariables({v: w})
                 break
         else:
diff --git a/rpython/translator/backendopt/test/test_escape.py b/rpython/translator/backendopt/test/test_escape.py
--- a/rpython/translator/backendopt/test/test_escape.py
+++ b/rpython/translator/backendopt/test/test_escape.py
@@ -111,7 +111,9 @@
         a.x = 12
         return a1.x
     t, adi, graph = build_adi(fn6, [int])
-    avar = graph.startblock.exits[0].target.inputargs[1]
+    op = graph.startblock.exits[0].target.operations[0]
+    assert op.opname == 'setfield'
+    avar = op.args[0]
     state = adi.getstate(avar)
     assert len(state.creation_points) == 2
     for crep in state.creation_points:
@@ -179,7 +181,7 @@
     t, adi, graph = build_adi(f, [])
     g_graph = graphof(t, g)
     a0var = graph.startblock.operations[0].result
-    b0var = graph.startblock.operations[3].result 
+    b0var = graph.startblock.operations[3].result
     a0state = adi.getstate(a0var)
     b0state = adi.getstate(b0var)
     a0crep, = a0state.creation_points
diff --git a/rpython/translator/backendopt/test/test_ssa.py b/rpython/translator/backendopt/test/test_ssa.py
--- a/rpython/translator/backendopt/test/test_ssa.py
+++ b/rpython/translator/backendopt/test/test_ssa.py
@@ -1,7 +1,7 @@
 from rpython.translator.backendopt.ssa import *
 from rpython.translator.translator import TranslationContext
-from rpython.flowspace.model import Block, Link, Variable, Constant
-from rpython.flowspace.model import SpaceOperation
+from rpython.flowspace.model import (
+    Block, Link, Variable, Constant, SpaceOperation, FunctionGraph)
 
 
 def test_data_flow_families():
@@ -60,16 +60,14 @@
     b2 = Block([x])
     b3 = Block([])
 
+    graph = FunctionGraph('x', b1)
     b2.operations.append(SpaceOperation('add', [x, c], y))
     b2.exitswitch = y
 
     b1.closeblock(Link([Constant(0)], b2))
     b2.closeblock(Link([y], b2), Link([], b3))
-    b3.closeblock(Link([y, c], None))
-
-    SSA_to_SSI({b1: True,     # reachable from outside
-                b2: False,
-                b3: False})
+    b3.closeblock(Link([y, c], graph.exceptblock))
+    SSA_to_SSI(graph)
 
     assert len(b1.inputargs) == 1
     assert len(b2.inputargs) == 2
@@ -100,10 +98,8 @@
 
     b3.operations.append(SpaceOperation('hello', [y], z))
     b1.closeblock(Link([x], b2), Link([], b3))
-
-    SSA_to_SSI({b1: True,     # reachable from outside
-                b2: False,
-                b3: False})
+    graph = FunctionGraph('x', b1)
+    SSA_to_SSI(graph)
 
     assert b1.inputargs == [x]
     assert b2.inputargs == [y]
diff --git a/rpython/translator/simplify.py b/rpython/translator/simplify.py
--- a/rpython/translator/simplify.py
+++ b/rpython/translator/simplify.py
@@ -7,13 +7,15 @@
 import py
 from collections import defaultdict
 
+from rpython.tool.algo.unionfind import UnionFind
 from rpython.flowspace.model import (Variable, Constant,
                                      c_last_exception, checkgraph, mkentrymap)
 from rpython.flowspace.operation import OverflowingOperation, op
 from rpython.rlib import rarithmetic
 from rpython.translator import unsimplify
-from rpython.translator.backendopt import ssa
 from rpython.rtyper.lltypesystem import lloperation, lltype
+from rpython.translator.backendopt.ssa import (
+        SSA_to_SSI, DataFlowFamilyBuilder)
 
 def get_graph(arg, translator):
     if isinstance(arg, Variable):
@@ -75,10 +77,12 @@
                 outputargs = []
                 for v in exit.args:
                     if isinstance(v, Variable):
-                        # this variable is valid in the context of block1
-                        # but it must come from 'link'
-                        i = block1.inputargs.index(v)
-                        v = link.args[i]
+                        try:
+                            i = block1.inputargs.index(v)
+                            v = link.args[i]
+                        except ValueError:
+                            # the variable was passed implicitly to block1
+                            pass
                     outputargs.append(v)
                 link.args = outputargs
                 link.target = exit.target
@@ -243,6 +247,59 @@
             seen.append(case)
         block.recloseblock(*exits)
 
+def constfold_exitswitch(graph):
+    """Remove trivial links by merging their source and target blocks
+
+    A link is trivial if it has no arguments, is the single exit of its
+    source and the single parent of its target.
+    """
+    block = graph.startblock
+    seen = set([block])
+    stack = list(block.exits)
+    while stack:
+        link = stack.pop()
+        target = link.target
+        if target in seen:
+            continue
+        source = link.prevblock
+        switch = source.exitswitch
+        if (isinstance(switch, Constant) and switch != c_last_exception):
+            exits = replace_exitswitch_by_constant(source, switch)
+            stack.extend(exits)
+        else:
+            seen.add(target)
+            stack.extend(target.exits)
+
+
+def remove_trivial_links(graph):
+    """Remove trivial links by merging their source and target blocks
+
+    A link is trivial if it has no arguments, is the single exit of its
+    source and the single parent of its target.
+    """
+    entrymap = mkentrymap(graph)
+    block = graph.startblock
+    seen = set([block])
+    stack = list(block.exits)
+    while stack:
+        link = stack.pop()
+        if link.target in seen:
+            continue
+        source = link.prevblock
+        target = link.target
+        if (not link.args and source.exitswitch is None and
+                len(entrymap[target]) == 1 and
+                target.exits):  # stop at the returnblock
+            assert len(source.exits) == 1
+            source.operations.extend(target.operations)
+            source.exitswitch = newexitswitch = target.exitswitch
+            source.recloseblock(*target.exits)
+            stack.extend(source.exits)
+        else:
+            seen.add(target)
+            stack.extend(target.exits)
+
+
 def join_blocks(graph):
     """Links can be deleted if they are the single exit of a block and
     the single entry point of the next block.  When this happens, we can
@@ -504,6 +561,77 @@
             if block.inputargs[i] not in read_vars:
                 del block.inputargs[i]
 
+class Representative(object):
+    def __init__(self, var):
+        self.rep = var
+
+    def absorb(self, other):
+        pass
+
+def all_equal(lst):
+    first = lst[0]
+    return all(first == x for x in lst[1:])
+
+def isspecialvar(v):
+    return isinstance(v, Variable) and v._name in ('last_exception_', 'last_exc_value_')
+
+def remove_identical_vars_SSA(graph):
+    """When the same variable is passed multiple times into the next block,
+    pass it only once.  This enables further optimizations by the annotator,
+    which otherwise doesn't realize that tests performed on one of the copies
+    of the variable also affect the other."""
+    uf = UnionFind(Representative)
+    entrymap = mkentrymap(graph)
+    del entrymap[graph.startblock]
+    entrymap.pop(graph.returnblock, None)
+    entrymap.pop(graph.exceptblock, None)
+    inputs = {}
+    for block, links in entrymap.items():
+        phis = zip(block.inputargs, zip(*[link.args for link in links]))
+        inputs[block] = phis
+
+    def simplify_phis(block):
+        phis = inputs[block]
+        to_remove = []
+        unique_phis = {}
+        for i, (input, phi_args) in enumerate(phis):
+            new_args = [uf.find_rep(arg) for arg in phi_args]
+            if all_equal(new_args) and not isspecialvar(new_args[0]):
+                uf.union(new_args[0], input)
+                to_remove.append(i)
+            else:
+                t = tuple(new_args)
+                if t in unique_phis:
+                    uf.union(unique_phis[t], input)
+                    to_remove.append(i)
+                else:
+                    unique_phis[t] = input
+        for i in reversed(to_remove):
+            del phis[i]
+        return bool(to_remove)
+
+    progress = True
+    while progress:
+        progress = False
+        for block in inputs:
+            if simplify_phis(block):
+                progress = True
+
+    renaming = {key: uf[key].rep for key in uf}
+    for block, links in entrymap.items():
+        if inputs[block]:
+            new_inputs, new_args = zip(*inputs[block])
+            new_args = map(list, zip(*new_args))
+        else:
+            new_inputs = []
+            new_args = [[] for _ in links]
+        block.inputargs = new_inputs
+        assert len(links) == len(new_args)
+        for link, args in zip(links, new_args):
+            link.args = args
+    for block in graph.iterblocks():
+        block.renamevariables(renaming)
+
 def remove_identical_vars(graph):
     """When the same variable is passed multiple times into the next block,
     pass it only once.  This enables further optimizations by the annotator,
@@ -530,7 +658,7 @@
     #    when for all possible incoming paths they would get twice the same
     #    value (this is really the purpose of remove_identical_vars()).
     #
-    builder = ssa.DataFlowFamilyBuilder(graph)
+    builder = DataFlowFamilyBuilder(graph)
     variable_families = builder.get_variable_families()  # vertical removal
     while True:
         if not builder.merge_identical_phi_nodes():    # horizontal removal
@@ -625,7 +753,7 @@
     # NB. this assumes RPythonicity: we can only iterate over something
     # that has a len(), and this len() cannot change as long as we are
     # using the iterator.
-    builder = ssa.DataFlowFamilyBuilder(graph)
+    builder = DataFlowFamilyBuilder(graph)
     variable_families = builder.get_variable_families()
     c_append = Constant('append')
     newlist_v = {}
@@ -958,12 +1086,14 @@
 # ____ all passes & simplify_graph
 
 all_passes = [
+    transform_dead_op_vars,
     eliminate_empty_blocks,
     remove_assertion_errors,
-    join_blocks,
+    remove_identical_vars_SSA,
+    constfold_exitswitch,
+    remove_trivial_links,
+    SSA_to_SSI,
     coalesce_bool,
-    transform_dead_op_vars,
-    remove_identical_vars,
     transform_ovfcheck,
     simplify_exceptions,
     transform_xxxitem,
@@ -974,7 +1104,6 @@
     """inplace-apply all the existing optimisations to the graph."""
     if passes is True:
         passes = all_passes
-    checkgraph(graph)
     for pass_ in passes:
         pass_(graph)
     checkgraph(graph)
diff --git a/rpython/translator/tool/make_dot.py b/rpython/translator/tool/make_dot.py
--- a/rpython/translator/tool/make_dot.py
+++ b/rpython/translator/tool/make_dot.py
@@ -1,7 +1,6 @@
 import os
 import inspect, linecache
 from rpython.flowspace.model import *
-from rpython.flowspace.objspace import build_flow
 from rpython.tool.udir import udir
 from py.process import cmdexec
 from rpython.tool.error import offset2lineno
@@ -238,14 +237,3 @@
     # not a keyword
     name = ''.join([CHAR_MAP[c] for c in name])
     return '_' + name
-
-
-if __name__ == '__main__':
-    def f(x):
-        i = 0
-        while i < x:
-            i += 1
-        return i
-
-    graph = build_flow(f)
-    make_dot('f', graph)

From noreply at buildbot.pypy.org  Sun Nov 16 21:24:43 2014
From: noreply at buildbot.pypy.org (rlamy)
Date: Sun, 16 Nov 2014 21:24:43 +0100 (CET)
Subject: [pypy-commit] pypy default: extract the costly call to
 check_no_flags() in a separate method
Message-ID: <20141116202443.685111D289E@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy 
Branch: 
Changeset: r74541:12ca083b8119
Date: 2014-11-16 20:24 +0000
http://bitbucket.org/pypy/pypy/changeset/12ca083b8119/

Log:	extract the costly call to check_no_flags() in a separate method

diff --git a/rpython/annotator/annrpython.py b/rpython/annotator/annrpython.py
--- a/rpython/annotator/annrpython.py
+++ b/rpython/annotator/annrpython.py
@@ -223,6 +223,7 @@
                 self.setbinding(v, annmodel.s_ImpossibleValue)
         # policy-dependent computation
         self.bookkeeper.compute_at_fixpoint()
+        self.bookkeeper.check_no_flags_on_instances()
 
     def annotation(self, arg):
         "Gives the SomeValue corresponding to the given Variable or Constant."
diff --git a/rpython/annotator/bookkeeper.py b/rpython/annotator/bookkeeper.py
--- a/rpython/annotator/bookkeeper.py
+++ b/rpython/annotator/bookkeeper.py
@@ -110,8 +110,8 @@
         finally:
             self.leave()
 
+    def check_no_flags_on_instances(self):
         # sanity check: no flags attached to heap stored instances
-
         seen = set()
 
         def check_no_flags(s_value_or_def):

From noreply at buildbot.pypy.org  Sun Nov 16 21:38:23 2014
From: noreply at buildbot.pypy.org (rlamy)
Date: Sun, 16 Nov 2014 21:38:23 +0100 (CET)
Subject: [pypy-commit] pypy default: Call the expensive check_no_flags()
	only in tests
Message-ID: <20141116203823.A91541D28C5@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy 
Branch: 
Changeset: r74542:d8042a6b7573
Date: 2014-11-16 20:38 +0000
http://bitbucket.org/pypy/pypy/changeset/d8042a6b7573/

Log:	Call the expensive check_no_flags() only in tests

diff --git a/rpython/annotator/annrpython.py b/rpython/annotator/annrpython.py
--- a/rpython/annotator/annrpython.py
+++ b/rpython/annotator/annrpython.py
@@ -223,6 +223,9 @@
                 self.setbinding(v, annmodel.s_ImpossibleValue)
         # policy-dependent computation
         self.bookkeeper.compute_at_fixpoint()
+
+    def validate(self):
+        """Check that the annotation results are valid"""
         self.bookkeeper.check_no_flags_on_instances()
 
     def annotation(self, arg):
diff --git a/rpython/annotator/test/test_annrpython.py b/rpython/annotator/test/test_annrpython.py
--- a/rpython/annotator/test/test_annrpython.py
+++ b/rpython/annotator/test/test_annrpython.py
@@ -48,6 +48,7 @@
     class RPythonAnnotator(_RPythonAnnotator):
         def build_types(self, *args):
             s = _RPythonAnnotator.build_types(self, *args)
+            self.validate()
             if option.view:
                 self.translator.view()
             return s
diff --git a/rpython/rtyper/test/test_llinterp.py b/rpython/rtyper/test/test_llinterp.py
--- a/rpython/rtyper/test/test_llinterp.py
+++ b/rpython/rtyper/test/test_llinterp.py
@@ -41,6 +41,7 @@
     t.config.set(**extraconfigopts)
     a = t.buildannotator(policy=policy)
     timelog("annotating", a.build_types, func, argtypes, main_entry_point=True)
+    a.validate()
     if viewbefore == 'auto':
         viewbefore = getattr(option, 'view', False)
     if viewbefore:

From noreply at buildbot.pypy.org  Sun Nov 16 23:33:16 2014
From: noreply at buildbot.pypy.org (rlamy)
Date: Sun, 16 Nov 2014 23:33:16 +0100 (CET)
Subject: [pypy-commit] pypy default: remove expensive check
Message-ID: <20141116223316.78F2B1D26C4@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy 
Branch: 
Changeset: r74543:6468becae2f6
Date: 2014-11-16 22:12 +0000
http://bitbucket.org/pypy/pypy/changeset/6468becae2f6/

Log:	remove expensive check

	Cuts backendopt time by ~30% on CPython and ~10% on PyPy

diff --git a/rpython/translator/backendopt/merge_if_blocks.py b/rpython/translator/backendopt/merge_if_blocks.py
--- a/rpython/translator/backendopt/merge_if_blocks.py
+++ b/rpython/translator/backendopt/merge_if_blocks.py
@@ -110,7 +110,6 @@
     else:
         return False
     merge_chain(chain, checkvars[0], varmap, graph)
-    checkgraph(graph)
     return True
 
 def merge_if_blocks(graph, verbose=True):

From noreply at buildbot.pypy.org  Sun Nov 16 23:35:40 2014
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Sun, 16 Nov 2014 23:35:40 +0100 (CET)
Subject: [pypy-commit] pypy default: remove unused import
Message-ID: <20141116223540.CF4981D26C4@cobra.cs.uni-duesseldorf.de>

Author: Alex Gaynor 
Branch: 
Changeset: r74544:88db3405c7e3
Date: 2014-11-16 14:35 -0800
http://bitbucket.org/pypy/pypy/changeset/88db3405c7e3/

Log:	remove unused import

diff --git a/rpython/translator/backendopt/merge_if_blocks.py b/rpython/translator/backendopt/merge_if_blocks.py
--- a/rpython/translator/backendopt/merge_if_blocks.py
+++ b/rpython/translator/backendopt/merge_if_blocks.py
@@ -1,4 +1,4 @@
-from rpython.flowspace.model import Constant, Variable, checkgraph, mkentrymap
+from rpython.flowspace.model import Constant, Variable, mkentrymap
 from rpython.translator.backendopt.support import log
 
 log = log.mergeifblocks

From noreply at buildbot.pypy.org  Mon Nov 17 00:41:35 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 17 Nov 2014 00:41:35 +0100 (CET)
Subject: [pypy-commit] cffi default: update docs
Message-ID: <20141116234135.BBDD01C0F1B@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r1572:3d09f5a1d3d1
Date: 2014-11-17 00:41 +0100
http://bitbucket.org/cffi/cffi/changeset/3d09f5a1d3d1/

Log:	update docs

diff --git a/doc/source/index.rst b/doc/source/index.rst
--- a/doc/source/index.rst
+++ b/doc/source/index.rst
@@ -859,7 +859,9 @@
 
 .. versionadded:: 0.8.2
    The ``ffi.cdef()`` call takes an optional argument ``packed``: if
-   True, then all structs declared within this cdef are "packed".  This
+   True, then all structs declared within this cdef are "packed".
+   (If you need both packed and non-packed structs,
+   use several cdefs in sequence.)  This
    has a meaning similar to ``__attribute__((packed))`` in GCC.  It
    specifies that all structure fields should have an alignment of one
    byte.  (Note that the packed attribute has no effect on bit fields so

From noreply at buildbot.pypy.org  Mon Nov 17 00:41:49 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 17 Nov 2014 00:41:49 +0100 (CET)
Subject: [pypy-commit] cffi default: Update the TODO
Message-ID: <20141116234149.463311C0F1B@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r1573:e1c6209aa8c4
Date: 2014-11-17 00:42 +0100
http://bitbucket.org/cffi/cffi/changeset/e1c6209aa8c4/

Log:	Update the TODO

diff --git a/TODO b/TODO
--- a/TODO
+++ b/TODO
@@ -1,10 +1,3 @@
 
 
-Next steps
-----------
-
-verify() handles "typedef ... some_integer_type", but this creates
-an opaque type that works like a struct (so we can't get the value
-out of it).
-
-accept and kill "static inline" in the cdefs
+Add other required types from stdint.h

From noreply at buildbot.pypy.org  Mon Nov 17 09:52:16 2014
From: noreply at buildbot.pypy.org (mattip)
Date: Mon, 17 Nov 2014 09:52:16 +0100 (CET)
Subject: [pypy-commit] pypy default: move decoration to declaration not
	definition for MSVC
Message-ID: <20141117085216.C9F9D1C1147@cobra.cs.uni-duesseldorf.de>

Author: mattip 
Branch: 
Changeset: r74545:a9c38f12f2a6
Date: 2014-11-17 09:41 +0100
http://bitbucket.org/pypy/pypy/changeset/a9c38f12f2a6/

Log:	move decoration to declaration not definition for MSVC

diff --git a/pypy/module/cppyy/include/capi.h b/pypy/module/cppyy/include/capi.h
--- a/pypy/module/cppyy/include/capi.h
+++ b/pypy/module/cppyy/include/capi.h
@@ -2,6 +2,7 @@
 #define CPPYY_CAPI
 
 #include 
+#include "src/precommondefs.h"
 
 #ifdef __cplusplus
 extern "C" {
@@ -15,102 +16,167 @@
     typedef void* (*cppyy_methptrgetter_t)(cppyy_object_t);
 
     /* name to opaque C++ scope representation -------------------------------- */
+    RPY_EXTERN
     int cppyy_num_scopes(cppyy_scope_t parent);
+    RPY_EXTERN
     char* cppyy_scope_name(cppyy_scope_t parent, int iscope);
 
+    RPY_EXTERN
     char* cppyy_resolve_name(const char* cppitem_name);
+    RPY_EXTERN
     cppyy_scope_t cppyy_get_scope(const char* scope_name);
+    RPY_EXTERN
     cppyy_type_t cppyy_get_template(const char* template_name);
+    RPY_EXTERN
     cppyy_type_t cppyy_actual_class(cppyy_type_t klass, cppyy_object_t obj);
 
     /* memory management ------------------------------------------------------ */
+    RPY_EXTERN
     cppyy_object_t cppyy_allocate(cppyy_type_t type);
+    RPY_EXTERN
     void cppyy_deallocate(cppyy_type_t type, cppyy_object_t self);
+    RPY_EXTERN
     void cppyy_destruct(cppyy_type_t type, cppyy_object_t self);
 
     /* method/function dispatching -------------------------------------------- */
+    RPY_EXTERN
     void   cppyy_call_v(cppyy_method_t method, cppyy_object_t self, int nargs, void* args);
+    RPY_EXTERN
     unsigned char cppyy_call_b(cppyy_method_t method, cppyy_object_t self, int nargs, void* args);
+    RPY_EXTERN
     char   cppyy_call_c(cppyy_method_t method, cppyy_object_t self, int nargs, void* args);
+    RPY_EXTERN
     short  cppyy_call_h(cppyy_method_t method, cppyy_object_t self, int nargs, void* args);
+    RPY_EXTERN
     int    cppyy_call_i(cppyy_method_t method, cppyy_object_t self, int nargs, void* args);
+    RPY_EXTERN
     long   cppyy_call_l(cppyy_method_t method, cppyy_object_t self, int nargs, void* args);
+    RPY_EXTERN
     long long cppyy_call_ll(cppyy_method_t method, cppyy_object_t self, int nargs, void* args);
+    RPY_EXTERN
     float  cppyy_call_f(cppyy_method_t method, cppyy_object_t self, int nargs, void* args);
+    RPY_EXTERN
     double cppyy_call_d(cppyy_method_t method, cppyy_object_t self, int nargs, void* args);
 
+    RPY_EXTERN
     void*  cppyy_call_r(cppyy_method_t method, cppyy_object_t self, int nargs, void* args);
+    RPY_EXTERN
     char*  cppyy_call_s(cppyy_method_t method, cppyy_object_t self, int nargs, void* args);
 
+    RPY_EXTERN
     cppyy_object_t cppyy_constructor(cppyy_method_t method, cppyy_type_t klass, int nargs, void* args);
+    RPY_EXTERN
     cppyy_object_t cppyy_call_o(cppyy_method_t method, cppyy_object_t self, int nargs, void* args, cppyy_type_t result_type);
 
+    RPY_EXTERN
     cppyy_methptrgetter_t cppyy_get_methptr_getter(cppyy_scope_t scope, cppyy_index_t idx);
 
     /* handling of function argument buffer ----------------------------------- */
+    RPY_EXTERN
     void*  cppyy_allocate_function_args(int nargs);
+    RPY_EXTERN
     void   cppyy_deallocate_function_args(void* args);
+    RPY_EXTERN
     size_t cppyy_function_arg_sizeof();
+    RPY_EXTERN
     size_t cppyy_function_arg_typeoffset();
 
     /* scope reflection information ------------------------------------------- */
+    RPY_EXTERN
     int cppyy_is_namespace(cppyy_scope_t scope);
+    RPY_EXTERN
     int cppyy_is_enum(const char* type_name);
 
     /* class reflection information ------------------------------------------- */
+    RPY_EXTERN
     char* cppyy_final_name(cppyy_type_t type);
+    RPY_EXTERN
     char* cppyy_scoped_final_name(cppyy_type_t type);
+    RPY_EXTERN
     int cppyy_has_complex_hierarchy(cppyy_type_t type);
+    RPY_EXTERN
     int cppyy_num_bases(cppyy_type_t type);
+    RPY_EXTERN
     char* cppyy_base_name(cppyy_type_t type, int base_index);
+    RPY_EXTERN
     int cppyy_is_subtype(cppyy_type_t derived, cppyy_type_t base);
 
     /* calculate offsets between declared and actual type, up-cast: direction > 0; down-cast: direction < 0 */
+    RPY_EXTERN
     ptrdiff_t cppyy_base_offset(cppyy_type_t derived, cppyy_type_t base, cppyy_object_t address, int direction);
 
     /* method/function reflection information --------------------------------- */
+    RPY_EXTERN
     int cppyy_num_methods(cppyy_scope_t scope);
+    RPY_EXTERN
     cppyy_index_t cppyy_method_index_at(cppyy_scope_t scope, int imeth);
+    RPY_EXTERN
     cppyy_index_t* cppyy_method_indices_from_name(cppyy_scope_t scope, const char* name);
 
+    RPY_EXTERN
     char* cppyy_method_name(cppyy_scope_t scope, cppyy_index_t idx);
+    RPY_EXTERN
     char* cppyy_method_result_type(cppyy_scope_t scope, cppyy_index_t idx);
+    RPY_EXTERN
     int cppyy_method_num_args(cppyy_scope_t scope, cppyy_index_t idx);
+    RPY_EXTERN
     int cppyy_method_req_args(cppyy_scope_t scope, cppyy_index_t idx);
+    RPY_EXTERN
     char* cppyy_method_arg_type(cppyy_scope_t scope, cppyy_index_t idx, int arg_index);
+    RPY_EXTERN
     char* cppyy_method_arg_default(cppyy_scope_t scope, cppyy_index_t idx, int arg_index);
+    RPY_EXTERN
     char* cppyy_method_signature(cppyy_scope_t scope, cppyy_index_t idx);
 
+    RPY_EXTERN
     int cppyy_method_is_template(cppyy_scope_t scope, cppyy_index_t idx);
+    RPY_EXTERN
     int cppyy_method_num_template_args(cppyy_scope_t scope, cppyy_index_t idx);
+    RPY_EXTERN
     char* cppyy_method_template_arg_name(cppyy_scope_t scope, cppyy_index_t idx, cppyy_index_t iarg);
 
+    RPY_EXTERN
     cppyy_method_t cppyy_get_method(cppyy_scope_t scope, cppyy_index_t idx);
+    RPY_EXTERN
     cppyy_index_t cppyy_get_global_operator(
         cppyy_scope_t scope, cppyy_scope_t lc, cppyy_scope_t rc, const char* op);
 
     /* method properties ------------------------------------------------------ */
+    RPY_EXTERN
     int cppyy_is_constructor(cppyy_type_t type, cppyy_index_t idx);
+    RPY_EXTERN
     int cppyy_is_staticmethod(cppyy_type_t type, cppyy_index_t idx);
 
     /* data member reflection information ------------------------------------- */
+    RPY_EXTERN
     int cppyy_num_datamembers(cppyy_scope_t scope);
+    RPY_EXTERN
     char* cppyy_datamember_name(cppyy_scope_t scope, int datamember_index);
+    RPY_EXTERN
     char* cppyy_datamember_type(cppyy_scope_t scope, int datamember_index);
+    RPY_EXTERN
     ptrdiff_t cppyy_datamember_offset(cppyy_scope_t scope, int datamember_index);
 
+    RPY_EXTERN
     int cppyy_datamember_index(cppyy_scope_t scope, const char* name);
 
     /* data member properties ------------------------------------------------- */
+    RPY_EXTERN
     int cppyy_is_publicdata(cppyy_type_t type, int datamember_index);
+    RPY_EXTERN
     int cppyy_is_staticdata(cppyy_type_t type, int datamember_index);
 
     /* misc helpers ----------------------------------------------------------- */
+    RPY_EXTERN
     long long cppyy_strtoll(const char* str);
+    RPY_EXTERN
     unsigned long long cppyy_strtoull(const char* str);
+    RPY_EXTERN
     void cppyy_free(void* ptr);
 
+    RPY_EXTERN
     cppyy_object_t cppyy_charp2stdstring(const char* str);
+    RPY_EXTERN
     cppyy_object_t cppyy_stdstring2stdstring(cppyy_object_t ptr);
 
 #ifdef __cplusplus
diff --git a/pypy/module/cppyy/src/dummy_backend.cxx b/pypy/module/cppyy/src/dummy_backend.cxx
--- a/pypy/module/cppyy/src/dummy_backend.cxx
+++ b/pypy/module/cppyy/src/dummy_backend.cxx
@@ -1,4 +1,3 @@
-#include "src/precommondefs.h"
 #include "cppyy.h"
 #include "capi.h"
 
@@ -349,29 +348,24 @@
 
 
 /* name to opaque C++ scope representation -------------------------------- */
-RPY_EXTERN
 int cppyy_num_scopes(cppyy_scope_t handle) {
     return 0;
 }
 
-RPY_EXTERN
 char* cppyy_resolve_name(const char* cppitem_name) {
     return cppstring_to_cstring(cppitem_name);
 }
 
-RPY_EXTERN
 cppyy_scope_t cppyy_get_scope(const char* scope_name) {
     return s_handles[scope_name];  // lookup failure will return 0 (== error)
 }
 
-RPY_EXTERN
 cppyy_type_t cppyy_actual_class(cppyy_type_t klass, cppyy_object_t /* obj */) {
     return klass;
 }
 
 
 /* memory management ------------------------------------------------------ */
-RPY_EXTERN
 void cppyy_destruct(cppyy_type_t handle, cppyy_object_t self) {
     if (handle == s_handles["example01"])
        delete (dummy::example01*)self;
@@ -379,7 +373,6 @@
 
 
 /* method/function dispatching -------------------------------------------- */
-RPY_EXTERN
 void cppyy_call_v(cppyy_method_t method, cppyy_object_t self, int nargs, void* args) {
     long idx = (long)method;
     if (idx == s_methods["static_example01::staticSetPayload_payload*_double"]) {
@@ -469,7 +462,6 @@
     }
 }
 
-RPY_EXTERN
 unsigned char cppyy_call_b(cppyy_method_t method, cppyy_object_t self, int nargs, void* args) {
     unsigned char result = 0;
     const long idx = (long)method;
@@ -482,7 +474,6 @@
     return result;
 }
 
-RPY_EXTERN
 char cppyy_call_c(cppyy_method_t method, cppyy_object_t self, int nargs, void* args) {
     char result = 0;
     const long idx = (long)method;
@@ -498,7 +489,6 @@
     return result;
 }
 
-RPY_EXTERN
 short cppyy_call_h(cppyy_method_t method, cppyy_object_t self, int nargs, void* args) {
     short result = 0;
     const long idx = (long)method; 
@@ -514,7 +504,6 @@
     return result;
 }
 
-RPY_EXTERN
 int cppyy_call_i(cppyy_method_t method, cppyy_object_t self, int nargs, void* args) {
     int result = 0;
     const long idx = (long)method;
@@ -547,7 +536,6 @@
     return result;
 }
 
-RPY_EXTERN
 long cppyy_call_l(cppyy_method_t method, cppyy_object_t self, int nargs, void* args) {
     long result = 0;
     const long idx = (long)method;
@@ -689,7 +677,6 @@
     return result;
 }
 
-RPY_EXTERN
 long long cppyy_call_ll(cppyy_method_t method, cppyy_object_t self, int nargs, void* args) {
     long long result = 0;
     const long idx = (long)method;
@@ -705,7 +692,6 @@
     return result;
 }   
 
-RPY_EXTERN
 float cppyy_call_f(cppyy_method_t method, cppyy_object_t self, int nargs, void* args) {
     float result = 0;
     const long idx = (long)method;
@@ -718,7 +704,6 @@
     return result;
 }   
 
-RPY_EXTERN
 double cppyy_call_d(cppyy_method_t method, cppyy_object_t self, int nargs, void* args) {
     double result = 0.;
     const long idx = (long)method;
@@ -740,7 +725,6 @@
     return result;
 }
 
-RPY_EXTERN
 char* cppyy_call_s(cppyy_method_t method, cppyy_object_t self, int nargs, void* args) {
     char* result = 0;
     const long idx = (long)method;
@@ -753,7 +737,6 @@
     return result;
 }
 
-RPY_EXTERN
 cppyy_object_t cppyy_constructor(cppyy_method_t method, cppyy_type_t handle, int nargs, void* args) {
     void* result = 0;
     const long idx = (long)method;
@@ -776,14 +759,12 @@
     return (cppyy_object_t)result;
 }
 
-RPY_EXTERN
 cppyy_methptrgetter_t cppyy_get_methptr_getter(cppyy_type_t /* handle */, cppyy_index_t /* method_index */) {
     return (cppyy_methptrgetter_t)0;
 }
 
 
 /* handling of function argument buffer ----------------------------------- */
-RPY_EXTERN
 void* cppyy_allocate_function_args(int nargs) {
     CPPYY_G__value* args = (CPPYY_G__value*)malloc(nargs*sizeof(CPPYY_G__value));
     for (int i = 0; i < nargs; ++i)
@@ -793,36 +774,30 @@
 
 
 /* handling of function argument buffer ----------------------------------- */
-RPY_EXTERN
 void cppyy_deallocate_function_args(void* args) {
     free(args);
 }
 
-RPY_EXTERN
 size_t cppyy_function_arg_sizeof() {
     return sizeof(CPPYY_G__value);
 }
 
-RPY_EXTERN
 size_t cppyy_function_arg_typeoffset() {
     return offsetof(CPPYY_G__value, type);
 }
 
 
 /* scope reflection information ------------------------------------------- */
-RPY_EXTERN
 int cppyy_is_namespace(cppyy_scope_t /* handle */) {
     return 0;
 }   
 
-RPY_EXTERN
 int cppyy_is_enum(const char* /* type_name */) {
     return 0;
 }
     
     
 /* class reflection information ------------------------------------------- */
-RPY_EXTERN
 char* cppyy_final_name(cppyy_type_t handle) {
     for (Handles_t::iterator isp = s_handles.begin(); isp != s_handles.end(); ++isp) {
         if (isp->second == handle)
@@ -831,75 +806,61 @@
     return cppstring_to_cstring("");
 }
 
-RPY_EXTERN
 char* cppyy_scoped_final_name(cppyy_type_t handle) {
     return cppyy_final_name(handle);
 }   
 
-RPY_EXTERN
 int cppyy_has_complex_hierarchy(cppyy_type_t /* handle */) {
     return 0;
 }
 
-RPY_EXTERN
 int cppyy_num_bases(cppyy_type_t /*handle*/) {
    return 0;
 }
 
 
 /* method/function reflection information --------------------------------- */
-RPY_EXTERN
 int cppyy_num_methods(cppyy_scope_t handle) {
     return s_scopes[handle].m_methods.size();
 }
 
-RPY_EXTERN
 cppyy_index_t cppyy_method_index_at(cppyy_scope_t /* scope */, int imeth) {
     return (cppyy_index_t)imeth;
 }
 
-RPY_EXTERN
 char* cppyy_method_name(cppyy_scope_t handle, cppyy_index_t method_index) {
     return cppstring_to_cstring(s_scopes[handle].m_methods[(int)method_index].m_name);
 }
 
-RPY_EXTERN
 char* cppyy_method_result_type(cppyy_scope_t handle, cppyy_index_t method_index) {
     return cppstring_to_cstring(s_scopes[handle].m_methods[method_index].m_returntype);
 }
     
-RPY_EXTERN
 int cppyy_method_num_args(cppyy_scope_t handle, cppyy_index_t method_index) {
     return s_scopes[handle].m_methods[method_index].m_argtypes.size();
 }
 
-RPY_EXTERN
 int cppyy_method_req_args(cppyy_scope_t handle, cppyy_index_t method_index) {
     return cppyy_method_num_args(handle, method_index);
 }
 
-RPY_EXTERN
 char* cppyy_method_arg_type(cppyy_scope_t handle, cppyy_index_t method_index, int arg_index) {
     return cppstring_to_cstring(s_scopes[handle].m_methods[method_index].m_argtypes[arg_index]);
 }
 
-RPY_EXTERN
 char* cppyy_method_arg_default(
         cppyy_scope_t /* handle */, cppyy_index_t /* method_index */, int /* arg_index */) {
     return cppstring_to_cstring("");
 }
 
-RPY_EXTERN
 char* cppyy_method_signature(cppyy_scope_t /* handle */, cppyy_index_t /* method_index */) {
     return cppstring_to_cstring("");
 }
 
-RPY_EXTERN
 int cppyy_method_is_template(cppyy_scope_t /* handle */, cppyy_index_t /* method_index */) {
     return 0;
 }
     
-RPY_EXTERN
 cppyy_method_t cppyy_get_method(cppyy_scope_t handle, cppyy_index_t method_index) {
     if (s_scopes.find(handle) != s_scopes.end()) {
         long id = s_scopes[handle].m_method_offset + (long)method_index;
@@ -911,7 +872,6 @@
 
 
 /* method properties -----------------------------------------------------  */
-RPY_EXTERN
 int cppyy_is_constructor(cppyy_type_t handle, cppyy_index_t method_index) {
     if (s_scopes.find(handle) != s_scopes.end())
         return s_scopes[handle].m_methods[method_index].m_type == kConstructor;
@@ -919,7 +879,6 @@
     return 0;
 }
 
-RPY_EXTERN
 int cppyy_is_staticmethod(cppyy_type_t handle, cppyy_index_t method_index) {
     if (s_scopes.find(handle) != s_scopes.end())
         return s_scopes[handle].m_methods[method_index].m_type == kStatic;
@@ -929,34 +888,28 @@
 
 
 /* data member reflection information ------------------------------------- */
-RPY_EXTERN
 int cppyy_num_datamembers(cppyy_scope_t handle) {
     return s_scopes[handle].m_datambrs.size();
 }
 
-RPY_EXTERN
 char* cppyy_datamember_name(cppyy_scope_t handle, int idatambr) {
     return cppstring_to_cstring(s_scopes[handle].m_datambrs[idatambr].m_name);
 }
 
-RPY_EXTERN
 char* cppyy_datamember_type(cppyy_scope_t handle, int idatambr) {
     return cppstring_to_cstring(s_scopes[handle].m_datambrs[idatambr].m_type);
 }
 
-RPY_EXTERN
 ptrdiff_t cppyy_datamember_offset(cppyy_scope_t handle, int idatambr) {
     return s_scopes[handle].m_datambrs[idatambr].m_offset;
 }
 
 
 /* data member properties ------------------------------------------------  */
-RPY_EXTERN
 int cppyy_is_publicdata(cppyy_scope_t handle, int idatambr) {
     return 1;
 }
 
-RPY_EXTERN
 int cppyy_is_staticdata(cppyy_scope_t handle, int idatambr) {
     return s_scopes[handle].m_datambrs[idatambr].m_isstatic;
 }
@@ -964,44 +917,37 @@
 
 /* misc helpers ----------------------------------------------------------- */
 #if defined(_MSC_VER)
-RPY_EXTERN
 long long cppyy_strtoll(const char* str) {
     return _strtoi64(str, NULL, 0);
 }
 
 extern "C" {
-RPY_EXTERN
 unsigned long long cppyy_strtoull(const char* str) {
     return _strtoui64(str, NULL, 0);
 }
 }
 #else
-RPY_EXTERN
 long long cppyy_strtoll(const char* str) {
     return strtoll(str, NULL, 0);
 }
 
 extern "C" {
-RPY_EXTERN
 unsigned long long cppyy_strtoull(const char* str) {
     return strtoull(str, NULL, 0);
 }
 }
 #endif
 
-RPY_EXTERN
 void cppyy_free(void* ptr) {
     free(ptr);
 }
 
-RPY_EXTERN
 cppyy_object_t cppyy_charp2stdstring(const char* str) {
     void* arena = new char[sizeof(std::string)];
     new (arena) std::string(str);
     return (cppyy_object_t)arena;
 }
 
-RPY_EXTERN
 cppyy_object_t cppyy_stdstring2stdstring(cppyy_object_t ptr) {
     void* arena = new char[sizeof(std::string)];
     new (arena) std::string(*(std::string*)ptr);

From noreply at buildbot.pypy.org  Mon Nov 17 11:03:42 2014
From: noreply at buildbot.pypy.org (fijal)
Date: Mon, 17 Nov 2014 11:03:42 +0100 (CET)
Subject: [pypy-commit] pypy optresult: hack on failargs a bit
Message-ID: <20141117100342.911E41C0F1B@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: optresult
Changeset: r74546:10ebae54d9f8
Date: 2014-11-17 09:55 +0200
http://bitbucket.org/pypy/pypy/changeset/10ebae54d9f8/

Log:	hack on failargs a bit

diff --git a/rpython/jit/metainterp/optimizeopt/rewrite.py b/rpython/jit/metainterp/optimizeopt/rewrite.py
--- a/rpython/jit/metainterp/optimizeopt/rewrite.py
+++ b/rpython/jit/metainterp/optimizeopt/rewrite.py
@@ -315,6 +315,7 @@
                     raise InvalidLoop('A GUARD_VALUE (%s) was proven to always fail' % r)
             op = old_guard_op.copy_and_change(rop.GUARD_VALUE,
                                       args = [old_guard_op.getarg(0), op.getarg(1)])
+            self.getvalue(old_guard_op).box = op
             self.optimizer.replaces_guard[op] = old_guard_op
             # hack hack hack.  Change the guard_opnum on
             # new_guard_op.getdescr() so that when resuming,
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_util.py b/rpython/jit/metainterp/optimizeopt/test/test_util.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_util.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_util.py
@@ -338,11 +338,21 @@
 
 class Storage(compile.ResumeGuardDescr):
     "for tests."
+    def __init__(self, metainterp_sd=None, original_greenkey=None):
+        self.metainterp_sd = metainterp_sd
+        self.original_greenkey = original_greenkey
+    def store_final_boxes(self, op, boxes, metainterp_sd):
+        op.setfailargs(boxes)
+    def __eq__(self, other):
+        return type(self) is type(other)      # xxx obscure
     def clone_if_mutable(self):
-        return Storage()
+        res = Storage(self.metainterp_sd, self.original_greenkey)
+        self.copy_all_attributes_into(res)
+        return res
 
-    def store_final_boxes(self, *args):
-        pass
+def _sortboxes(boxes):
+    _kind2count = {history.INT: 1, history.REF: 2, history.FLOAT: 3}
+    return sorted(boxes, key=lambda box: _kind2count[box.type])
 
 class BaseTest(object):
 
@@ -366,6 +376,8 @@
         if fail_args is None:
             return None
         descr = Storage()
+        descr.rd_frame_info_list = resume.FrameInfo(None, "code", 11)
+        descr.rd_snapshot = resume.Snapshot(None, _sortboxes(fail_args))
         return descr
 
     def assert_equal(self, optimized, expected, text_right=None):
diff --git a/rpython/jit/metainterp/optimizeopt/util.py b/rpython/jit/metainterp/optimizeopt/util.py
--- a/rpython/jit/metainterp/optimizeopt/util.py
+++ b/rpython/jit/metainterp/optimizeopt/util.py
@@ -170,5 +170,24 @@
             remap[op2] = op1
         if op1.getopnum() not in (rop.JUMP, rop.LABEL) and not op1.is_guard():      # xxx obscure
             assert op1.getdescr() == op2.getdescr()
+        if op1.getfailargs() or op2.getfailargs():
+            assert len(op1.getfailargs()) == len(op2.getfailargs())
+            if strict_fail_args:
+                for x, y in zip(op1.getfailargs(), op2.getfailargs()):
+                    if x is None:
+                        assert remap.get(y, y) is None
+                    else:
+                        assert x.same_box(remap.get(y, y))
+            else:
+                fail_args1 = set(op1.getfailargs())
+                fail_args2 = set([remap.get(y, y) for y in op2.getfailargs()])
+                for x in fail_args1:
+                    for y in fail_args2:
+                        if x.same_box(y):
+                            fail_args2.remove(y)
+                            break
+                    else:
+                        assert False
+
     assert len(oplist1) == len(oplist2)
     return True
diff --git a/rpython/jit/metainterp/optimizeopt/virtualize.py b/rpython/jit/metainterp/optimizeopt/virtualize.py
--- a/rpython/jit/metainterp/optimizeopt/virtualize.py
+++ b/rpython/jit/metainterp/optimizeopt/virtualize.py
@@ -46,7 +46,7 @@
     def visitor_walk_recursive(self, visitor):
         # checks for recursion: it is False unless
         # we have already seen the very same keybox
-        if self.box is None and not visitor.already_seen_virtual(self.keybox):
+        if self.box is None and not visitor.already_seen_virtual(self.source_op):
             self._visitor_walk_recursive(visitor)
 
     def _visitor_walk_recursive(self, visitor):
@@ -189,7 +189,7 @@
     def _visitor_walk_recursive(self, visitor):
         lst = self._get_field_descr_list()
         fieldboxes = [self._fields[ofs].get_key_box() for ofs in lst]
-        visitor.register_virtual_fields(self.keybox, fieldboxes)
+        visitor.register_virtual_fields(self.source_op, fieldboxes)
         for ofs in lst:
             fieldvalue = self._fields[ofs]
             fieldvalue.visitor_walk_recursive(visitor)
diff --git a/rpython/jit/metainterp/resoperation.py b/rpython/jit/metainterp/resoperation.py
--- a/rpython/jit/metainterp/resoperation.py
+++ b/rpython/jit/metainterp/resoperation.py
@@ -93,7 +93,8 @@
         if descr is None:
             descr = self.getdescr()
         newop = ResOperation(opnum, args, descr)
-        newop.copy_value_from(self)
+        if self.type != 'v':
+            newop.copy_value_from(self)
         return newop
 
     @specialize.argtype(1)
diff --git a/rpython/jit/metainterp/resume.py b/rpython/jit/metainterp/resume.py
--- a/rpython/jit/metainterp/resume.py
+++ b/rpython/jit/metainterp/resume.py
@@ -335,7 +335,7 @@
         self._register_boxes(fieldboxes)
 
     def register_box(self, box):
-        if (isinstance(box, Box) and box not in self.liveboxes_from_env
+        if (not isinstance(box, Const) and box not in self.liveboxes_from_env
                                  and box not in self.liveboxes):
             self.liveboxes[box] = UNASSIGNED
 

From noreply at buildbot.pypy.org  Mon Nov 17 11:03:43 2014
From: noreply at buildbot.pypy.org (fijal)
Date: Mon, 17 Nov 2014 11:03:43 +0100 (CET)
Subject: [pypy-commit] pypy optresult: hack hack hack until we get the right
 oparse (I don't want to care)
Message-ID: <20141117100343.E93DE1C0F1B@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: optresult
Changeset: r74547:f1b9844b75c5
Date: 2014-11-17 12:03 +0200
http://bitbucket.org/pypy/pypy/changeset/f1b9844b75c5/

Log:	hack hack hack until we get the right oparse (I don't want to care)

diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py
@@ -114,7 +114,8 @@
             def _oparser_uses_descr_of_guard(self, oparse, fail_args):
                 # typically called 3 times: once when parsing 'ops',
                 # once when parsing 'preamble', once when parsing 'expected'.
-                self.oparse = oparse
+                if self.oparse is None:
+                    self.oparse = oparse
                 self.rd_frame_info_list, self.rd_snapshot = snapshot(fail_args)
             def _clone_if_mutable(self):
                 assert self is fdescr
diff --git a/rpython/jit/metainterp/optimizeopt/util.py b/rpython/jit/metainterp/optimizeopt/util.py
--- a/rpython/jit/metainterp/optimizeopt/util.py
+++ b/rpython/jit/metainterp/optimizeopt/util.py
@@ -168,7 +168,8 @@
             assert op1.same_box(remap[op2])
         else:
             remap[op2] = op1
-        if op1.getopnum() not in (rop.JUMP, rop.LABEL) and not op1.is_guard():      # xxx obscure
+        if op1.getopnum() not in (rop.JUMP, rop.LABEL) and not op1.is_guard():
+            # xxx obscure
             assert op1.getdescr() == op2.getdescr()
         if op1.getfailargs() or op2.getfailargs():
             assert len(op1.getfailargs()) == len(op2.getfailargs())

From noreply at buildbot.pypy.org  Mon Nov 17 13:58:21 2014
From: noreply at buildbot.pypy.org (kostialopuhin)
Date: Mon, 17 Nov 2014 13:58:21 +0100 (CET)
Subject: [pypy-commit] extradoc extradoc: initial version of tornado stm
	blog post
Message-ID: <20141117125821.BCFD41C31CF@cobra.cs.uni-duesseldorf.de>

Author: Konstantin Lopuhin 
Branch: extradoc
Changeset: r5457:02eb68ef992d
Date: 2014-11-02 01:23 +0400
http://bitbucket.org/pypy/extradoc/changeset/02eb68ef992d/

Log:	initial version of tornado stm blog post

diff --git a/blog/draft/tornado-stm.rst b/blog/draft/tornado-stm.rst
new file mode 100644
--- /dev/null
+++ b/blog/draft/tornado-stm.rst
@@ -0,0 +1,144 @@
+Tornado without a GIL on PyPy STM
+=================================
+
+Python has a GIL, right? Not quite - PyPy STM is a Python implementation
+without a GIL, so it can scale CPU-bound work to several cores.
+More than that, it proposes an easier to reason about model
+that is not based on threads (although you can use threads too).
+PyPy STM is developed by Armin Rigo and Remi Meier,
+and supported by community `donations `_.
+You can read more about it in the
+`docs `_.
+
+Although PyPy STM is still a work in progress, in many cases it can already
+run CPU-bound code faster than regular PyPy when using multiple cores.
+Here we will see how to slightly modify Tornado IO loop to use
+`transaction `_
+module.
+This module is `descibed `_
+in the docs and is really simple to use if you have several things to do where
+you do not care about the order they are run, as long as they
+are run separately. So an event loop of Tornado, or any other asynchronous
+framework, looks a bit like this (with some simplifications)::
+
+    while True:
+        for callback in list(self._callbacks):
+            self._run_callback(callback)
+        event_pairs = self._impl.poll()
+        self._events.update(event_pairs)
+        while self._events:
+            fd, events = self._events.popitem()
+            handler = self._handlers[fd]
+            self._handle_event(fd, handler, events)
+
+We get IO events, and run handlers for all of them, these handlers can
+also register new callbacks, which we run too. Then using such a framework,
+it is very nice to have a garanty that all things are run serially,
+so you do not have to put any locks. So this is an ideal case for the
+transaction module - it gives us garanties that things appear
+to be run serially, so in user code we do not need any locks. We just
+need to change the code above to something like::
+
+    while True:
+        for callback in list(self._callbacks):
+            transaction.add(
+            self._run_callback, callback)   # added
+        transaction.run()                   # added
+        event_pairs = self._impl.poll()
+        self._events.update(event_pairs)
+        while self._events:
+            fd, events = self._events.popitem()
+            handler = self._handlers[fd]
+            transaction.add(                # added
+                self._handle_event, fd, handler, events)
+        transaction.run()                   # added
+
+The actual commit is
+`here `_,
+- we had to extract a little function to run the callback.
+
+Now we need a simple benchmark, lets start with
+`this `_
+- just calculate a list of primes up to the given number, and return it
+as JSON::
+
+    def is_prime(n):
+        for i in xrange(2, n):
+            if n % i == 0:
+                return False
+        return True
+
+    class MainHandler(tornado.web.RequestHandler):
+        def get(self, num):
+            num = int(num)
+            primes = [n for n in xrange(2, num + 1) if is_prime(n)]
+            self.write(json.dumps({'primes': primes}))
+
+
+We can benchmark it with ``siege``::
+
+    siege -c 50 -t 20s http://localhost:8888/10000
+
+But this does not scale. The CPU load is at 101-104 %, and we handle 30 %
+less request per second. The reason for the slowdown is STM overhead,
+which needs to keep track of all writes and reads in order to detect conflicts.
+And the reason for using only one core is, obviously, conflicts!
+Fortunately, we can see what this conflicts are, if we run code like this
+(here 4 is the number of cores to use)::
+
+    PYPYSTM=stm.log ./primes.py 4
+
+Than we can use `print_stm_log.py `_
+to analyse this log. It lists the most expensive conflicts::
+
+    14.793s lost in aborts, 0.000s paused (1258x STM_CONTENTION_INEVITABLE)
+    File "/home/ubuntu/tornado-stm/tornado/tornado/httpserver.py", line 455, in __init__
+        self._start_time = time.time()
+    File "/home/ubuntu/tornado-stm/tornado/tornado/httpserver.py", line 455, in __init__
+        self._start_time = time.time()
+
+    ...
+
+There are only three kinds of conflicts, they are described in
+`stm source `_,
+Here we see that two theads call into external function (get current time),
+and we can not rollback any of them, so one of them must wait till the other
+transaction finishes.
+For now we can hack around this by disabling this timing - this is only
+needed for internal profiling in tornado.
+
+If we do it, we get the following results:
+
+============  =========
+Impl.           req/s
+============  =========
+PyPy 2.4        14.4
+------------  ---------
+CPython 2.7      3.2
+------------  ---------
+PyPy-STM 1       9.3
+------------  ---------
+PyPy-STM 2      16.4
+------------  ---------
+PyPy-STM 3      20.4
+------------  ---------
+PyPy STM 4      24.2
+============  =========
+
+As we can see, in this benchmark PyPy STM using just two cores
+can beat regular PyPy!
+This is not linear scaling, there are still conflicts left, and this
+is a very simple example but still, it works! And it was easy!
+
+Although it is defintily not ready for production use, you can alreay try
+to run things, report bugs, and see what is missing in user-facing tools
+and libraries.
+
+Benchmark setup:
+
+* Amazon c3.xlarge (4 cores) running Ubuntu 14.04
+* pypy-c-r74011-stm-jit
+* http://bitbucket.org/kostialopuhin/tornado-stm-bench at a038bf9
+* for PyPy-STM in this test the variation is rather high (around 20%),
+  best results from ``./bench_primes.sh`` were reported
+

From noreply at buildbot.pypy.org  Mon Nov 17 13:58:23 2014
From: noreply at buildbot.pypy.org (kostialopuhin)
Date: Mon, 17 Nov 2014 13:58:23 +0100 (CET)
Subject: [pypy-commit] extradoc extradoc: pff,
 the variation was high because I did not wait untill all requests
 finished
Message-ID: <20141117125823.159131C31CF@cobra.cs.uni-duesseldorf.de>

Author: Konstantin Lopuhin 
Branch: extradoc
Changeset: r5458:4eee97b07851
Date: 2014-11-02 01:27 +0400
http://bitbucket.org/pypy/extradoc/changeset/4eee97b07851/

Log:	pff, the variation was high because I did not wait untill all
	requests finished

diff --git a/blog/draft/tornado-stm.rst b/blog/draft/tornado-stm.rst
--- a/blog/draft/tornado-stm.rst
+++ b/blog/draft/tornado-stm.rst
@@ -139,6 +139,6 @@
 * Amazon c3.xlarge (4 cores) running Ubuntu 14.04
 * pypy-c-r74011-stm-jit
 * http://bitbucket.org/kostialopuhin/tornado-stm-bench at a038bf9
-* for PyPy-STM in this test the variation is rather high (around 20%),
-  best results from ``./bench_primes.sh`` were reported
+* for PyPy-STM in this test the variation is higher,
+  best results after warmup are given
 

From noreply at buildbot.pypy.org  Mon Nov 17 13:58:24 2014
From: noreply at buildbot.pypy.org (kostialopuhin)
Date: Mon, 17 Nov 2014 13:58:24 +0100 (CET)
Subject: [pypy-commit] extradoc extradoc: fix and simplify
Message-ID: <20141117125824.48D881C31CF@cobra.cs.uni-duesseldorf.de>

Author: Konstantin Lopuhin 
Branch: extradoc
Changeset: r5459:857b38852bd6
Date: 2014-11-02 01:42 +0400
http://bitbucket.org/pypy/extradoc/changeset/857b38852bd6/

Log:	fix and simplify

diff --git a/blog/draft/tornado-stm.rst b/blog/draft/tornado-stm.rst
--- a/blog/draft/tornado-stm.rst
+++ b/blog/draft/tornado-stm.rst
@@ -1,25 +1,22 @@
 Tornado without a GIL on PyPy STM
 =================================
 
-Python has a GIL, right? Not quite - PyPy STM is a Python implementation
+Python has a GIL, right? Not quite - PyPy STM is a python implementation
 without a GIL, so it can scale CPU-bound work to several cores.
-More than that, it proposes an easier to reason about model
-that is not based on threads (although you can use threads too).
 PyPy STM is developed by Armin Rigo and Remi Meier,
 and supported by community `donations `_.
 You can read more about it in the
 `docs `_.
 
 Although PyPy STM is still a work in progress, in many cases it can already
-run CPU-bound code faster than regular PyPy when using multiple cores.
+run CPU-bound code faster than regular PyPy, when using multiple cores.
 Here we will see how to slightly modify Tornado IO loop to use
 `transaction `_
 module.
 This module is `descibed `_
-in the docs and is really simple to use if you have several things to do where
-you do not care about the order they are run, as long as they
-are run separately. So an event loop of Tornado, or any other asynchronous
-framework, looks a bit like this (with some simplifications)::
+in the docs and is really simple to use - please see an example there.
+An event loop of Tornado, or any other asynchronous
+web server, looks like this (with some simplifications)::
 
     while True:
         for callback in list(self._callbacks):
@@ -32,10 +29,10 @@
             self._handle_event(fd, handler, events)
 
 We get IO events, and run handlers for all of them, these handlers can
-also register new callbacks, which we run too. Then using such a framework,
-it is very nice to have a garanty that all things are run serially,
-so you do not have to put any locks. So this is an ideal case for the
-transaction module - it gives us garanties that things appear
+also register new callbacks, which we run too. When using such a framework,
+it is very nice to have a guaranty that all handlers are run serially,
+so you do not have to put any locks. This is an ideal case for the
+transaction module - it gives us guaranties that things appear
 to be run serially, so in user code we do not need any locks. We just
 need to change the code above to something like::
 
@@ -96,12 +93,11 @@
         self._start_time = time.time()
     File "/home/ubuntu/tornado-stm/tornado/tornado/httpserver.py", line 455, in __init__
         self._start_time = time.time()
-
     ...
 
 There are only three kinds of conflicts, they are described in
 `stm source `_,
-Here we see that two theads call into external function (get current time),
+Here we see that two threads call into external function to get current time,
 and we can not rollback any of them, so one of them must wait till the other
 transaction finishes.
 For now we can hack around this by disabling this timing - this is only
@@ -130,7 +126,7 @@
 This is not linear scaling, there are still conflicts left, and this
 is a very simple example but still, it works! And it was easy!
 
-Although it is defintily not ready for production use, you can alreay try
+Although it is definitely not ready for production use, you can already try
 to run things, report bugs, and see what is missing in user-facing tools
 and libraries.
 

From noreply at buildbot.pypy.org  Mon Nov 17 13:58:25 2014
From: noreply at buildbot.pypy.org (kostialopuhin)
Date: Mon, 17 Nov 2014 13:58:25 +0100 (CET)
Subject: [pypy-commit] extradoc extradoc: wip
Message-ID: <20141117125825.825FD1C31CF@cobra.cs.uni-duesseldorf.de>

Author: Konstantin Lopuhin 
Branch: extradoc
Changeset: r5460:9691e27e3379
Date: 2014-11-13 22:32 +0300
http://bitbucket.org/pypy/extradoc/changeset/9691e27e3379/

Log:	wip

diff --git a/blog/draft/tornado-stm.rst b/blog/draft/tornado-stm.rst
--- a/blog/draft/tornado-stm.rst
+++ b/blog/draft/tornado-stm.rst
@@ -13,7 +13,7 @@
 Here we will see how to slightly modify Tornado IO loop to use
 `transaction `_
 module.
-This module is `descibed `_
+This module is `described `_
 in the docs and is really simple to use - please see an example there.
 An event loop of Tornado, or any other asynchronous
 web server, looks like this (with some simplifications)::
@@ -54,6 +54,9 @@
 `here `_,
 - we had to extract a little function to run the callback.
 
+Part 1: a simple benchmark: primes
+----------------------------------
+
 Now we need a simple benchmark, lets start with
 `this `_
 - just calculate a list of primes up to the given number, and return it
@@ -69,7 +72,7 @@
         def get(self, num):
             num = int(num)
             primes = [n for n in xrange(2, num + 1) if is_prime(n)]
-            self.write(json.dumps({'primes': primes}))
+            self.write({'primes': primes})
 
 
 We can benchmark it with ``siege``::
@@ -103,7 +106,7 @@
 For now we can hack around this by disabling this timing - this is only
 needed for internal profiling in tornado.
 
-If we do it, we get the following results:
+If we do it, we get the following results (but see caveats below):
 
 ============  =========
 Impl.           req/s
@@ -121,20 +124,109 @@
 PyPy STM 4      24.2
 ============  =========
 
+.. image:: results-1.png
+
 As we can see, in this benchmark PyPy STM using just two cores
 can beat regular PyPy!
 This is not linear scaling, there are still conflicts left, and this
-is a very simple example but still, it works! And it was easy!
+is a very simple example but still, it works!
+
+But its not that simple yet :)
+
+First, these are best case numbers after long (much longer than for regular
+PyPy) warmup. Second, it can sometimes crash (although removing old pyc files
+fixes it). Third, benchmark meta-parameters are also tuned.
+
+Here we get reletively good results only when there are a lot of concurrent
+clients - as a results, a lot of requests pile up, the server is not keeping
+with the load, and transaction module is busy with work running this piled up
+requests. If we decrease the number of concurrent clients, results get worse.
+Another thing we can tune is how heavy is each request - again, if we ask
+primes up to a slower number, than less time is spent doing calculations,
+more time is spent in conflicts, and results get worse.
+
+Besides the ``time.time()`` conflict described above, there are a lot of others.
+The bulk of time is lost in this conflicts::
+
+    14.153s lost in aborts, 0.000s paused (270x STM_CONTENTION_INEVITABLE)
+    File "/home/ubuntu/tornado-stm/tornado/tornado/web.py", line 1082, in compute_etag
+        hasher = hashlib.sha1()
+    File "/home/ubuntu/tornado-stm/tornado/tornado/web.py", line 1082, in compute_etag
+        hasher = hashlib.sha1()
+
+    13.484s lost in aborts, 0.000s paused (130x STM_CONTENTION_WRITE_READ)
+    File "/home/ubuntu/pypy/lib_pypy/transaction.py", line 164, in _run_thread
+        got_exception)
+
+The first one is presumably calling into some C function from stdlib, and we get
+the same conflict as for ``time.time()`` above, but is can be fixed on PyPy
+side, as we can be sure that computing sha1 is pure.
+
+It is easy to hack around this one too, just removing etag support, but if
+we do it, performance is much worse, only slightly faster than regular PyPy,
+with the top conflict being::
+
+    83.066s lost in aborts, 0.000s paused (459x STM_CONTENTION_WRITE_WRITE)
+    File "/home/arigo/hg/pypy/stmgc-c7/lib-python/2.7/_weakrefset.py", line 70, in __contains__
+    File "/home/arigo/hg/pypy/stmgc-c7/lib-python/2.7/_weakrefset.py", line 70, in __contains__
+
+**FIXME** why does it happen?
+
+The second conflict (without etag tweaks) originates
+in the transaction module, from this piece of code::
+
+    while True:
+        self._do_it(self._grab_next_thing_to_do(tloc_pending),
+                    got_exception)
+        counter[0] += 1
+
+**FIXME** why does it happen?
+
+Tornado modification used in this blog post is based on 3.2.dev2.
+As of now, the latest version is 4.0.2, and if we
+`apply `_
+the same changes to this version, then we no longer get any scaling on this benchmark,
+and there are no conflicts that take any substantial time.
+
+**FIXME** - maybe this is just me messing something up
+
+
+Part 2: a more interesting benchmark: A-star
+--------------------------------------------
+
+Although we have seen that PyPy STM is not all moonlight and roses,
+it is interesting to see how it works on a more realistic application.
+
+`astar.py `_
+is a simple game where several players move on a map
+(represented as a list of lists of integers),
+build and destroy walls, and ask server to give them
+shortest paths between two points
+using A-star search, adopted from `ActiveState recipie `_.
+
+The benchmark `bench_astar.py `_
+is simulating players, and tries to put the main load on A-star search,
+but also does some wall building and desctruction. There are no locks
+around map modifications, as normal tornado is executing all callbacks
+serially, and we can keep this guaranty with atomic blocks of PyPy STM.
+This is also an example of a program that is not trivial
+to scale to multiple cores with separate processes (assuming
+more interesting shared state and logic).
+
+**TODO** - results
 
 Although it is definitely not ready for production use, you can already try
 to run things, report bugs, and see what is missing in user-facing tools
 and libraries.
 
-Benchmark setup:
+
+Benchmarks setup:
 
 * Amazon c3.xlarge (4 cores) running Ubuntu 14.04
-* pypy-c-r74011-stm-jit
+* pypy-c-r74011-stm-jit for the primes benchmark (but it has more bugs
+  then more recent versions), and
+  `pypy-c-r74378-74379-stm-jit `_
+  for all other stuff
 * http://bitbucket.org/kostialopuhin/tornado-stm-bench at a038bf9
 * for PyPy-STM in this test the variation is higher,
-  best results after warmup are given
-
+  best results after long warmup are given

From noreply at buildbot.pypy.org  Mon Nov 17 13:58:26 2014
From: noreply at buildbot.pypy.org (kostialopuhin)
Date: Mon, 17 Nov 2014 13:58:26 +0100 (CET)
Subject: [pypy-commit] extradoc extradoc: astar benchmark results
Message-ID: <20141117125826.B06451C31CF@cobra.cs.uni-duesseldorf.de>

Author: Konstantin Lopuhin 
Branch: extradoc
Changeset: r5461:a666434a0a38
Date: 2014-11-14 10:37 +0300
http://bitbucket.org/pypy/extradoc/changeset/a666434a0a38/

Log:	astar benchmark results

diff --git a/blog/draft/tornado-stm.rst b/blog/draft/tornado-stm.rst
--- a/blog/draft/tornado-stm.rst
+++ b/blog/draft/tornado-stm.rst
@@ -140,10 +140,10 @@
 Here we get reletively good results only when there are a lot of concurrent
 clients - as a results, a lot of requests pile up, the server is not keeping
 with the load, and transaction module is busy with work running this piled up
-requests. If we decrease the number of concurrent clients, results get worse.
+requests. If we decrease the number of concurrent clients, results get slightly worse.
 Another thing we can tune is how heavy is each request - again, if we ask
 primes up to a slower number, than less time is spent doing calculations,
-more time is spent in conflicts, and results get worse.
+more time is spent in tornado, and results get much worse.
 
 Besides the ``time.time()`` conflict described above, there are a lot of others.
 The bulk of time is lost in this conflicts::
@@ -213,7 +213,29 @@
 to scale to multiple cores with separate processes (assuming
 more interesting shared state and logic).
 
-**TODO** - results
+This benchmark is very noisy due to randomness of client interactions
+(also it could be not linear), so just lower and upper bounds for
+number of requests are reported
+
+============  ==========
+Impl.           req/s
+============  ==========
+PyPy 2.4        5 .. 7
+------------  ----------
+CPython 2.7   0.5 .. 0.9
+------------  ----------
+PyPy-STM 1      2 .. 4
+------------  ----------
+PyPy STM 4      2 .. 6
+============  ==========
+
+The bulk of conflicts are the same as in the previous benchmark with etag
+calculation removed::
+
+    91.655s lost in aborts, 0.000s paused (249x STM_CONTENTION_WRITE_READ)
+    File "/home/ubuntu/pypy/lib_pypy/transaction.py", line 164, in _run_thread
+        got_exception)
+
 
 Although it is definitely not ready for production use, you can already try
 to run things, report bugs, and see what is missing in user-facing tools
@@ -226,7 +248,5 @@
 * pypy-c-r74011-stm-jit for the primes benchmark (but it has more bugs
   then more recent versions), and
   `pypy-c-r74378-74379-stm-jit `_
-  for all other stuff
-* http://bitbucket.org/kostialopuhin/tornado-stm-bench at a038bf9
-* for PyPy-STM in this test the variation is higher,
-  best results after long warmup are given
+  for astar benchmark (put it inside pypy source checkout at 38c9afbd253c)
+* http://bitbucket.org/kostialopuhin/tornado-stm-bench at 65144cda7a1f

From noreply at buildbot.pypy.org  Mon Nov 17 13:58:27 2014
From: noreply at buildbot.pypy.org (kostialopuhin)
Date: Mon, 17 Nov 2014 13:58:27 +0100 (CET)
Subject: [pypy-commit] extradoc extradoc: fix spelling, finish
Message-ID: <20141117125827.CF1881C31CF@cobra.cs.uni-duesseldorf.de>

Author: Konstantin Lopuhin 
Branch: extradoc
Changeset: r5462:d5135af1d14a
Date: 2014-11-14 10:50 +0300
http://bitbucket.org/pypy/extradoc/changeset/d5135af1d14a/

Log:	fix spelling, finish

diff --git a/blog/draft/tornado-stm.rst b/blog/draft/tornado-stm.rst
--- a/blog/draft/tornado-stm.rst
+++ b/blog/draft/tornado-stm.rst
@@ -88,7 +88,7 @@
 
     PYPYSTM=stm.log ./primes.py 4
 
-Than we can use `print_stm_log.py `_
+Then we can use `print_stm_log.py `_
 to analyse this log. It lists the most expensive conflicts::
 
     14.793s lost in aborts, 0.000s paused (1258x STM_CONTENTION_INEVITABLE)
@@ -133,20 +133,20 @@
 
 But its not that simple yet :)
 
-First, these are best case numbers after long (much longer than for regular
+First, these are best-case numbers after long (much longer than for regular
 PyPy) warmup. Second, it can sometimes crash (although removing old pyc files
 fixes it). Third, benchmark meta-parameters are also tuned.
 
-Here we get reletively good results only when there are a lot of concurrent
+Here we get relatively good results only when there are a lot of concurrent
 clients - as a results, a lot of requests pile up, the server is not keeping
 with the load, and transaction module is busy with work running this piled up
 requests. If we decrease the number of concurrent clients, results get slightly worse.
 Another thing we can tune is how heavy is each request - again, if we ask
-primes up to a slower number, than less time is spent doing calculations,
+primes up to a lower number, then less time is spent doing calculations,
 more time is spent in tornado, and results get much worse.
 
 Besides the ``time.time()`` conflict described above, there are a lot of others.
-The bulk of time is lost in this conflicts::
+The bulk of time is lost in these two conflicts::
 
     14.153s lost in aborts, 0.000s paused (270x STM_CONTENTION_INEVITABLE)
     File "/home/ubuntu/tornado-stm/tornado/tornado/web.py", line 1082, in compute_etag
@@ -188,8 +188,6 @@
 the same changes to this version, then we no longer get any scaling on this benchmark,
 and there are no conflicts that take any substantial time.
 
-**FIXME** - maybe this is just me messing something up
-
 
 Part 2: a more interesting benchmark: A-star
 --------------------------------------------
@@ -206,7 +204,7 @@
 
 The benchmark `bench_astar.py `_
 is simulating players, and tries to put the main load on A-star search,
-but also does some wall building and desctruction. There are no locks
+but also does some wall building and destruction. There are no locks
 around map modifications, as normal tornado is executing all callbacks
 serially, and we can keep this guaranty with atomic blocks of PyPy STM.
 This is also an example of a program that is not trivial
@@ -229,8 +227,10 @@
 PyPy STM 4      2 .. 6
 ============  ==========
 
-The bulk of conflicts are the same as in the previous benchmark with etag
-calculation removed::
+Clearly this is a very benchmark, but still we can see that scaling is worse
+and STM overhead is sometimes higher.
+The bulk of conflicts come from the transaction module (we have seen it
+above)::
 
     91.655s lost in aborts, 0.000s paused (249x STM_CONTENTION_WRITE_READ)
     File "/home/ubuntu/pypy/lib_pypy/transaction.py", line 164, in _run_thread
@@ -246,7 +246,7 @@
 
 * Amazon c3.xlarge (4 cores) running Ubuntu 14.04
 * pypy-c-r74011-stm-jit for the primes benchmark (but it has more bugs
-  then more recent versions), and
+  than more recent versions), and
   `pypy-c-r74378-74379-stm-jit `_
   for astar benchmark (put it inside pypy source checkout at 38c9afbd253c)
 * http://bitbucket.org/kostialopuhin/tornado-stm-bench at 65144cda7a1f

From noreply at buildbot.pypy.org  Mon Nov 17 13:58:29 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 17 Nov 2014 13:58:29 +0100 (CET)
Subject: [pypy-commit] extradoc extradoc: Merged in kostialopuhin/extradoc
	(pull request #3)
Message-ID: <20141117125829.0B3C91C31CF@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: extradoc
Changeset: r5463:97f61c269c25
Date: 2014-11-17 13:58 +0100
http://bitbucket.org/pypy/extradoc/changeset/97f61c269c25/

Log:	Merged in kostialopuhin/extradoc (pull request #3)

	Blog post abount running tornado on STM

diff --git a/blog/draft/tornado-stm.rst b/blog/draft/tornado-stm.rst
new file mode 100644
--- /dev/null
+++ b/blog/draft/tornado-stm.rst
@@ -0,0 +1,252 @@
+Tornado without a GIL on PyPy STM
+=================================
+
+Python has a GIL, right? Not quite - PyPy STM is a python implementation
+without a GIL, so it can scale CPU-bound work to several cores.
+PyPy STM is developed by Armin Rigo and Remi Meier,
+and supported by community `donations `_.
+You can read more about it in the
+`docs `_.
+
+Although PyPy STM is still a work in progress, in many cases it can already
+run CPU-bound code faster than regular PyPy, when using multiple cores.
+Here we will see how to slightly modify Tornado IO loop to use
+`transaction `_
+module.
+This module is `described `_
+in the docs and is really simple to use - please see an example there.
+An event loop of Tornado, or any other asynchronous
+web server, looks like this (with some simplifications)::
+
+    while True:
+        for callback in list(self._callbacks):
+            self._run_callback(callback)
+        event_pairs = self._impl.poll()
+        self._events.update(event_pairs)
+        while self._events:
+            fd, events = self._events.popitem()
+            handler = self._handlers[fd]
+            self._handle_event(fd, handler, events)
+
+We get IO events, and run handlers for all of them, these handlers can
+also register new callbacks, which we run too. When using such a framework,
+it is very nice to have a guaranty that all handlers are run serially,
+so you do not have to put any locks. This is an ideal case for the
+transaction module - it gives us guaranties that things appear
+to be run serially, so in user code we do not need any locks. We just
+need to change the code above to something like::
+
+    while True:
+        for callback in list(self._callbacks):
+            transaction.add(
+            self._run_callback, callback)   # added
+        transaction.run()                   # added
+        event_pairs = self._impl.poll()
+        self._events.update(event_pairs)
+        while self._events:
+            fd, events = self._events.popitem()
+            handler = self._handlers[fd]
+            transaction.add(                # added
+                self._handle_event, fd, handler, events)
+        transaction.run()                   # added
+
+The actual commit is
+`here `_,
+- we had to extract a little function to run the callback.
+
+Part 1: a simple benchmark: primes
+----------------------------------
+
+Now we need a simple benchmark, lets start with
+`this `_
+- just calculate a list of primes up to the given number, and return it
+as JSON::
+
+    def is_prime(n):
+        for i in xrange(2, n):
+            if n % i == 0:
+                return False
+        return True
+
+    class MainHandler(tornado.web.RequestHandler):
+        def get(self, num):
+            num = int(num)
+            primes = [n for n in xrange(2, num + 1) if is_prime(n)]
+            self.write({'primes': primes})
+
+
+We can benchmark it with ``siege``::
+
+    siege -c 50 -t 20s http://localhost:8888/10000
+
+But this does not scale. The CPU load is at 101-104 %, and we handle 30 %
+less request per second. The reason for the slowdown is STM overhead,
+which needs to keep track of all writes and reads in order to detect conflicts.
+And the reason for using only one core is, obviously, conflicts!
+Fortunately, we can see what this conflicts are, if we run code like this
+(here 4 is the number of cores to use)::
+
+    PYPYSTM=stm.log ./primes.py 4
+
+Then we can use `print_stm_log.py `_
+to analyse this log. It lists the most expensive conflicts::
+
+    14.793s lost in aborts, 0.000s paused (1258x STM_CONTENTION_INEVITABLE)
+    File "/home/ubuntu/tornado-stm/tornado/tornado/httpserver.py", line 455, in __init__
+        self._start_time = time.time()
+    File "/home/ubuntu/tornado-stm/tornado/tornado/httpserver.py", line 455, in __init__
+        self._start_time = time.time()
+    ...
+
+There are only three kinds of conflicts, they are described in
+`stm source `_,
+Here we see that two threads call into external function to get current time,
+and we can not rollback any of them, so one of them must wait till the other
+transaction finishes.
+For now we can hack around this by disabling this timing - this is only
+needed for internal profiling in tornado.
+
+If we do it, we get the following results (but see caveats below):
+
+============  =========
+Impl.           req/s
+============  =========
+PyPy 2.4        14.4
+------------  ---------
+CPython 2.7      3.2
+------------  ---------
+PyPy-STM 1       9.3
+------------  ---------
+PyPy-STM 2      16.4
+------------  ---------
+PyPy-STM 3      20.4
+------------  ---------
+PyPy STM 4      24.2
+============  =========
+
+.. image:: results-1.png
+
+As we can see, in this benchmark PyPy STM using just two cores
+can beat regular PyPy!
+This is not linear scaling, there are still conflicts left, and this
+is a very simple example but still, it works!
+
+But its not that simple yet :)
+
+First, these are best-case numbers after long (much longer than for regular
+PyPy) warmup. Second, it can sometimes crash (although removing old pyc files
+fixes it). Third, benchmark meta-parameters are also tuned.
+
+Here we get relatively good results only when there are a lot of concurrent
+clients - as a results, a lot of requests pile up, the server is not keeping
+with the load, and transaction module is busy with work running this piled up
+requests. If we decrease the number of concurrent clients, results get slightly worse.
+Another thing we can tune is how heavy is each request - again, if we ask
+primes up to a lower number, then less time is spent doing calculations,
+more time is spent in tornado, and results get much worse.
+
+Besides the ``time.time()`` conflict described above, there are a lot of others.
+The bulk of time is lost in these two conflicts::
+
+    14.153s lost in aborts, 0.000s paused (270x STM_CONTENTION_INEVITABLE)
+    File "/home/ubuntu/tornado-stm/tornado/tornado/web.py", line 1082, in compute_etag
+        hasher = hashlib.sha1()
+    File "/home/ubuntu/tornado-stm/tornado/tornado/web.py", line 1082, in compute_etag
+        hasher = hashlib.sha1()
+
+    13.484s lost in aborts, 0.000s paused (130x STM_CONTENTION_WRITE_READ)
+    File "/home/ubuntu/pypy/lib_pypy/transaction.py", line 164, in _run_thread
+        got_exception)
+
+The first one is presumably calling into some C function from stdlib, and we get
+the same conflict as for ``time.time()`` above, but is can be fixed on PyPy
+side, as we can be sure that computing sha1 is pure.
+
+It is easy to hack around this one too, just removing etag support, but if
+we do it, performance is much worse, only slightly faster than regular PyPy,
+with the top conflict being::
+
+    83.066s lost in aborts, 0.000s paused (459x STM_CONTENTION_WRITE_WRITE)
+    File "/home/arigo/hg/pypy/stmgc-c7/lib-python/2.7/_weakrefset.py", line 70, in __contains__
+    File "/home/arigo/hg/pypy/stmgc-c7/lib-python/2.7/_weakrefset.py", line 70, in __contains__
+
+**FIXME** why does it happen?
+
+The second conflict (without etag tweaks) originates
+in the transaction module, from this piece of code::
+
+    while True:
+        self._do_it(self._grab_next_thing_to_do(tloc_pending),
+                    got_exception)
+        counter[0] += 1
+
+**FIXME** why does it happen?
+
+Tornado modification used in this blog post is based on 3.2.dev2.
+As of now, the latest version is 4.0.2, and if we
+`apply `_
+the same changes to this version, then we no longer get any scaling on this benchmark,
+and there are no conflicts that take any substantial time.
+
+
+Part 2: a more interesting benchmark: A-star
+--------------------------------------------
+
+Although we have seen that PyPy STM is not all moonlight and roses,
+it is interesting to see how it works on a more realistic application.
+
+`astar.py `_
+is a simple game where several players move on a map
+(represented as a list of lists of integers),
+build and destroy walls, and ask server to give them
+shortest paths between two points
+using A-star search, adopted from `ActiveState recipie `_.
+
+The benchmark `bench_astar.py `_
+is simulating players, and tries to put the main load on A-star search,
+but also does some wall building and destruction. There are no locks
+around map modifications, as normal tornado is executing all callbacks
+serially, and we can keep this guaranty with atomic blocks of PyPy STM.
+This is also an example of a program that is not trivial
+to scale to multiple cores with separate processes (assuming
+more interesting shared state and logic).
+
+This benchmark is very noisy due to randomness of client interactions
+(also it could be not linear), so just lower and upper bounds for
+number of requests are reported
+
+============  ==========
+Impl.           req/s
+============  ==========
+PyPy 2.4        5 .. 7
+------------  ----------
+CPython 2.7   0.5 .. 0.9
+------------  ----------
+PyPy-STM 1      2 .. 4
+------------  ----------
+PyPy STM 4      2 .. 6
+============  ==========
+
+Clearly this is a very benchmark, but still we can see that scaling is worse
+and STM overhead is sometimes higher.
+The bulk of conflicts come from the transaction module (we have seen it
+above)::
+
+    91.655s lost in aborts, 0.000s paused (249x STM_CONTENTION_WRITE_READ)
+    File "/home/ubuntu/pypy/lib_pypy/transaction.py", line 164, in _run_thread
+        got_exception)
+
+
+Although it is definitely not ready for production use, you can already try
+to run things, report bugs, and see what is missing in user-facing tools
+and libraries.
+
+
+Benchmarks setup:
+
+* Amazon c3.xlarge (4 cores) running Ubuntu 14.04
+* pypy-c-r74011-stm-jit for the primes benchmark (but it has more bugs
+  than more recent versions), and
+  `pypy-c-r74378-74379-stm-jit `_
+  for astar benchmark (put it inside pypy source checkout at 38c9afbd253c)
+* http://bitbucket.org/kostialopuhin/tornado-stm-bench at 65144cda7a1f

From noreply at buildbot.pypy.org  Mon Nov 17 16:04:57 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 17 Nov 2014 16:04:57 +0100 (CET)
Subject: [pypy-commit] creflect default: more command-line tweaks
Message-ID: <20141117150457.E2B3F1C31CF@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r50:555bd1e5574e
Date: 2014-09-19 16:35 +0200
http://bitbucket.org/cffi/creflect/changeset/555bd1e5574e/

Log:	more command-line tweaks

diff --git a/creflect/cmdline.py b/creflect/cmdline.py
--- a/creflect/cmdline.py
+++ b/creflect/cmdline.py
@@ -1,19 +1,19 @@
 """Usage:
 
-    creflect [options] input.rfl.c [output.c]
+    creflect [options] input{.rfl.c,.h} output.c
 
 Read the 'input.rfl.c' file and expand the CREFLECT sections in it
 into regular C code.  Write the result to 'output.c'.
 
-The default output file name is built from the input file name, by
-removing the '.rfl' part.  The input and/or output file names
-can be specified as '-' to read from stdin and/or write to stdout.
+The input and/or output file names can be specified as '-' to read
+ from stdin and/or write to stdout.
 
 Options:
     -m, --main      for debugging, include a main() function that
                       prints the recorded reflection information
     -n, --no-copy   output only the translated creflect sections,
                       without copying the text around them
+    -i, --include   emit a line ``#include "inputfile"''; implies -n
     -h, --help      display this help and exit
     --version       output version information and exit
 """
@@ -28,17 +28,21 @@
 
 def main(argv):
     try:
-        options, args = getopt.gnu_getopt(argv, "mnh",
-            ["main", "no-copy", "help", "version"])
+        options, args = getopt.gnu_getopt(argv, "mnih",
+            ["main", "no-copy", "include", "help", "version"])
     except getopt.GetoptError, e:
         return error(e)
     #
     from . import __version__
     include_main = False
     include_text_outside_creflect = True
+    include_includes = False
     for option, value in options:
         if option == '-n' or option == '--no-copy':
             include_text_outside_creflect = False
+        elif option == '-i' or option == '--include':
+            include_text_outside_creflect = False
+            include_includes = True
         elif option == '-m' or option == '--main':
             include_main = True
         elif option == '-h' or option == '--help':
@@ -48,22 +52,10 @@
             print 'CReflect %s' % __version__
             return 0
     #
-    if len(args) == 0:
-        return error("missing input file")
-    inputfile = args[0]
-    if len(args) > 1:
-        outputfile = args[1]
-        if len(args) > 2:
-            return error("too many arguments")
-    else:
-        if inputfile == '-':
-            return error("destination file must be specified if input is '-'")
-        i = inputfile.lower().rfind('.rfl')
-        if i < 0:
-            return error("cannot find '.rfl' in the input file name: specify "
-                         "the output file name explicitly")
-        outputfile = inputfile[:i] + inputfile[i+4:]
-    if inputfile != '-' and inputfile == outputfile:
+    if len(args) != 2:
+        return error("expected exactly 2 arguments, got %d" % (len(args),))
+    inputfile, outputfile = args
+    if inputfile == outputfile:
         return error("not overwriting the file %r" % (inputfile,))
     #
     from . import driver
@@ -79,6 +71,8 @@
         outputf = open(outputfile, 'w')
     #
     try:
+        if include_includes:
+            outputf.write('#include "%s"\n' % (inputfile,))
         blocks = driver.copy_file_and_expand(inputf, outputf,
             include_text_outside_creflect = include_text_outside_creflect)
         if include_main:
diff --git a/creflect/driver.py b/creflect/driver.py
--- a/creflect/driver.py
+++ b/creflect/driver.py
@@ -38,6 +38,8 @@
         if funcname is END:
             raise CDefError("line %d: 'CREFLECT: end' without "
                             "'CREFLECT: funcname()' before" % lineno)
+        if not include_text_outside_creflect:
+            outputf.write("\n")
         #
         # inside a 'CREFLECT:' section
         csource = []

From noreply at buildbot.pypy.org  Mon Nov 17 16:04:59 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 17 Nov 2014 16:04:59 +0100 (CET)
Subject: [pypy-commit] creflect default: Refactoring in progress...
Message-ID: <20141117150459.209511C31CF@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r51:6ddbcda0f7b3
Date: 2014-11-17 16:05 +0100
http://bitbucket.org/cffi/creflect/changeset/6ddbcda0f7b3/

Log:	Refactoring in progress...

diff --git a/creflect/creflect.h b/creflect/creflect.h
new file mode 100644
--- /dev/null
+++ b/creflect/creflect.h
@@ -0,0 +1,61 @@
+#include 
+
+
+typedef struct {
+    const char *name;
+    void *type;
+    size_t offset;
+    int numbits, bitshift;
+} crx_field_t;
+
+typedef struct crx_type_s crx_type_t;   /* opaque */
+typedef void (*crx_trampoline_t)(void *fn, void *args[], void *res);
+
+#define CRX_SELF struct _crx_builder_s *
+typedef struct _crx_builder_s {
+    crx_type_t *(*get_void_type)(CRX_SELF);
+    crx_type_t *(*get_char_type)(CRX_SELF);
+    crx_type_t *(*get_bool_type)(CRX_SELF);
+    crx_type_t *(*get_signed_type)(CRX_SELF, size_t,
+                                const char *);
+    crx_type_t *(*get_unsigned_type)(CRX_SELF, size_t,
+                                  const char *);
+    crx_type_t *(*get_float_type)(CRX_SELF, size_t,
+                               const char *);
+    crx_type_t *(*get_function_type)(CRX_SELF, crx_type_t *,
+                                     crx_type_t *[], int, crx_trampoline_t *);
+    crx_type_t *(*get_ellipsis_function_type)(CRX_SELF, crx_type_t *,
+                                              crx_type_t *[], int);
+    crx_type_t *(*get_pointer_type)(CRX_SELF, crx_type_t *);
+    crx_type_t *(*get_const_type)(CRX_SELF, crx_type_t *);
+    crx_type_t *(*get_array_type)(CRX_SELF, crx_type_t *, size_t);
+    crx_type_t *(*get_incomplete_array_type)(CRX_SELF, crx_type_t *);
+    crx_type_t *(*get_struct_type)(CRX_SELF, const char *);
+    crx_type_t *(*get_union_type)(CRX_SELF, const char *);
+    crx_type_t *(*get_enum_type)(CRX_SELF, const char *);
+    void (*complete)(CRX_SELF, crx_type_t *,
+                     size_t, size_t, crx_field_t[], int);
+    void (*complete_enum)(CRX_SELF, crx_type_t *, crx_type_t *);
+    void (*define_type)(CRX_SELF, const char *, crx_type_t *);
+    void (*define_var)(CRX_SELF, const char *, crx_type_t *,
+                       void *);
+    void (*error)(CRX_SELF, const char *, crx_type_t *);
+} crx_builder_t;
+#undef CRX_SELF
+
+
+#define CRX_INT_TYPE(cb, expr, guessname)                               \
+    _creflect__int_type(cb, expr > 0, sizeof(expr), expr == 1, guessname)
+
+__attribute__((unused))
+static crx_type_t *_creflect__int_type(crx_builder_t *cb, int expr_positive,
+                                       size_t size_of_expr, int expr_equal_one,
+                                       const char *guessname)
+{
+    if (!expr_positive)
+        return cb->get_signed_type(cb, size_of_expr, guessname);
+    else if (size_of_expr == 1 && expr_equal_one)
+        return cb->get_bool_type(cb);
+    else
+        return cb->get_unsigned_type(cb, size_of_expr, guessname);
+}
diff --git a/test/cgcompile.c b/test/cgcompile.c
new file mode 100644
--- /dev/null
+++ b/test/cgcompile.c
@@ -0,0 +1,190 @@
+#include 
+#include 
+#include 
+
+
+struct crx_type_s {
+    char text[1];
+};
+
+
+static crx_type_t *newtype(const char *a)
+{
+    size_t la = strlen(a);
+    crx_type_t *t = malloc(la + 1);
+    strcpy(t->text, a);
+    return t;
+}
+
+static crx_type_t *newtype2(const char *a, const char *b)
+{
+    size_t la = strlen(a);
+    size_t lb = strlen(b);
+    crx_type_t *t = malloc(la + lb + 1);
+    strcat(strcpy(t->text, a), b);
+    return t;
+}
+
+static crx_type_t *tst_get_void_type(crx_builder_t *cb)
+{
+    return newtype("void");
+}
+
+static crx_type_t *tst_get_char_type(crx_builder_t *cb)
+{
+    return newtype("char");
+}
+
+static crx_type_t *tst_get_bool_type(crx_builder_t *cb)
+{
+    return newtype("_Bool");
+}
+
+static crx_type_t *tst_get_signed_type(crx_builder_t *cb, size_t sz,
+                                       const char *g)
+{
+#define TT(name)  if (strcmp(g, #name) == 0) { found = sizeof(name); }
+    int found = 0;
+    TT(signed char);
+    TT(short);
+    TT(int);
+    TT(long);
+    TT(long long);
+    assert(found == sz);
+    return newtype(g);
+}
+
+static crx_type_t *tst_get_unsigned_type(crx_builder_t *cb, size_t sz,
+                                         const char *g)
+{
+    int found = 0;
+    TT(unsigned char);
+    TT(unsigned short);
+    TT(unsigned int);
+    TT(unsigned long);
+    TT(unsigned long long);
+    assert(found == sz);
+    return newtype(g);
+}
+
+static crx_type_t *tst_get_float_type(crx_builder_t *cb, size_t sz,
+                                      const char *g)
+{
+    int found = 0;
+    TT(float);
+    TT(double);
+    assert(found == sz);
+#undef TT
+    return newtype(g);
+}
+
+static crx_type_t *tst_get_function_type(crx_builder_t *cb, crx_type_t *ret,
+                                         crx_type_t *args[], int nargs,
+                                         crx_trampoline_t *trampl)
+{
+    abort();
+}
+
+static crx_type_t *tst_get_ellipsis_function_type(crx_builder_t *cb,
+                                                  crx_type_t *ret,
+                                                  crx_type_t *args[], int nargs)
+{
+    abort();
+}
+
+static crx_type_t *tst_get_pointer_type(crx_builder_t *cb, crx_type_t *t)
+{
+    return newtype2("PTR ", t->text);
+}
+
+static crx_type_t *tst_get_const_type(crx_builder_t *cb, crx_type_t *t)
+{
+    return newtype2("CONST ", t->text);
+}
+
+static crx_type_t *tst_get_array_type(crx_builder_t *cb, crx_type_t *t,
+                                      size_t len)
+{
+    char c[32];
+    sprintf(c, "ARRAY[%zd] ", len);
+    return newtype2(c, t->text);
+}
+
+static crx_type_t *tst_get_incomplete_array_type(crx_builder_t *cb,
+                                                 crx_type_t *t)
+{
+    return newtype2("ARRAY[] ", t->text);
+}
+
+static crx_type_t *tst_get_struct_type(crx_builder_t *cb, const char *name)
+{
+    return newtype2("STRUCT ", name);
+}
+
+static crx_type_t *tst_get_union_type(crx_builder_t *cb, const char *name)
+{
+    return newtype2("UNION ", name);
+}
+
+static crx_type_t *tst_get_enum_type(crx_builder_t *cb, const char *name)
+{
+    return newtype2("ENUM ", name);
+}
+
+static void tst_complete(crx_builder_t *cb, crx_type_t *t,
+                         size_t sz, size_t align,
+                         crx_field_t fields[], int nfields)
+{
+    abort();
+}
+
+static void tst_complete_enum(crx_builder_t *cb, crx_type_t *t,
+                              crx_type_t *inttype)
+{
+    abort();
+}
+
+static void tst_define_type(crx_builder_t *cb, const char *name, crx_type_t *t)
+{
+    printf("TYPEDEF %s = %s\n", name, t->text);
+}
+
+static void tst_define_var(crx_builder_t *cb, const char *name, crx_type_t *t,
+                           void *addr)
+{
+    printf("VAR %s: %s\n", name, t->text);
+}
+
+static void tst_error(crx_builder_t *cb, const char *msg, crx_type_t *t)
+{
+    printf("ERROR %s %s\n", msg, t ? t->text : "");
+}
+
+static crx_builder_t maincb = {
+    tst_get_void_type,
+    tst_get_char_type,
+    tst_get_bool_type,
+    tst_get_signed_type,
+    tst_get_unsigned_type,
+    tst_get_float_type,
+    tst_get_function_type,
+    tst_get_ellipsis_function_type,
+    tst_get_pointer_type,
+    tst_get_const_type,
+    tst_get_array_type,
+    tst_get_incomplete_array_type,
+    tst_get_struct_type,
+    tst_get_union_type,
+    tst_get_enum_type,
+    tst_complete,
+    tst_complete_enum,
+    tst_define_type,
+    tst_define_var,
+    tst_error,
+};
+
+int main(void)
+{
+    TESTFN(&maincb);
+    return 0;
+}
diff --git a/test/codegen/001.c b/test/codegen/001.c
--- a/test/codegen/001.c
+++ b/test/codegen/001.c
@@ -2,10 +2,6 @@
 
 # ____________________________________________________________
 
-int test001(char *r)
+void test001(crx_builder_t *cb)
 {
-    if (!r)
-        return 1;
-    *r = 0;
-    return 0;
 }
diff --git a/test/codegen/002.c b/test/codegen/002.c
--- a/test/codegen/002.c
+++ b/test/codegen/002.c
@@ -2,16 +2,17 @@
 
 # ____________________________________________________________
 
-int test002(char *r)
+void test002(crx_builder_t *cb)
 {
-    if (!r)
-        return 22;
+    crx_type_t *t1, *t2;
     {
         num_t *p1;
         char *p2;
         p1 = (void *)&p2;
         *p1 = (void *)0;    /* check that 'num_t' is a pointer type */
-        r += sprintf(r, "typedef void *num_t;\n");
+        t1 = cb->get_void_type(cb);
+        t2 = cb->get_pointer_type(cb, t1);
+        cb->define_type(cb, "num_t", t2);
+#expect TYPEDEF num_t = PTR void
     }
-    return 0;
 }
diff --git a/test/codegen/002b.c b/test/codegen/002b.c
--- a/test/codegen/002b.c
+++ b/test/codegen/002b.c
@@ -2,10 +2,9 @@
 
 # ____________________________________________________________
 
-int test002b(char *r)
+void test002b(crx_builder_t *cb)
 {
-    if (!r)
-        return 24;
+    crx_type_t *t1, *t2, *t3, *t4;
     {
         num_t *p1;
         char *p2;
@@ -15,7 +14,11 @@
         *p1 = (void *)&p3;  /* check that 'num_t' is a pointer type */
         **p1 = (void *)&p4;  /* check that '*num_t' is a pointer type */
         ***p1 = (void *)0;    /* check that '**num_t' is a pointer type */
-        r += sprintf(r, "typedef void ***num_t;\n");
+        t1 = cb->get_void_type(cb);
+        t2 = cb->get_pointer_type(cb, t1);
+        t3 = cb->get_pointer_type(cb, t2);
+        t4 = cb->get_pointer_type(cb, t3);
+        cb->define_type(cb, "num_t", t4);
+#expect TYPEDEF num_t = PTR PTR PTR void
     }
-    return 0;
 }
diff --git a/test/codegen/002c.c b/test/codegen/002c.c
--- a/test/codegen/002c.c
+++ b/test/codegen/002c.c
@@ -2,16 +2,18 @@
 
 # ____________________________________________________________
 
-int test002c(char *r)
+void test002c(crx_builder_t *cb)
 {
-    if (!r)
-        return 28;
+    crx_type_t *t1, *t2, *t3;
     {
         num_t *p1;
         char *p2;
         p1 = (void *)&p2;
         *p1 = (void *)0;    /* check that 'num_t' is a pointer type */
-        r += sprintf(r, "typedef const void *num_t;\n");
+        t1 = cb->get_void_type(cb);
+        t2 = cb->get_const_type(cb, t1);
+        t3 = cb->get_pointer_type(cb, t2);
+        cb->define_type(cb, "num_t", t3);
+#expect TYPEDEF num_t = PTR CONST void
     }
-    return 0;
 }
diff --git a/test/codegen/003.c b/test/codegen/003.c
--- a/test/codegen/003.c
+++ b/test/codegen/003.c
@@ -2,48 +2,16 @@
 
 # ____________________________________________________________
 
-int test003(char *r)
+void test003(crx_builder_t *cb)
 {
-    if (!r)
-        return 8 + 18 + 9;
+    crx_type_t *t1;
     {
         num_t *p1;
         char b[sizeof(*p1)];
         p1 = (void *)b;
         (void)(*p1 << 1);  /* check that 'num_t' is an integer type */
-        r += sprintf(r, "typedef ");
         *p1 = -1;  /* check that 'num_t' is not declared 'const' */
-        if (*p1 > 0) {
-            if (sizeof(*p1) == 1 && *p1 == 1)
-                r += sprintf(r, "_Bool");
-            else if (sizeof(*p1) == sizeof(unsigned int))
-                r += sprintf(r, "unsigned int");
-            else if (sizeof(*p1) == sizeof(unsigned short))
-                r += sprintf(r, "unsigned short");
-            else if (sizeof(*p1) == sizeof(unsigned char))
-                r += sprintf(r, "unsigned char");
-            else if (sizeof(*p1) == sizeof(unsigned long))
-                r += sprintf(r, "unsigned long");
-            else if (sizeof(*p1) == sizeof(unsigned long long))
-                r += sprintf(r, "unsigned long long");
-            else
-                r += sprintf(r, "uint%u_t", (int)sizeof(*p1) * 8);
-        }
-        else {
-            if (sizeof(*p1) == sizeof(int))
-                r += sprintf(r, "int");
-            else if (sizeof(*p1) == sizeof(short))
-                r += sprintf(r, "short");
-            else if (sizeof(*p1) == sizeof(signed char))
-                r += sprintf(r, "signed char");
-            else if (sizeof(*p1) == sizeof(long))
-                r += sprintf(r, "long");
-            else if (sizeof(*p1) == sizeof(long long))
-                r += sprintf(r, "long long");
-            else
-                r += sprintf(r, "int%u_t", (int)sizeof(*p1) * 8);
-        }
-        r += sprintf(r, " num_t;\n");
+        t1 = CRX_INT_TYPE(cb, *p1, "int");
+        cb->define_type(cb, "num_t", t1);
     }
-    return 0;
 }
diff --git a/test/codegen/003b.c b/test/codegen/003b.c
--- a/test/codegen/003b.c
+++ b/test/codegen/003b.c
@@ -2,48 +2,15 @@
 
 # ____________________________________________________________
 
-int test003b(char *r)
+void test003b(creflect_builder_t *cb)
 {
-    if (!r)
-        return 8 + 18 + 9;
     {
         num_t *p1;
         char b[sizeof(*p1)];
         p1 = (void *)b;
         (void)(*p1 << 1);  /* check that 'num_t' is an integer type */
-        r += sprintf(r, "typedef ");
         *p1 = -1;  /* check that 'num_t' is not declared 'const' */
-        if (*p1 > 0) {
-            if (sizeof(*p1) == 1 && *p1 == 1)
-                r += sprintf(r, "_Bool");
-            else if (sizeof(*p1) == sizeof(unsigned long))
-                r += sprintf(r, "unsigned long");
-            else if (sizeof(*p1) == sizeof(unsigned int))
-                r += sprintf(r, "unsigned int");
-            else if (sizeof(*p1) == sizeof(unsigned short))
-                r += sprintf(r, "unsigned short");
-            else if (sizeof(*p1) == sizeof(unsigned char))
-                r += sprintf(r, "unsigned char");
-            else if (sizeof(*p1) == sizeof(unsigned long long))
-                r += sprintf(r, "unsigned long long");
-            else
-                r += sprintf(r, "uint%u_t", (int)sizeof(*p1) * 8);
-        }
-        else {
-            if (sizeof(*p1) == sizeof(long))
-                r += sprintf(r, "long");
-            else if (sizeof(*p1) == sizeof(int))
-                r += sprintf(r, "int");
-            else if (sizeof(*p1) == sizeof(short))
-                r += sprintf(r, "short");
-            else if (sizeof(*p1) == sizeof(signed char))
-                r += sprintf(r, "signed char");
-            else if (sizeof(*p1) == sizeof(long long))
-                r += sprintf(r, "long long");
-            else
-                r += sprintf(r, "int%u_t", (int)sizeof(*p1) * 8);
-        }
-        r += sprintf(r, " num_t;\n");
+        void *t = CB__INT_TYPE(*p1, "long");
+        cb->define_type("num_t", t);
     }
-    return 0;
 }
diff --git a/test/codegen/003c.c b/test/codegen/003c.c
--- a/test/codegen/003c.c
+++ b/test/codegen/003c.c
@@ -2,48 +2,15 @@
 
 # ____________________________________________________________
 
-int test003c(char *r)
+void test003c(creflect_builder_t *cb)
 {
-    if (!r)
-        return 8 + 18 + 9;
     {
         num_t *p1;
         char b[sizeof(*p1)];
         p1 = (void *)b;
         (void)(*p1 << 1);  /* check that 'num_t' is an integer type */
-        r += sprintf(r, "typedef ");
         *p1 = -1;  /* check that 'num_t' is not declared 'const' */
-        if (*p1 > 0) {
-            if (sizeof(*p1) == 1 && *p1 == 1)
-                r += sprintf(r, "_Bool");
-            else if (sizeof(*p1) == sizeof(unsigned long long))
-                r += sprintf(r, "unsigned long long");
-            else if (sizeof(*p1) == sizeof(unsigned long))
-                r += sprintf(r, "unsigned long");
-            else if (sizeof(*p1) == sizeof(unsigned int))
-                r += sprintf(r, "unsigned int");
-            else if (sizeof(*p1) == sizeof(unsigned short))
-                r += sprintf(r, "unsigned short");
-            else if (sizeof(*p1) == sizeof(unsigned char))
-                r += sprintf(r, "unsigned char");
-            else
-                r += sprintf(r, "uint%u_t", (int)sizeof(*p1) * 8);
-        }
-        else {
-            if (sizeof(*p1) == sizeof(long long))
-                r += sprintf(r, "long long");
-            else if (sizeof(*p1) == sizeof(long))
-                r += sprintf(r, "long");
-            else if (sizeof(*p1) == sizeof(int))
-                r += sprintf(r, "int");
-            else if (sizeof(*p1) == sizeof(short))
-                r += sprintf(r, "short");
-            else if (sizeof(*p1) == sizeof(signed char))
-                r += sprintf(r, "signed char");
-            else
-                r += sprintf(r, "int%u_t", (int)sizeof(*p1) * 8);
-        }
-        r += sprintf(r, " num_t;\n");
+        void *t = CB__INT_TYPE(*p1, "long long");
+        cb->define_type("num_t", t);
     }
-    return 0;
 }
diff --git a/test/codegen/003d.c b/test/codegen/003d.c
--- a/test/codegen/003d.c
+++ b/test/codegen/003d.c
@@ -2,48 +2,15 @@
 
 # ____________________________________________________________
 
-int test003d(char *r)
+void test003d(creflect_builder_t *cb)
 {
-    if (!r)
-        return 8 + 18 + 9;
     {
         num_t *p1;
         char b[sizeof(*p1)];
         p1 = (void *)b;
         (void)(*p1 << 1);  /* check that 'num_t' is an integer type */
-        r += sprintf(r, "typedef ");
         *p1 = -1;  /* check that 'num_t' is not declared 'const' */
-        if (*p1 > 0) {
-            if (sizeof(*p1) == 1 && *p1 == 1)
-                r += sprintf(r, "_Bool");
-            else if (sizeof(*p1) == sizeof(unsigned char))
-                r += sprintf(r, "unsigned char");
-            else if (sizeof(*p1) == sizeof(unsigned short))
-                r += sprintf(r, "unsigned short");
-            else if (sizeof(*p1) == sizeof(unsigned int))
-                r += sprintf(r, "unsigned int");
-            else if (sizeof(*p1) == sizeof(unsigned long))
-                r += sprintf(r, "unsigned long");
-            else if (sizeof(*p1) == sizeof(unsigned long long))
-                r += sprintf(r, "unsigned long long");
-            else
-                r += sprintf(r, "uint%u_t", (int)sizeof(*p1) * 8);
-        }
-        else {
-            if (sizeof(*p1) == sizeof(signed char))
-                r += sprintf(r, "signed char");
-            else if (sizeof(*p1) == sizeof(short))
-                r += sprintf(r, "short");
-            else if (sizeof(*p1) == sizeof(int))
-                r += sprintf(r, "int");
-            else if (sizeof(*p1) == sizeof(long))
-                r += sprintf(r, "long");
-            else if (sizeof(*p1) == sizeof(long long))
-                r += sprintf(r, "long long");
-            else
-                r += sprintf(r, "int%u_t", (int)sizeof(*p1) * 8);
-        }
-        r += sprintf(r, " num_t;\n");
+        void *t = CB__INT_TYPE(*p1, "char");
+        cb->define_type(cb, "num_t", t);
     }
-    return 0;
 }
diff --git a/test/codegen/003e.c b/test/codegen/003e.c
--- a/test/codegen/003e.c
+++ b/test/codegen/003e.c
@@ -2,48 +2,18 @@
 
 # ____________________________________________________________
 
-int test003e(char *r)
+void test003e(crx_self_t *self, crx_builder_t *cb)
 {
-    if (!r)
-        return 8 + 18 + 9;
+    crx_type_t *t1;
     {
         num_t *p1;
         char b[sizeof(*p1)];
         p1 = (void *)b;
         (void)(*p1 << 1);  /* check that 'num_t' is an integer type */
-        r += sprintf(r, "typedef ");
         *p1 = -1;  /* check that 'num_t' is not declared 'const' */
-        if (*p1 > 0) {
-            if (sizeof(*p1) == 1 && *p1 == 1)
-                r += sprintf(r, "_Bool");
-            else if (sizeof(*p1) == sizeof(char))
-                r += sprintf(r, "char");
-            else if (sizeof(*p1) == sizeof(unsigned short))
-                r += sprintf(r, "unsigned short");
-            else if (sizeof(*p1) == sizeof(unsigned int))
-                r += sprintf(r, "unsigned int");
-            else if (sizeof(*p1) == sizeof(unsigned long))
-                r += sprintf(r, "unsigned long");
-            else if (sizeof(*p1) == sizeof(unsigned long long))
-                r += sprintf(r, "unsigned long long");
-            else
-                r += sprintf(r, "uint%u_t", (int)sizeof(*p1) * 8);
-        }
-        else {
-            if (sizeof(*p1) == sizeof(char))
-                r += sprintf(r, "char");
-            else if (sizeof(*p1) == sizeof(short))
-                r += sprintf(r, "short");
-            else if (sizeof(*p1) == sizeof(int))
-                r += sprintf(r, "int");
-            else if (sizeof(*p1) == sizeof(long))
-                r += sprintf(r, "long");
-            else if (sizeof(*p1) == sizeof(long long))
-                r += sprintf(r, "long long");
-            else
-                r += sprintf(r, "int%u_t", (int)sizeof(*p1) * 8);
-        }
-        r += sprintf(r, " num_t;\n");
+        char b2[sizeof(*p1) == 1 ? 1 : -1];  /* check that 'num_t' is a single-byte integer */
+        (void)b2;
+        t1 = cb->get_char_type(self);
+        cb->define_type(self, "num_t", t1);
     }
-    return 0;
 }
diff --git a/test/codegen/003f.c b/test/codegen/003f.c
--- a/test/codegen/003f.c
+++ b/test/codegen/003f.c
@@ -2,48 +2,15 @@
 
 # ____________________________________________________________
 
-int test003f(char *r)
+void test003f(creflect_builder_t *cb)
 {
-    if (!r)
-        return 8 + 18 + 9;
     {
         num_t *p1;
         char b[sizeof(*p1)];
         p1 = (void *)b;
         (void)(*p1 << 1);  /* check that 'num_t' is an integer type */
-        r += sprintf(r, "typedef ");
         *p1 = -1;  /* check that 'num_t' is not declared 'const' */
-        if (*p1 > 0) {
-            if (sizeof(*p1) == 1 && *p1 == 1)
-                r += sprintf(r, "_Bool");
-            else if (sizeof(*p1) == sizeof(unsigned long long))
-                r += sprintf(r, "unsigned long long");
-            else if (sizeof(*p1) == sizeof(unsigned long))
-                r += sprintf(r, "unsigned long");
-            else if (sizeof(*p1) == sizeof(unsigned int))
-                r += sprintf(r, "unsigned int");
-            else if (sizeof(*p1) == sizeof(unsigned short))
-                r += sprintf(r, "unsigned short");
-            else if (sizeof(*p1) == sizeof(unsigned char))
-                r += sprintf(r, "unsigned char");
-            else
-                r += sprintf(r, "uint%u_t", (int)sizeof(*p1) * 8);
-        }
-        else {
-            if (sizeof(*p1) == sizeof(long long))
-                r += sprintf(r, "long long");
-            else if (sizeof(*p1) == sizeof(long))
-                r += sprintf(r, "long");
-            else if (sizeof(*p1) == sizeof(int))
-                r += sprintf(r, "int");
-            else if (sizeof(*p1) == sizeof(short))
-                r += sprintf(r, "short");
-            else if (sizeof(*p1) == sizeof(signed char))
-                r += sprintf(r, "signed char");
-            else
-                r += sprintf(r, "int%u_t", (int)sizeof(*p1) * 8);
-        }
-        r += sprintf(r, " num_t;\n");
+        void *t = CB__INT_TYPE(cb, *p1, "unsigned long long");
+        cb->define_type(cb, "num_t", t);
     }
-    return 0;
 }
diff --git a/test/codegen/004.c b/test/codegen/004.c
--- a/test/codegen/004.c
+++ b/test/codegen/004.c
@@ -2,10 +2,8 @@
 
 # ____________________________________________________________
 
-int test004(char *r)
+void test004(creflect_builder_t *cb)
 {
-    if (!r)
-        return 8 + 18 + 10;
     {
         num_t *p1;
         char *p2;
@@ -13,8 +11,18 @@
         p1 = (void *)&p2;
         *p1 = (void *)b;    /* check that 'num_t' is a pointer type */
         (void)(**p1 << 1);  /* check that '*num_t' is an integer type */
-        r += sprintf(r, "typedef ");
         **p1 = -1;  /* check that '*num_t' is not declared 'const' */
+        void *t;
+        if (*p1 > 0) {
+            if (sizeof(*p1) == 1 && *p1 == 1)
+                t = cb->get_bool_type();
+            else
+                t = cb->get_signed_type(sizeof(*p1), "int");
+        }
+        else {
+            t = cb->get_unsigned_type(sizeof(*p1), "int");
+        }
+        
         if (**p1 > 0) {
             if (sizeof(**p1) == 1 && **p1 == 1)
                 r += sprintf(r, "_Bool");
diff --git a/test/codegen/006.c b/test/codegen/006.c
--- a/test/codegen/006.c
+++ b/test/codegen/006.c
@@ -10,10 +10,10 @@
     {
         num_t *p1;
         char b[sizeof(*p1)];
+        memset(b, -1, sizeof(b));
         p1 = (void *)b;
         (void)(*p1 << 1);  /* check that 'num_t' is an integer type */
         r += sprintf(r, "typedef const ");
-        memset(b, -1, sizeof(b));
         if (*p1 > 0) {
             if (sizeof(*p1) == 1 && *p1 == 1)
                 r += sprintf(r, "_Bool");
diff --git a/test/codegen/007.c b/test/codegen/007.c
new file mode 100644
--- /dev/null
+++ b/test/codegen/007.c
@@ -0,0 +1,22 @@
+typedef int (*myfunc_t)(long, unsigned long long);
+
+# ____________________________________________________________
+
+static void __creflect_t_myfunc_t(void *func, void *args[], void *result) {
+    myfunc_t f = func;
+    *(int *)result = f(*(long *)args[0], *(unsigned long long *)args[1]);
+}
+
+int test007(char *r)
+{
+    if (!r)
+        return 65;
+    {
+        myfunc_t *p1;
+        char *p2;
+        p1 = (void *)&p2;
+        *p1 = (void *)0;    /* check that 'myfunc_t' is a pointer type */
+        r += sprintf(r, "typedef int (*myfunc_t)(long, unsigned long long)/*%p*/;\n", &__creflect_t_myfunc_t);
+    }
+    return 0;
+}
diff --git a/test/codegen/func-001.c b/test/codegen/func-001.c
--- a/test/codegen/func-001.c
+++ b/test/codegen/func-001.c
@@ -6,7 +6,7 @@
 
 # ____________________________________________________________
 
-static void __creflect_t1(void *func, void *args[], void *result) {
+static void __creflect_t2(void *func, void *args[], void *result) {
     int(*f)(void) = func;
     *(int *)result = f();
 }
@@ -15,11 +15,12 @@
     return f();
 }
 
-int testfunc_001(char *r)
+void testfunc_001(crx_builder_t *cb)
 {
-    if (!r)
-        return 3 + 24 + 24 + 18 + 1;
-    r += sprintf(r, "/*%p*/", &__creflect_d_f);
-    r += sprintf(r, "int f(void)/*%p*/;\n", &__creflect_t1);
-    return 0;
+    crx_type *t1, *t2;
+    {
+        t1 = cb->get_signed_type(cb, sizeof(int), "int");
+        t2 = cb->get_function_type(cb, t1, 0, 0, &__creflect_t2);
+        cb->define_var(cb, "f", t2, &__creflect_d_f);
+    }
 }
diff --git a/test/codegen/func-001b.c b/test/codegen/func-001b.c
new file mode 100644
--- /dev/null
+++ b/test/codegen/func-001b.c
@@ -0,0 +1,46 @@
+unsigned int f(long long);
+long g(int, int);
+long h(int, int);
+
+# ____________________________________________________________
+
+unsigned int f(long long a) { return (unsigned int)(a / 1291); }
+long g(int a, int b) { return b - a; }
+long h(int a, int b) { return b * (long)a; }
+
+# ____________________________________________________________
+
+static void __creflect_t1(void *func, void *args[], void *result) {
+    unsigned int(*f)(long long) = func;
+    *(unsigned int *)result = f(*(long long *)args[0]);
+}
+
+static unsigned int __creflect_d_f(long long a0) {
+    return f(a0);
+}
+
+static void __creflect_t2(void *func, void *args[], void *result) {
+    long(*f)(int, int) = func;
+    *(long *)result = f(*(int *)args[0], *(int *)args[1]);
+}
+
+static long __creflect_d_g(int a0, int a1) {
+    return g(a0, a1);
+}
+
+static long __creflect_d_h(int a0, int a1) {
+    return h(a0, a1);
+}
+
+int testfunc_002(char *r)
+{
+    if (!r)
+        return 3 + 999 + 1;
+    r += sprintf(r, "unsigned int");
+    r += sprintf(r, " /*%p*/f(long long)/*%p*/;\n", &__creflect_d_f, &__creflect_t1);
+    r += sprintf(r, "long");
+    r += sprintf(r, " /*%p*/g(int, int)/*%p*/;\n", &__creflect_d_g, &__creflect_t2);
+    r += sprintf(r, "long");
+    r += sprintf(r, " /*%p*/h(int, int)/*%p*/;\n", &__creflect_d_h, &__creflect_t2);
+    return 0;
+}
diff --git a/test/codegen/func-002.c b/test/codegen/func-002.c
--- a/test/codegen/func-002.c
+++ b/test/codegen/func-002.c
@@ -1,46 +1,20 @@
-unsigned int f(long long);
-long g(int, int);
-long h(int, int);
+
+int f(int, ...);
 
 # ____________________________________________________________
 
-unsigned int f(long long a) { return (unsigned int)(a / 1291); }
-long g(int a, int b) { return b - a; }
-long h(int a, int b) { return b * (long)a; }
+int f(int a, ...) { return 42 * a; }
 
 # ____________________________________________________________
 
-static void __creflect_t1(void *func, void *args[], void *result) {
-    unsigned int(*f)(long long) = func;
-    *(unsigned int *)result = f(*(long long *)args[0]);
+void testfunc_002(crx_builder_t *cb)
+{
+    crx_type *t1, *a2[1], *t2;
+    {
+        int (*p1)(int, ...) = f;  /* check that 'f' is a function with exactly the given signature */
+        t1 = cb->get_signed_type(cb, sizeof(int), "int");
+        a2[0] = t1;
+        t2 = cb->get_ellipsis_function_type(cb, t1, a2, 1);
+        cb->define_var(cb, "f", t2, p1);
+    }
 }
-
-static unsigned int __creflect_d_f(long long a0) {
-    return f(a0);
-}
-
-static void __creflect_t2(void *func, void *args[], void *result) {
-    long(*f)(int, int) = func;
-    *(long *)result = f(*(int *)args[0], *(int *)args[1]);
-}
-
-static long __creflect_d_g(int a0, int a1) {
-    return g(a0, a1);
-}
-
-static long __creflect_d_h(int a0, int a1) {
-    return h(a0, a1);
-}
-
-int testfunc_002(char *r)
-{
-    if (!r)
-        return 3 + 999 + 1;
-    r += sprintf(r, "unsigned int");
-    r += sprintf(r, " /*%p*/f(long long)/*%p*/;\n", &__creflect_d_f, &__creflect_t1);
-    r += sprintf(r, "long");
-    r += sprintf(r, " /*%p*/g(int, int)/*%p*/;\n", &__creflect_d_g, &__creflect_t2);
-    r += sprintf(r, "long");
-    r += sprintf(r, " /*%p*/h(int, int)/*%p*/;\n", &__creflect_d_h, &__creflect_t2);
-    return 0;
-}
diff --git a/test/codegen/func-003.c b/test/codegen/func-003.c
--- a/test/codegen/func-003.c
+++ b/test/codegen/func-003.c
@@ -1,24 +1,25 @@
-typedef int(*func_t)(long);
+typedef int(*func_t)(long, long);
 
 # ____________________________________________________________
 
-static void __creflect_t1(void *func, void *args[], void *result) {
-    int(*f)(long) = func;
-    *(int *)result = f(*(long *)args[0]);
+static void __creflect_t3(void *func, void *args[], void *result) {
+    func_t f = func;
+    *(int *)result = f(*(long *)args[0], *(long *)args[1]);
 }
 
-int testfunc_003(char *r)
+void testfunc_003(crx_builder_t *cb)
 {
-    if (!r)
-        return 8 + 3 + 17 + 24 + 4 + 1;
-    r += sprintf(r, "typedef ");
+    crx_type *t1, *t2, *a3[2], *t3;
     {
         func_t *p1;
         char *p2;
         p1 = (void *)&p2;
         *p1 = (void *)0;    /* check that 'func_t' is a pointer type */
-        r += sprintf(r, "int");
+        t1 = cb->get_signed_type(cb, sizeof(int), "int");
+        t2 = cb->get_signed_type(cb, sizeof(long), "long");
+        a3[0] = t2;
+        a3[1] = t2;
+        t3 = cb->get_function_type(cb, t1, a3, 2, &__creflect_t3);
+        cb->define_type(cb, "func_t", t3);
     }
-    r += sprintf(r, "(*func_t)(long)/*%p*/;\n", __creflect_t1);
-    return 0;
 }
diff --git a/test/codegen/glob-001.c b/test/codegen/glob-001.c
--- a/test/codegen/glob-001.c
+++ b/test/codegen/glob-001.c
@@ -5,12 +5,12 @@
 int testglob_001(char *r)
 {
     if (!r)
-        return 3 + 24 + 4 + 24 + 1;
+        return 28 + 17;
     {
-        void *p1 = someglob;  /* check that 'someglob' is a pointer, of a type compatible with 'void *' */
+        void *p1 = someglob;  /* check that 'someglob' is a pointer */
+        r += sprintf(r, "/*%p*/", &someglob);
         if (0) { someglob = p1; }  /* check that 'someglob' is a pointer variable, and not an array or a constant */
-        r += sprintf(r, "void *someglob");
+        r += sprintf(r, "void *someglob;\n");
     }
-    r += sprintf(r, ";/*%p*/\n", &someglob);
     return 0;
 }
diff --git a/test/codegen/glob-002.c b/test/codegen/glob-002.c
--- a/test/codegen/glob-002.c
+++ b/test/codegen/glob-002.c
@@ -2,15 +2,14 @@
 
 # ____________________________________________________________
 
-int testglob_002(char *r)
+void testglob_002(crx_builder_t *cb)
 {
-    if (!r)
-        return 3 + 24 + 4 + 24 + 1;
+    crx_type *t1, *t2;
     {
         char (*p1)[] = &someglob;  /* check that 'someglob' is of type 'char[]' */
         (void)p1;
-        r += sprintf(r, "char someglob[%ld]", (long)(sizeof(someglob) / sizeof(*someglob)));
+        t1 = cb->get_char_type(cb);
+        t2 = cb->get_array_type(cb, t1, sizeof(someglob) / sizeof(*someglob));
+        cb->define_var(cb, "someglob", t2, &someglob);
     }
-    r += sprintf(r, ";/*%p*/\n", someglob);
-    return 0;
 }
diff --git a/test/codegen/glob-004.c b/test/codegen/glob-004.c
new file mode 100644
--- /dev/null
+++ b/test/codegen/glob-004.c
@@ -0,0 +1,21 @@
+int (*someglob)(long, long);
+
+# ____________________________________________________________
+
+static void __creflect_t1(void *func, void *args[], void *result) {
+    int(*f)(long, long) = func;
+    *(int *)result = f(*(long *)args[0], *(long *)args[1]);
+}
+
+int testglob_004(char *r)
+{
+    if (!r)
+        return 100;
+    {
+        int (**p1)(long, long) = &someglob;  /* check the exact type of 'someglob' */
+        (void)p1;
+        r += sprintf(r, "/*%p*/", &someglob);
+        r += sprintf(r, "int (*someglob)(long, long)/*%p*/;\n", &__creflect_t1);
+    }
+    return 0;
+}
diff --git a/test/codegen/struct-001.c b/test/codegen/struct-001.c
--- a/test/codegen/struct-001.c
+++ b/test/codegen/struct-001.c
@@ -1,54 +1,29 @@
 struct foo_s {
-  int aa;
-  unsigned int bb;
+    int aa;
+    unsigned int bb;
 };
 
 # ____________________________________________________________
 
-int teststruct_001(char *r)
+void teststruct_001(crx_builder_t *cb)
 {
-    if (!r)
-        return 69 + 30 + 18 + 6 + 30 + 18 + 6 + 4;
-    r += sprintf(r, "struct foo_s /*%lld,%lld*/{\n", (long long)sizeof(struct foo_s), (long long)(((char *)&((struct{char a; struct foo_s b;} *)0)->b) - (char *)0));
+    crx_type_t *t1, *t2, *t3;
+    crx_field_t d1[2];
+    t1 = cb->get_struct_type(cb, "foo_s");
     {
         struct foo_s *p1;
-        long long o = ((char *)&((struct foo_s *)0)->aa) - (char *)0;  /* check that 'struct foo_s::aa' is not an array */
+        size_t o = ((char *)&((struct foo_s *)0)->aa) - (char *)0;  /* check that 'struct foo_s::aa' is not an array */
         char b[sizeof(p1->aa)];
         r += sprintf(r, "  /*%lld*/", o);
         p1 = (void *)(((char *)b) - o);
         (void)(p1->aa << 1);  /* check that 'struct foo_s::aa' is an integer type */
         p1->aa = -1;  /* check that 'struct foo_s::aa' is not declared 'const' */
-        if (p1->aa > 0) {
-            if (sizeof(p1->aa) == 1 && p1->aa == 1)
-                r += sprintf(r, "_Bool");
-            else if (sizeof(p1->aa) == sizeof(unsigned int))
-                r += sprintf(r, "unsigned int");
-            else if (sizeof(p1->aa) == sizeof(unsigned short))
-                r += sprintf(r, "unsigned short");
-            else if (sizeof(p1->aa) == sizeof(unsigned char))
-                r += sprintf(r, "unsigned char");
-            else if (sizeof(p1->aa) == sizeof(unsigned long))
-                r += sprintf(r, "unsigned long");
-            else if (sizeof(p1->aa) == sizeof(unsigned long long))
-                r += sprintf(r, "unsigned long long");
-            else
-                r += sprintf(r, "uint%u_t", (int)sizeof(p1->aa) * 8);
-        }
-        else {
-            if (sizeof(p1->aa) == sizeof(int))
-                r += sprintf(r, "int");
-            else if (sizeof(p1->aa) == sizeof(short))
-                r += sprintf(r, "short");
-            else if (sizeof(p1->aa) == sizeof(signed char))
-                r += sprintf(r, "signed char");
-            else if (sizeof(p1->aa) == sizeof(long))
-                r += sprintf(r, "long");
-            else if (sizeof(p1->aa) == sizeof(long long))
-                r += sprintf(r, "long long");
-            else
-                r += sprintf(r, "int%u_t", (int)sizeof(p1->aa) * 8);
-        }
-        r += sprintf(r, " aa;\n");
+        t2 = CRX_INT_TYPE(cb, p1->aa, "int");
+        d1[0].name = "aa";
+        d1[0].type = t2;
+        d1[0].offset = o;
+        d1[0].numbits = -1;
+        d1[0].bitshift = -1;
     }
     {
         struct foo_s *p1;
@@ -58,38 +33,14 @@
         p1 = (void *)(((char *)b) - o);
         (void)(p1->bb << 1);  /* check that 'struct foo_s::bb' is an integer type */
         p1->bb = -1;  /* check that 'struct foo_s::bb' is not declared 'const' */
-        if (p1->bb > 0) {
-            if (sizeof(p1->bb) == 1 && p1->bb == 1)
-                r += sprintf(r, "_Bool");
-            else if (sizeof(p1->bb) == sizeof(unsigned int))
-                r += sprintf(r, "unsigned int");
-            else if (sizeof(p1->bb) == sizeof(unsigned short))
-                r += sprintf(r, "unsigned short");
-            else if (sizeof(p1->bb) == sizeof(unsigned char))
-                r += sprintf(r, "unsigned char");
-            else if (sizeof(p1->bb) == sizeof(unsigned long))
-                r += sprintf(r, "unsigned long");
-            else if (sizeof(p1->bb) == sizeof(unsigned long long))
-                r += sprintf(r, "unsigned long long");
-            else
-                r += sprintf(r, "uint%u_t", (int)sizeof(p1->bb) * 8);
-        }
-        else {
-            if (sizeof(p1->bb) == sizeof(int))
-                r += sprintf(r, "int");
-            else if (sizeof(p1->bb) == sizeof(short))
-                r += sprintf(r, "short");
-            else if (sizeof(p1->bb) == sizeof(signed char))
-                r += sprintf(r, "signed char");
-            else if (sizeof(p1->bb) == sizeof(long))
-                r += sprintf(r, "long");
-            else if (sizeof(p1->bb) == sizeof(long long))
-                r += sprintf(r, "long long");
-            else
-                r += sprintf(r, "int%u_t", (int)sizeof(p1->bb) * 8);
-        }
-        r += sprintf(r, " bb;\n");
+        t3 = CRX_INT_TYPE(cb, p1->bb, "int");
+        d1[0].name = "bb";
+        d1[0].type = t3;
+        d1[0].offset = o;
+        d1[0].numbits = -1;
+        d1[0].bitshift = -1;
     }
-    r += sprintf(r, "};\n");
-    return 0;
+    cb->complete(cb, t1, sizeof(struct foo_s),
+                 offsetof(struct{char a; struct foo_s b;}, b),
+                 d1, 2);
 }
diff --git a/test/codegen/struct-001b.c b/test/codegen/struct-001b.c
new file mode 100644
--- /dev/null
+++ b/test/codegen/struct-001b.c
@@ -0,0 +1,56 @@
+struct foo_s /*4,4*/{
+  /*0*/const int aa;
+};
+
+# ____________________________________________________________
+#include 
+
+int teststruct_001b(char *r)
+{
+    if (!r)
+        return 100;
+    r += sprintf(r, "struct foo_s /*%lld,%lld*/{\n", (long long)sizeof(struct foo_s), (long long)(((char *)&((struct{char a; struct foo_s b;} *)0)->b) - (char *)0));
+    {
+        struct foo_s *p1;
+        long long o = ((char *)&((struct foo_s *)0)->aa) - (char *)0;  /* check that 'struct foo_s::aa' is not an array */
+        char b[sizeof(p1->aa)];
+        memset(b, -1, sizeof(b));
+        r += sprintf(r, "  /*%lld*/", o);
+        p1 = (void *)(((char *)b) - o);
+        (void)(p1->aa << 1);  /* check that 'struct foo_s::aa' is an integer type */
+        r += sprintf(r, "const ");
+        if (p1->aa > 0) {
+            if (sizeof(p1->aa) == 1 && p1->aa == 1)
+                r += sprintf(r, "_Bool");
+            else if (sizeof(p1->aa) == sizeof(unsigned int))
+                r += sprintf(r, "unsigned int");
+            else if (sizeof(p1->aa) == sizeof(unsigned short))
+                r += sprintf(r, "unsigned short");
+            else if (sizeof(p1->aa) == sizeof(unsigned char))
+                r += sprintf(r, "unsigned char");
+            else if (sizeof(p1->aa) == sizeof(unsigned long))
+                r += sprintf(r, "unsigned long");
+            else if (sizeof(p1->aa) == sizeof(unsigned long long))
+                r += sprintf(r, "unsigned long long");
+            else
+                r += sprintf(r, "uint%u_t", (int)sizeof(p1->aa) * 8);
+        }
+        else {
+            if (sizeof(p1->aa) == sizeof(int))
+                r += sprintf(r, "int");
+            else if (sizeof(p1->aa) == sizeof(short))
+                r += sprintf(r, "short");
+            else if (sizeof(p1->aa) == sizeof(signed char))
+                r += sprintf(r, "signed char");
+            else if (sizeof(p1->aa) == sizeof(long))
+                r += sprintf(r, "long");
+            else if (sizeof(p1->aa) == sizeof(long long))
+                r += sprintf(r, "long long");
+            else
+                r += sprintf(r, "int%u_t", (int)sizeof(p1->aa) * 8);
+        }
+        r += sprintf(r, " aa;\n");
+    }
+    r += sprintf(r, "};\n");
+    return 0;
+}
diff --git a/test/codegen/struct-002.c b/test/codegen/struct-002.c
--- a/test/codegen/struct-002.c
+++ b/test/codegen/struct-002.c
@@ -1,26 +1,23 @@
 struct foo_s {
-    void *aa;
+  void *aa;
 };
 
 # ____________________________________________________________
 
-static void __creflect1(void r(char *, void**, int))
+int teststruct_002(char *r)
 {
-    void *a[2];
-    __CREFLECT_PREV(r);
-    a[0] = (void *)sizeof(struct foo_s);  /* size */
-    a[1] = &((struct{char a; struct foo_s b;}*)0)->b;  /* align */
-    r("struct foo_s", a, 2);
+    if (!r)
+        return 100;
+    r += sprintf(r, "struct foo_s /*%lld,%lld*/{\n", (long long)sizeof(struct foo_s), (long long)(((char *)&((struct{char a; struct foo_s b;} *)0)->b) - (char *)0));
     {
         struct foo_s *p1;
-        void *o = &((struct foo_s *)0)->aa;  /* offset */
-        char *p2;
-        a[1] = o;
-        p1 = (void *)(((char *)&p2) - (long)o);
+        long long o = ((char *)&((struct foo_s *)0)->aa) - (char *)0;  /* check that 'struct foo_s::aa' is not an array */
+        char b[sizeof(p1->aa)];
+        r += sprintf(r, "  /*%lld*/", o);
+        p1 = (void *)(((char *)b) - o);
         p1->aa = (void *)0;    /* check that 'struct foo_s::aa' is a pointer type */
+        r += sprintf(r, "void *aa;\n");
     }
-    r("{aa:void*", a, 2);
+    r += sprintf(r, "};\n");
+    return 0;
 }
-
-#expect  struct foo_s  8  8
-#expect  {aa:void*  99  0
diff --git a/test/codegen/struct-003.c b/test/codegen/struct-003.c
--- a/test/codegen/struct-003.c
+++ b/test/codegen/struct-003.c
@@ -1,30 +1,56 @@
 struct foo_s {
-    int *aa;
+  int *aa;
 };
 
 # ____________________________________________________________
 
-static void __creflect1(void r(char *, void**, int))
+int teststruct_003(char *r)
 {
-    void *a[3];
-    __CREFLECT_PREV(r);
-    a[0] = (void *)sizeof(struct foo_s);  /* size */
-    a[1] = &((struct{char a; struct foo_s b;}*)0)->b;  /* align */
-    r("struct foo_s", a, 2);
+    if (!r)
+        return 100;
+    r += sprintf(r, "struct foo_s /*%lld,%lld*/{\n", (long long)sizeof(struct foo_s), (long long)(((char *)&((struct{char a; struct foo_s b;} *)0)->b) - (char *)0));
     {
         struct foo_s *p1;
-        void *o = &((struct foo_s *)0)->aa;  /* offset */
         char *p2;
+        long long o = ((char *)&((struct foo_s *)0)->aa) - (char *)0;  /* check that 'struct foo_s::aa' is not an array */
         char b[sizeof(*p1->aa)];  /* check that '*struct foo_s::aa' is a valid type */
-        a[1] = o;
-        p1 = (void *)(((char *)&p2) - (long)o);
+        r += sprintf(r, "  /*%lld*/", o);
+        p1 = (void *)(((char *)&p2) - o);
         p1->aa = (void *)b;    /* check that 'struct foo_s::aa' is a pointer type */
         (void)(*p1->aa << 1);  /* check that '*struct foo_s::aa' is an integer type */
-        *p1->aa = -1;
-        a[2] = (void *)(*p1->aa > 0 ? sizeof(*p1->aa) : -sizeof(*p1->aa));
+        *p1->aa = -1;  /* check that '*struct foo_s::aa' is not declared 'const' */
+        if (*p1->aa > 0) {
+            if (sizeof(*p1->aa) == 1 && *p1->aa == 1)
+                r += sprintf(r, "_Bool");
+            else if (sizeof(*p1->aa) == sizeof(unsigned int))
+                r += sprintf(r, "unsigned int");
+            else if (sizeof(*p1->aa) == sizeof(unsigned short))
+                r += sprintf(r, "unsigned short");
+            else if (sizeof(*p1->aa) == sizeof(unsigned char))
+                r += sprintf(r, "unsigned char");
+            else if (sizeof(*p1->aa) == sizeof(unsigned long))
+                r += sprintf(r, "unsigned long");
+            else if (sizeof(*p1->aa) == sizeof(unsigned long long))
+                r += sprintf(r, "unsigned long long");
+            else
+                r += sprintf(r, "uint%u_t", (int)sizeof(*p1->aa) * 8);
+        }
+        else {
+            if (sizeof(*p1->aa) == sizeof(int))
+                r += sprintf(r, "int");
+            else if (sizeof(*p1->aa) == sizeof(short))
+                r += sprintf(r, "short");
+            else if (sizeof(*p1->aa) == sizeof(signed char))
+                r += sprintf(r, "signed char");
+            else if (sizeof(*p1->aa) == sizeof(long))
+                r += sprintf(r, "long");
+            else if (sizeof(*p1->aa) == sizeof(long long))
+                r += sprintf(r, "long long");
+            else
+                r += sprintf(r, "int%u_t", (int)sizeof(*p1->aa) * 8);
+        }
+        r += sprintf(r, " *aa;\n");
     }
-    r("{aa:int?*", a, 3);
+    r += sprintf(r, "};\n");
+    return 0;
 }
-
-#expect  struct foo_s  8  8
-#expect  {aa:int?*  99  0  -4
diff --git a/test/test_cgcompile.py b/test/test_cgcompile.py
--- a/test/test_cgcompile.py
+++ b/test/test_cgcompile.py
@@ -14,15 +14,6 @@
     for line in inputlines:
         if line.startswith('#expect'):
             expected.append(line[8:].rstrip())
-    if not expected:
-        for line in inputlines:
-            if line.startswith('# _______'):
-                break
-            line = line.split('//')[0].rstrip()
-            if line:
-                expected.append(line)
-        else:
-            raise ValueError("no '# _______' found in %r" % (filename,))
     #
     basename = os.path.splitext(filename)[0]
     infile = str(udir.join('cg-' + filename))
@@ -30,34 +21,14 @@
     assert infile != outfile
     f = open(infile, 'w')
     f.write('#include \n')
-    f.write('#define TESTFN(r) test%s(r)\n' % (basename.replace('-','_'),))
+    f.write('#include "%s/../../creflect/creflect.h"\n' % path)
+    f.write('#define TESTFN(cb) test%s(cb)\n' % (basename.replace('-','_'),))
     f.write('\n')
     for line in inputlines:
         if not line.startswith('#') or line.startswith('#include'):
             f.write(line)
-    f.write(r'''
-#include 
-int main(void)
-{
-    char buffer[8192];
-    int i, r, r2;
-    r = TESTFN((char *)0);
-    assert(r > 0);
-    assert(r <= 8188);
-    for (i = 0; i <= r+3; i++)
-        buffer[r] = 0xfd;
-    r2 = TESTFN(buffer);
-    for (i = 0; i < r; i++) {
-        assert(buffer[i] != (char)0xfd);
-        if (buffer[i] == 0)
-            break;
-    }
-    assert(i < r);
-    assert(i == 0 || buffer[i-1] == '\n');
-    printf("%s", buffer);
-    return (r2 != 0);
-}
-''')
+    f.write('\n')
+    f.write('#include "%s/../cgcompile.c"\n' % path)
     f.close()
     #
     err = os.system("gcc -Werror -Wall '%s' -o '%s'" % (infile, outfile))
@@ -67,7 +38,12 @@
     lines = g.readlines()
     err = g.close()
     assert not err
-    r_remove_addr = re.compile(r"/[*][-0-9a-fx,]+[*]/")
+    if any('/*' in line for line in expected):
+        # only remove the comments that are addresses
+        r_remove_addr = re.compile(r"/[*]0x[-0-9a-f]+[*]/")
+    else:
+        # remove the comments that are addresses or sizes
+        r_remove_addr = re.compile(r"/[*][-0-9a-fx,]+[*]/")
     got = [r_remove_addr.sub('', line.rstrip()) for line in lines]
     compare_lists(got, expected)
 

From noreply at buildbot.pypy.org  Mon Nov 17 16:19:28 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 17 Nov 2014 16:19:28 +0100 (CET)
Subject: [pypy-commit] creflect default: Adapt more tests
Message-ID: <20141117151928.683401C1147@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r52:052baa9904a8
Date: 2014-11-17 16:19 +0100
http://bitbucket.org/cffi/creflect/changeset/052baa9904a8/

Log:	Adapt more tests

diff --git a/creflect/creflect.h b/creflect/creflect.h
--- a/creflect/creflect.h
+++ b/creflect/creflect.h
@@ -39,7 +39,7 @@
     void (*define_type)(CRX_SELF, const char *, crx_type_t *);
     void (*define_var)(CRX_SELF, const char *, crx_type_t *,
                        void *);
-    void (*error)(CRX_SELF, const char *, crx_type_t *);
+    void (*error)(CRX_SELF, const char *);
 } crx_builder_t;
 #undef CRX_SELF
 
diff --git a/test/cgcompile.c b/test/cgcompile.c
--- a/test/cgcompile.c
+++ b/test/cgcompile.c
@@ -155,9 +155,9 @@
     printf("VAR %s: %s\n", name, t->text);
 }
 
-static void tst_error(crx_builder_t *cb, const char *msg, crx_type_t *t)
+static void tst_error(crx_builder_t *cb, const char *msg)
 {
-    printf("ERROR %s %s\n", msg, t ? t->text : "");
+    printf("ERROR: %s\n", msg);
 }
 
 static crx_builder_t maincb = {
diff --git a/test/codegen/003.c b/test/codegen/003.c
--- a/test/codegen/003.c
+++ b/test/codegen/003.c
@@ -13,5 +13,6 @@
         *p1 = -1;  /* check that 'num_t' is not declared 'const' */
         t1 = CRX_INT_TYPE(cb, *p1, "int");
         cb->define_type(cb, "num_t", t1);
+#expect TYPEDEF num_t = int
     }
 }
diff --git a/test/codegen/003b.c b/test/codegen/003b.c
--- a/test/codegen/003b.c
+++ b/test/codegen/003b.c
@@ -2,15 +2,17 @@
 
 # ____________________________________________________________
 
-void test003b(creflect_builder_t *cb)
+void test003b(crx_builder_t *cb)
 {
+    crx_type_t *t1;
     {
         num_t *p1;
         char b[sizeof(*p1)];
         p1 = (void *)b;
         (void)(*p1 << 1);  /* check that 'num_t' is an integer type */
         *p1 = -1;  /* check that 'num_t' is not declared 'const' */
-        void *t = CB__INT_TYPE(*p1, "long");
-        cb->define_type("num_t", t);
+        t1 = CRX_INT_TYPE(cb, *p1, "long");
+        cb->define_type(cb, "num_t", t1);
+#expect TYPEDEF num_t = long
     }
 }
diff --git a/test/codegen/003c.c b/test/codegen/003c.c
--- a/test/codegen/003c.c
+++ b/test/codegen/003c.c
@@ -2,15 +2,17 @@
 
 # ____________________________________________________________
 
-void test003c(creflect_builder_t *cb)
+void test003c(crx_builder_t *cb)
 {
+    crx_type_t *t1;
     {
         num_t *p1;
         char b[sizeof(*p1)];
         p1 = (void *)b;
         (void)(*p1 << 1);  /* check that 'num_t' is an integer type */
         *p1 = -1;  /* check that 'num_t' is not declared 'const' */
-        void *t = CB__INT_TYPE(*p1, "long long");
-        cb->define_type("num_t", t);
+        t1 = CRX_INT_TYPE(cb, *p1, "long long");
+        cb->define_type(cb, "num_t", t1);
+#expect TYPEDEF num_t = long long
     }
 }
diff --git a/test/codegen/003d.c b/test/codegen/003d.c
--- a/test/codegen/003d.c
+++ b/test/codegen/003d.c
@@ -2,15 +2,17 @@
 
 # ____________________________________________________________
 
-void test003d(creflect_builder_t *cb)
+void test003d(crx_builder_t *cb)
 {
+    crx_type_t *t1;
     {
         num_t *p1;
         char b[sizeof(*p1)];
         p1 = (void *)b;
         (void)(*p1 << 1);  /* check that 'num_t' is an integer type */
         *p1 = -1;  /* check that 'num_t' is not declared 'const' */
-        void *t = CB__INT_TYPE(*p1, "char");
-        cb->define_type(cb, "num_t", t);
+        t1 = CRX_INT_TYPE(cb, *p1, "signed char");
+        cb->define_type(cb, "num_t", t1);
+#expect TYPEDEF num_t = signed char
     }
 }
diff --git a/test/codegen/003e.c b/test/codegen/003e.c
--- a/test/codegen/003e.c
+++ b/test/codegen/003e.c
@@ -2,7 +2,7 @@
 
 # ____________________________________________________________
 
-void test003e(crx_self_t *self, crx_builder_t *cb)
+void test003e(crx_builder_t *cb)
 {
     crx_type_t *t1;
     {
@@ -11,9 +11,8 @@
         p1 = (void *)b;
         (void)(*p1 << 1);  /* check that 'num_t' is an integer type */
         *p1 = -1;  /* check that 'num_t' is not declared 'const' */
-        char b2[sizeof(*p1) == 1 ? 1 : -1];  /* check that 'num_t' is a single-byte integer */
-        (void)b2;
-        t1 = cb->get_char_type(self);
-        cb->define_type(self, "num_t", t1);
+        t1 = cb->get_char_type(cb);
+        cb->define_type(cb, "num_t", t1);
+#expect TYPEDEF num_t = char
     }
 }
diff --git a/test/codegen/003f.c b/test/codegen/003f.c
--- a/test/codegen/003f.c
+++ b/test/codegen/003f.c
@@ -2,15 +2,17 @@
 
 # ____________________________________________________________
 
-void test003f(creflect_builder_t *cb)
+void test003f(crx_builder_t *cb)
 {
+    crx_type_t *t1;
     {
         num_t *p1;
         char b[sizeof(*p1)];
         p1 = (void *)b;
         (void)(*p1 << 1);  /* check that 'num_t' is an integer type */
         *p1 = -1;  /* check that 'num_t' is not declared 'const' */
-        void *t = CB__INT_TYPE(cb, *p1, "unsigned long long");
-        cb->define_type(cb, "num_t", t);
+        t1 = CRX_INT_TYPE(cb, *p1, "unsigned long long");
+        cb->define_type(cb, "num_t", t1);
+#expect TYPEDEF num_t = unsigned long long
     }
 }
diff --git a/test/codegen/004.c b/test/codegen/004.c
--- a/test/codegen/004.c
+++ b/test/codegen/004.c
@@ -2,8 +2,9 @@
 
 # ____________________________________________________________
 
-void test004(creflect_builder_t *cb)
+void test004(crx_builder_t *cb)
 {
+    crx_type_t *t1, *t2;
     {
         num_t *p1;
         char *p2;
@@ -12,48 +13,9 @@
         *p1 = (void *)b;    /* check that 'num_t' is a pointer type */
         (void)(**p1 << 1);  /* check that '*num_t' is an integer type */
         **p1 = -1;  /* check that '*num_t' is not declared 'const' */
-        void *t;
-        if (*p1 > 0) {
-            if (sizeof(*p1) == 1 && *p1 == 1)
-                t = cb->get_bool_type();
-            else
-                t = cb->get_signed_type(sizeof(*p1), "int");
-        }
-        else {
-            t = cb->get_unsigned_type(sizeof(*p1), "int");
-        }
-        
-        if (**p1 > 0) {
-            if (sizeof(**p1) == 1 && **p1 == 1)
-                r += sprintf(r, "_Bool");
-            else if (sizeof(**p1) == sizeof(unsigned int))
-                r += sprintf(r, "unsigned int");
-            else if (sizeof(**p1) == sizeof(unsigned short))
-                r += sprintf(r, "unsigned short");
-            else if (sizeof(**p1) == sizeof(unsigned char))
-                r += sprintf(r, "unsigned char");
-            else if (sizeof(**p1) == sizeof(unsigned long))
-                r += sprintf(r, "unsigned long");
-            else if (sizeof(**p1) == sizeof(unsigned long long))
-                r += sprintf(r, "unsigned long long");
-            else
-                r += sprintf(r, "uint%u_t", (int)sizeof(**p1) * 8);
-        }
-        else {
-            if (sizeof(**p1) == sizeof(int))
-                r += sprintf(r, "int");
-            else if (sizeof(**p1) == sizeof(short))
-                r += sprintf(r, "short");
-            else if (sizeof(**p1) == sizeof(signed char))
-                r += sprintf(r, "signed char");
-            else if (sizeof(**p1) == sizeof(long))
-                r += sprintf(r, "long");
-            else if (sizeof(**p1) == sizeof(long long))
-                r += sprintf(r, "long long");
-            else
-                r += sprintf(r, "int%u_t", (int)sizeof(**p1) * 8);
-        }
-        r += sprintf(r, " *num_t;\n");
+        t1 = CRX_INT_TYPE(cb, **p1, "int");
+        t2 = cb->get_pointer_type(cb, t1);
+        cb->define_type(cb, "num_t", t2);
+#expect TYPEDEF num_t = PTR int
     }
-    return 0;
 }
diff --git a/test/codegen/004b.c b/test/codegen/004b.c
--- a/test/codegen/004b.c
+++ b/test/codegen/004b.c
@@ -2,10 +2,9 @@
 
 # ____________________________________________________________
 
-int test004b(char *r)
+void test004b(crx_builder_t *cb)
 {
-    if (!r)
-        return 8 + 18 + 11;
+    crx_type_t *t1, *t2, *t3;
     {
         num_t *p1;
         char *p2;
@@ -15,39 +14,11 @@
         *p1 = (void *)&p3;  /* check that 'num_t' is a pointer type */
         **p1 = (void *)b;    /* check that '*num_t' is a pointer type */
         (void)(***p1 << 1);  /* check that '**num_t' is an integer type */
-        r += sprintf(r, "typedef ");
         ***p1 = -1;  /* check that '**num_t' is not declared 'const' */
-        if (***p1 > 0) {
-            if (sizeof(***p1) == 1 && ***p1 == 1)
-                r += sprintf(r, "_Bool");
-            else if (sizeof(***p1) == sizeof(unsigned int))
-                r += sprintf(r, "unsigned int");
-            else if (sizeof(***p1) == sizeof(unsigned short))
-                r += sprintf(r, "unsigned short");
-            else if (sizeof(***p1) == sizeof(unsigned char))
-                r += sprintf(r, "unsigned char");
-            else if (sizeof(***p1) == sizeof(unsigned long))
-                r += sprintf(r, "unsigned long");
-            else if (sizeof(***p1) == sizeof(unsigned long long))
-                r += sprintf(r, "unsigned long long");
-            else
-                r += sprintf(r, "uint%u_t", (int)sizeof(***p1) * 8);
-        }
-        else {
-            if (sizeof(***p1) == sizeof(int))
-                r += sprintf(r, "int");
-            else if (sizeof(***p1) == sizeof(short))
-                r += sprintf(r, "short");
-            else if (sizeof(***p1) == sizeof(signed char))
-                r += sprintf(r, "signed char");
-            else if (sizeof(***p1) == sizeof(long))
-                r += sprintf(r, "long");
-            else if (sizeof(***p1) == sizeof(long long))
-                r += sprintf(r, "long long");
-            else
-                r += sprintf(r, "int%u_t", (int)sizeof(***p1) * 8);
-        }
-        r += sprintf(r, " **num_t;\n");
+        t1 = CRX_INT_TYPE(cb, ***p1, "int");
+        t2 = cb->get_pointer_type(cb, t1);
+        t3 = cb->get_pointer_type(cb, t2);
+        cb->define_type(cb, "num_t", t3);
+#expect TYPEDEF num_t = PTR PTR int
     }
-    return 0;
 }
diff --git a/test/codegen/005.c b/test/codegen/005.c
--- a/test/codegen/005.c
+++ b/test/codegen/005.c
@@ -2,57 +2,22 @@
 
 # ____________________________________________________________
 
-int test005(char *r)
+void test005(crx_builder_t *cb)
 {
-    int r1 = 0;
-    if (!r)
-        return 57 + 8 + 18 + 35;
+    crx_type_t *t1, *t2;
     {
         foo_t *p1;
         char b[sizeof(**p1)];  /* check that 'foo_t[]' is a valid type */
-        char *r0 = r;
         p1 = (void *)b;
         if ((void *)p1 != (void *)*p1) {
-            r = r0;
-            r += sprintf(r, "#error type 'foo_t' is not an array, but a pointer type\n");
-            r1 = -1;
-            goto f2;
+            cb->error(cb, "type 'foo_t' is not an array, but a pointer type");
+            return;
         }
         (void)(**p1 << 1);  /* check that 'foo_t[]' is an integer type */
-        r += sprintf(r, "typedef ");
         **p1 = -1;  /* check that 'foo_t[]' is not declared 'const' */
-        if (**p1 > 0) {
-            if (sizeof(**p1) == 1 && **p1 == 1)
-                r += sprintf(r, "_Bool");
-            else if (sizeof(**p1) == sizeof(unsigned int))
-                r += sprintf(r, "unsigned int");
-            else if (sizeof(**p1) == sizeof(unsigned short))
-                r += sprintf(r, "unsigned short");
-            else if (sizeof(**p1) == sizeof(unsigned char))
-                r += sprintf(r, "unsigned char");
-            else if (sizeof(**p1) == sizeof(unsigned long))
-                r += sprintf(r, "unsigned long");
-            else if (sizeof(**p1) == sizeof(unsigned long long))
-                r += sprintf(r, "unsigned long long");
-            else
-                r += sprintf(r, "uint%u_t", (int)sizeof(**p1) * 8);
-        }
-        else {
-            if (sizeof(**p1) == sizeof(int))
-                r += sprintf(r, "int");
-            else if (sizeof(**p1) == sizeof(short))
-                r += sprintf(r, "short");
-            else if (sizeof(**p1) == sizeof(signed char))
-                r += sprintf(r, "signed char");
-            else if (sizeof(**p1) == sizeof(long))
-                r += sprintf(r, "long");
-            else if (sizeof(**p1) == sizeof(long long))
-                r += sprintf(r, "long long");
-            else
-                r += sprintf(r, "int%u_t", (int)sizeof(**p1) * 8);
-        }
-        r += sprintf(r, " foo_t[%lld];\n", (long long)(sizeof(*p1) / sizeof(**p1)));
-        f2:;
+        t1 = CRX_INT_TYPE(cb, **p1, "int");
+        t2 = cb->get_array_type(cb, t1, sizeof(*p1) / sizeof(**p1));
+        cb->define_type(cb, "foo_t", t2);
+#expect TYPEDEF foo_t = ARRAY[27] int
     }
-    return r1;
 }
diff --git a/test/codegen/005b.c b/test/codegen/005b.c
--- a/test/codegen/005b.c
+++ b/test/codegen/005b.c
@@ -2,59 +2,25 @@
 
 # ____________________________________________________________
 
-int test005b(char *r)
+void test005b(crx_builder_t *cb)
 {
-    int r1 = 0;
-    if (!r)
-        return 57 + 8 + 18 + 36;
+    crx_type_t *t1, *t2, *t3;
     {
         foo_t *p1;
         char *p3;
-        char *r0 = r;
         char b[sizeof(***p1)];  /* check that '*foo_t[]' is a valid type */
         p1 = (void *)&p3;
         if ((void *)p1 != (void *)*p1) {
-            r = r0;
-            r += sprintf(r, "#error type 'foo_t' is not an array, but a pointer type\n");
-            r1 = -1;
-            goto f2;
+            cb->error(cb, "type 'foo_t' is not an array, but a pointer type");
+            return;
         }
         **p1 = (void *)b;    /* check that 'foo_t[]' is a pointer type */
         (void)(***p1 << 1);  /* check that '*foo_t[]' is an integer type */
-        r += sprintf(r, "typedef ");
         ***p1 = -1;  /* check that '*foo_t[]' is not declared 'const' */
-        if (***p1 > 0) {
-            if (sizeof(***p1) == 1 && ***p1 == 1)
-                r += sprintf(r, "_Bool");
-            else if (sizeof(***p1) == sizeof(unsigned int))
-                r += sprintf(r, "unsigned int");
-            else if (sizeof(***p1) == sizeof(unsigned short))
-                r += sprintf(r, "unsigned short");
-            else if (sizeof(***p1) == sizeof(unsigned char))
-                r += sprintf(r, "unsigned char");
-            else if (sizeof(***p1) == sizeof(unsigned long))
-                r += sprintf(r, "unsigned long");
-            else if (sizeof(***p1) == sizeof(unsigned long long))
-                r += sprintf(r, "unsigned long long");
-            else
-                r += sprintf(r, "uint%u_t", (int)sizeof(***p1) * 8);
-        }
-        else {
-            if (sizeof(***p1) == sizeof(int))
-                r += sprintf(r, "int");
-            else if (sizeof(***p1) == sizeof(short))
-                r += sprintf(r, "short");
-            else if (sizeof(***p1) == sizeof(signed char))
-                r += sprintf(r, "signed char");
-            else if (sizeof(***p1) == sizeof(long))
-                r += sprintf(r, "long");
-            else if (sizeof(***p1) == sizeof(long long))
-                r += sprintf(r, "long long");
-            else
-                r += sprintf(r, "int%u_t", (int)sizeof(***p1) * 8);
-        }
-        r += sprintf(r, " *foo_t[%lld];\n", (long long)(sizeof(*p1) / sizeof(**p1)));
-        f2:;
+        t1 = CRX_INT_TYPE(cb, ***p1, "int");
+        t2 = cb->get_pointer_type(cb, t1);
+        t3 = cb->get_array_type(cb, t2, sizeof(*p1) / sizeof(**p1));
+        cb->define_type(cb, "foo_t", t3);
+#expect TYPEDEF foo_t = ARRAY[27] PTR int
     }
-    return r1;
 }

From noreply at buildbot.pypy.org  Mon Nov 17 18:58:23 2014
From: noreply at buildbot.pypy.org (pjenvey)
Date: Mon, 17 Nov 2014 18:58:23 +0100 (CET)
Subject: [pypy-commit] pypy default: py3k compat
Message-ID: <20141117175823.DA63F1C104D@cobra.cs.uni-duesseldorf.de>

Author: Philip Jenvey 
Branch: 
Changeset: r74548:143e371730db
Date: 2014-11-17 09:39 -0800
http://bitbucket.org/pypy/pypy/changeset/143e371730db/

Log:	py3k compat

diff --git a/pypy/module/micronumpy/nditer.py b/pypy/module/micronumpy/nditer.py
--- a/pypy/module/micronumpy/nditer.py
+++ b/pypy/module/micronumpy/nditer.py
@@ -89,10 +89,9 @@
     for w_item in lst:
         if not space.isinstance_w(w_item, space.w_str) and not \
                 space.isinstance_w(w_item, space.w_unicode):
-            typename = space.type(w_item).getname(space)
             raise oefmt(space.w_TypeError,
-                        'expected string or Unicode object, %s found',
-                        typename)
+                        "expected string or Unicode object, %T found",
+                        w_item)
         item = space.str_w(w_item)
         if item == 'external_loop':
             nditer.external_loop = True

From noreply at buildbot.pypy.org  Mon Nov 17 19:06:54 2014
From: noreply at buildbot.pypy.org (pjenvey)
Date: Mon, 17 Nov 2014 19:06:54 +0100 (CET)
Subject: [pypy-commit] pypy py3k: merge default
Message-ID: <20141117180654.4340F1C104D@cobra.cs.uni-duesseldorf.de>

Author: Philip Jenvey 
Branch: py3k
Changeset: r74549:9a1ca4c16f23
Date: 2014-11-17 10:05 -0800
http://bitbucket.org/pypy/pypy/changeset/9a1ca4c16f23/

Log:	merge default

diff too long, truncating to 2000 out of 7347 lines

diff --git a/lib-python/conftest.py b/lib-python/conftest.py
--- a/lib-python/conftest.py
+++ b/lib-python/conftest.py
@@ -60,7 +60,7 @@
                  skip=None):
         self.basename = basename
         self._usemodules = usemodules.split() + [
-            '_socket', 'binascii', 'rctime', 'select', 'signal']
+            '_socket', 'binascii', 'time', 'select', 'signal']
         if not sys.platform == 'win32':
             self._usemodules.extend(['_posixsubprocess', 'fcntl'])
         self._compiler = compiler
diff --git a/pypy/config/pypyoption.py b/pypy/config/pypyoption.py
--- a/pypy/config/pypyoption.py
+++ b/pypy/config/pypyoption.py
@@ -30,7 +30,7 @@
 # --allworkingmodules
 working_modules = default_modules.copy()
 working_modules.update([
-    "_socket", "unicodedata", "mmap", "fcntl", "_locale", "pwd", "rctime" ,
+    "_socket", "unicodedata", "mmap", "fcntl", "_locale", "pwd", "time" ,
     "select", "zipimport", "_lsprof", "crypt", "signal", "_rawffi", "termios",
     "zlib", "bz2", "struct", "_hashlib", "_md5", "_minimal_curses",
     "thread", "itertools", "pyexpat", "_ssl", "cpyext", "array",
@@ -41,7 +41,7 @@
 
 translation_modules = default_modules.copy()
 translation_modules.update([
-    "fcntl", "rctime", "select", "signal", "_rawffi", "zlib", "struct",
+    "fcntl", "time", "select", "signal", "_rawffi", "zlib", "struct",
     "array", "binascii",
     # the following are needed for pyrepl (and hence for the
     # interactive prompt/pdb)
@@ -66,19 +66,15 @@
     default_modules.add("_locale")
 
 if sys.platform == "sunos5":
-    working_modules.remove('mmap')   # depend on ctypes, can't get at c-level 'errono'
-    working_modules.remove('rctime') # depend on ctypes, missing tm_zone/tm_gmtoff
-    working_modules.remove('signal') # depend on ctypes, can't get at c-level 'errono'
     working_modules.remove('fcntl')  # LOCK_NB not defined
     working_modules.remove("_minimal_curses")
     working_modules.remove("termios")
-    working_modules.remove("_multiprocessing")   # depends on rctime
     if "cppyy" in working_modules:
         working_modules.remove("cppyy")  # depends on ctypes
 
 
 module_dependencies = {
-    '_multiprocessing': [('objspace.usemodules.rctime', True),
+    '_multiprocessing': [('objspace.usemodules.time', True),
                          ('objspace.usemodules.thread', True)],
     'cpyext': [('objspace.usemodules.array', True)],
     'cppyy': [('objspace.usemodules.cpyext', True)],
diff --git a/pypy/doc/config/objspace.usemodules.rctime.txt b/pypy/doc/config/objspace.usemodules.rctime.txt
deleted file mode 100644
--- a/pypy/doc/config/objspace.usemodules.rctime.txt
+++ /dev/null
@@ -1,7 +0,0 @@
-Use the 'rctime' module. 
-
-'rctime' is our `rffi`_ based implementation of the builtin 'time' module.
-It supersedes the less complete :config:`objspace.usemodules.time`,
-at least for C-like targets (the C and LLVM backends).
-
-.. _`rffi`: ../rffi.html
diff --git a/pypy/doc/config/objspace.usemodules.time.txt b/pypy/doc/config/objspace.usemodules.time.txt
--- a/pypy/doc/config/objspace.usemodules.time.txt
+++ b/pypy/doc/config/objspace.usemodules.time.txt
@@ -1,5 +1,1 @@
 Use the 'time' module. 
-
-Obsolete; use :config:`objspace.usemodules.rctime` for our up-to-date version
-of the application-level 'time' module, at least for C-like targets (the C
-and LLVM backends).
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -43,3 +43,11 @@
 .. branch nditer-external_loop
 
 Implement `external_loop` arguement to numpy's nditer
+
+.. branch kill-rctime
+
+Rename pypy/module/rctime to pypy/module/time, since it contains the implementation of the 'time' module.
+
+.. branch: ssa-flow
+
+Use SSA form for flow graphs inside build_flow() and part of simplify_graph()
diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -500,11 +500,6 @@
                 if name not in modules:
                     modules.append(name)
 
-        # a bit of custom logic: rctime take precedence over time
-        # XXX this could probably be done as a "requires" in the config
-        if 'rctime' in modules and 'time' in modules:
-            modules.remove('time')
-
         self._builtinmodule_list = modules
         return self._builtinmodule_list
 
diff --git a/pypy/module/_lsprof/test/test_cprofile.py b/pypy/module/_lsprof/test/test_cprofile.py
--- a/pypy/module/_lsprof/test/test_cprofile.py
+++ b/pypy/module/_lsprof/test/test_cprofile.py
@@ -1,6 +1,6 @@
 class AppTestCProfile(object):
     spaceconfig = {
-        "usemodules": ['_lsprof', 'rctime'],
+        "usemodules": ['_lsprof', 'time'],
     }
 
     def setup_class(cls):
diff --git a/pypy/module/_md5/test/test_md5.py b/pypy/module/_md5/test/test_md5.py
--- a/pypy/module/_md5/test/test_md5.py
+++ b/pypy/module/_md5/test/test_md5.py
@@ -7,7 +7,7 @@
 
 class AppTestMD5(object):
     spaceconfig = {
-        'usemodules': ['_md5', 'binascii', 'rctime', 'struct'],
+        'usemodules': ['_md5', 'binascii', 'time', 'struct'],
     }
 
     def setup_class(cls):
diff --git a/pypy/module/_multibytecodec/src/cjkcodecs/multibytecodec.h b/pypy/module/_multibytecodec/src/cjkcodecs/multibytecodec.h
--- a/pypy/module/_multibytecodec/src/cjkcodecs/multibytecodec.h
+++ b/pypy/module/_multibytecodec/src/cjkcodecs/multibytecodec.h
@@ -97,24 +97,24 @@
   Py_UNICODE *outbuf_start, *outbuf, *outbuf_end;
 };
 
-RPY_EXPORTED_FOR_TESTS
+RPY_EXTERN
 struct pypy_cjk_dec_s *pypy_cjk_dec_new(const MultibyteCodec *codec);
-RPY_EXPORTED_FOR_TESTS
+RPY_EXTERN
 Py_ssize_t pypy_cjk_dec_init(struct pypy_cjk_dec_s *d,
                              char *inbuf, Py_ssize_t inlen);
-RPY_EXPORTED_FOR_TESTS
+RPY_EXTERN
 void pypy_cjk_dec_free(struct pypy_cjk_dec_s *);
-RPY_EXPORTED_FOR_TESTS
+RPY_EXTERN
 Py_ssize_t pypy_cjk_dec_chunk(struct pypy_cjk_dec_s *);
-RPY_EXPORTED_FOR_TESTS
+RPY_EXTERN
 Py_UNICODE *pypy_cjk_dec_outbuf(struct pypy_cjk_dec_s *);
-RPY_EXPORTED_FOR_TESTS
+RPY_EXTERN
 Py_ssize_t pypy_cjk_dec_outlen(struct pypy_cjk_dec_s *);
-RPY_EXPORTED_FOR_TESTS
+RPY_EXTERN
 Py_ssize_t pypy_cjk_dec_inbuf_remaining(struct pypy_cjk_dec_s *d);
-RPY_EXPORTED_FOR_TESTS
+RPY_EXTERN
 Py_ssize_t pypy_cjk_dec_inbuf_consumed(struct pypy_cjk_dec_s* d);
-RPY_EXPORTED_FOR_TESTS
+RPY_EXTERN
 Py_ssize_t pypy_cjk_dec_replace_on_error(struct pypy_cjk_dec_s* d,
                                          Py_UNICODE *, Py_ssize_t, Py_ssize_t);
 
@@ -125,35 +125,35 @@
   unsigned char *outbuf_start, *outbuf, *outbuf_end;
 };
 
-RPY_EXPORTED_FOR_TESTS
+RPY_EXTERN
 struct pypy_cjk_enc_s *pypy_cjk_enc_new(const MultibyteCodec *codec);
-RPY_EXPORTED_FOR_TESTS
+RPY_EXTERN
 Py_ssize_t pypy_cjk_enc_init(struct pypy_cjk_enc_s *d,
                              Py_UNICODE *inbuf, Py_ssize_t inlen);
-RPY_EXPORTED_FOR_TESTS
+RPY_EXTERN
 void pypy_cjk_enc_free(struct pypy_cjk_enc_s *);
-RPY_EXPORTED_FOR_TESTS
+RPY_EXTERN
 Py_ssize_t pypy_cjk_enc_chunk(struct pypy_cjk_enc_s *, Py_ssize_t);
-RPY_EXPORTED_FOR_TESTS
+RPY_EXTERN
 Py_ssize_t pypy_cjk_enc_reset(struct pypy_cjk_enc_s *);
-RPY_EXPORTED_FOR_TESTS
+RPY_EXTERN
 char *pypy_cjk_enc_outbuf(struct pypy_cjk_enc_s *);
-RPY_EXPORTED_FOR_TESTS
+RPY_EXTERN
 Py_ssize_t pypy_cjk_enc_outlen(struct pypy_cjk_enc_s *);
-RPY_EXPORTED_FOR_TESTS
+RPY_EXTERN
 Py_ssize_t pypy_cjk_enc_inbuf_remaining(struct pypy_cjk_enc_s *d);
-RPY_EXPORTED_FOR_TESTS
+RPY_EXTERN
 Py_ssize_t pypy_cjk_enc_inbuf_consumed(struct pypy_cjk_enc_s* d);
-RPY_EXPORTED_FOR_TESTS
+RPY_EXTERN
 Py_ssize_t pypy_cjk_enc_replace_on_error(struct pypy_cjk_enc_s* d,
                                          char *, Py_ssize_t, Py_ssize_t);
-RPY_EXPORTED_FOR_TESTS
+RPY_EXTERN
 const MultibyteCodec *pypy_cjk_enc_getcodec(struct pypy_cjk_enc_s *);
 
 /* list of codecs defined in the .c files */
 
 #define DEFINE_CODEC(name)                              \
-    RPY_EXPORTED_FOR_TESTS MultibyteCodec *pypy_cjkcodec_##name(void);
+    RPY_EXTERN MultibyteCodec *pypy_cjkcodec_##name(void);
 
 // _codecs_cn
 DEFINE_CODEC(gb2312)
diff --git a/pypy/module/_multiprocessing/interp_semaphore.py b/pypy/module/_multiprocessing/interp_semaphore.py
--- a/pypy/module/_multiprocessing/interp_semaphore.py
+++ b/pypy/module/_multiprocessing/interp_semaphore.py
@@ -254,7 +254,7 @@
         start = _GetTickCount()
 
         while True:
-            from pypy.module.rctime.interp_time import State
+            from pypy.module.time.interp_time import State
             interrupt_event = space.fromcache(State).get_interrupt_event()
             handles = [self.handle, interrupt_event]
 
diff --git a/pypy/module/_random/test/test_random.py b/pypy/module/_random/test/test_random.py
--- a/pypy/module/_random/test/test_random.py
+++ b/pypy/module/_random/test/test_random.py
@@ -1,6 +1,6 @@
 class AppTestRandom:
     spaceconfig = {
-        "usemodules": ['_random', 'rctime'],
+        "usemodules": ['_random', 'time'],
     }
 
     def test_dict(self):
diff --git a/pypy/module/_ssl/thread_lock.py b/pypy/module/_ssl/thread_lock.py
--- a/pypy/module/_ssl/thread_lock.py
+++ b/pypy/module/_ssl/thread_lock.py
@@ -65,7 +65,7 @@
 eci = rthread.eci.merge(ExternalCompilationInfo(
     separate_module_sources=[separate_module_source],
     post_include_bits=[
-        "RPY_EXPORTED_FOR_TESTS int _PyPy_SSL_SetupThreads(void);"],
+        "RPY_EXTERN int _PyPy_SSL_SetupThreads(void);"],
     libraries = libraries,
 ))
 
diff --git a/pypy/module/bz2/test/test_bz2_file.py b/pypy/module/bz2/test/test_bz2_file.py
--- a/pypy/module/bz2/test/test_bz2_file.py
+++ b/pypy/module/bz2/test/test_bz2_file.py
@@ -54,7 +54,7 @@
     # XXX: CheckAllocation fails on py3 (seems to false positive on
     # BZ2File's RLocks)
     spaceconfig = {
-        'usemodules': ['bz2', 'binascii', 'rctime', 'struct', 'thread']
+        'usemodules': ['bz2', 'binascii', 'time', 'struct', 'thread']
     }
 
     def setup_class(cls):
diff --git a/pypy/module/cppyy/include/capi.h b/pypy/module/cppyy/include/capi.h
--- a/pypy/module/cppyy/include/capi.h
+++ b/pypy/module/cppyy/include/capi.h
@@ -2,6 +2,7 @@
 #define CPPYY_CAPI
 
 #include 
+#include "src/precommondefs.h"
 
 #ifdef __cplusplus
 extern "C" {
@@ -15,102 +16,167 @@
     typedef void* (*cppyy_methptrgetter_t)(cppyy_object_t);
 
     /* name to opaque C++ scope representation -------------------------------- */
+    RPY_EXTERN
     int cppyy_num_scopes(cppyy_scope_t parent);
+    RPY_EXTERN
     char* cppyy_scope_name(cppyy_scope_t parent, int iscope);
 
+    RPY_EXTERN
     char* cppyy_resolve_name(const char* cppitem_name);
+    RPY_EXTERN
     cppyy_scope_t cppyy_get_scope(const char* scope_name);
+    RPY_EXTERN
     cppyy_type_t cppyy_get_template(const char* template_name);
+    RPY_EXTERN
     cppyy_type_t cppyy_actual_class(cppyy_type_t klass, cppyy_object_t obj);
 
     /* memory management ------------------------------------------------------ */
+    RPY_EXTERN
     cppyy_object_t cppyy_allocate(cppyy_type_t type);
+    RPY_EXTERN
     void cppyy_deallocate(cppyy_type_t type, cppyy_object_t self);
+    RPY_EXTERN
     void cppyy_destruct(cppyy_type_t type, cppyy_object_t self);
 
     /* method/function dispatching -------------------------------------------- */
+    RPY_EXTERN
     void   cppyy_call_v(cppyy_method_t method, cppyy_object_t self, int nargs, void* args);
+    RPY_EXTERN
     unsigned char cppyy_call_b(cppyy_method_t method, cppyy_object_t self, int nargs, void* args);
+    RPY_EXTERN
     char   cppyy_call_c(cppyy_method_t method, cppyy_object_t self, int nargs, void* args);
+    RPY_EXTERN
     short  cppyy_call_h(cppyy_method_t method, cppyy_object_t self, int nargs, void* args);
+    RPY_EXTERN
     int    cppyy_call_i(cppyy_method_t method, cppyy_object_t self, int nargs, void* args);
+    RPY_EXTERN
     long   cppyy_call_l(cppyy_method_t method, cppyy_object_t self, int nargs, void* args);
+    RPY_EXTERN
     long long cppyy_call_ll(cppyy_method_t method, cppyy_object_t self, int nargs, void* args);
+    RPY_EXTERN
     float  cppyy_call_f(cppyy_method_t method, cppyy_object_t self, int nargs, void* args);
+    RPY_EXTERN
     double cppyy_call_d(cppyy_method_t method, cppyy_object_t self, int nargs, void* args);
 
+    RPY_EXTERN
     void*  cppyy_call_r(cppyy_method_t method, cppyy_object_t self, int nargs, void* args);
+    RPY_EXTERN
     char*  cppyy_call_s(cppyy_method_t method, cppyy_object_t self, int nargs, void* args);
 
+    RPY_EXTERN
     cppyy_object_t cppyy_constructor(cppyy_method_t method, cppyy_type_t klass, int nargs, void* args);
+    RPY_EXTERN
     cppyy_object_t cppyy_call_o(cppyy_method_t method, cppyy_object_t self, int nargs, void* args, cppyy_type_t result_type);
 
+    RPY_EXTERN
     cppyy_methptrgetter_t cppyy_get_methptr_getter(cppyy_scope_t scope, cppyy_index_t idx);
 
     /* handling of function argument buffer ----------------------------------- */
+    RPY_EXTERN
     void*  cppyy_allocate_function_args(int nargs);
+    RPY_EXTERN
     void   cppyy_deallocate_function_args(void* args);
+    RPY_EXTERN
     size_t cppyy_function_arg_sizeof();
+    RPY_EXTERN
     size_t cppyy_function_arg_typeoffset();
 
     /* scope reflection information ------------------------------------------- */
+    RPY_EXTERN
     int cppyy_is_namespace(cppyy_scope_t scope);
+    RPY_EXTERN
     int cppyy_is_enum(const char* type_name);
 
     /* class reflection information ------------------------------------------- */
+    RPY_EXTERN
     char* cppyy_final_name(cppyy_type_t type);
+    RPY_EXTERN
     char* cppyy_scoped_final_name(cppyy_type_t type);
+    RPY_EXTERN
     int cppyy_has_complex_hierarchy(cppyy_type_t type);
+    RPY_EXTERN
     int cppyy_num_bases(cppyy_type_t type);
+    RPY_EXTERN
     char* cppyy_base_name(cppyy_type_t type, int base_index);
+    RPY_EXTERN
     int cppyy_is_subtype(cppyy_type_t derived, cppyy_type_t base);
 
     /* calculate offsets between declared and actual type, up-cast: direction > 0; down-cast: direction < 0 */
+    RPY_EXTERN
     ptrdiff_t cppyy_base_offset(cppyy_type_t derived, cppyy_type_t base, cppyy_object_t address, int direction);
 
     /* method/function reflection information --------------------------------- */
+    RPY_EXTERN
     int cppyy_num_methods(cppyy_scope_t scope);
+    RPY_EXTERN
     cppyy_index_t cppyy_method_index_at(cppyy_scope_t scope, int imeth);
+    RPY_EXTERN
     cppyy_index_t* cppyy_method_indices_from_name(cppyy_scope_t scope, const char* name);
 
+    RPY_EXTERN
     char* cppyy_method_name(cppyy_scope_t scope, cppyy_index_t idx);
+    RPY_EXTERN
     char* cppyy_method_result_type(cppyy_scope_t scope, cppyy_index_t idx);
+    RPY_EXTERN
     int cppyy_method_num_args(cppyy_scope_t scope, cppyy_index_t idx);
+    RPY_EXTERN
     int cppyy_method_req_args(cppyy_scope_t scope, cppyy_index_t idx);
+    RPY_EXTERN
     char* cppyy_method_arg_type(cppyy_scope_t scope, cppyy_index_t idx, int arg_index);
+    RPY_EXTERN
     char* cppyy_method_arg_default(cppyy_scope_t scope, cppyy_index_t idx, int arg_index);
+    RPY_EXTERN
     char* cppyy_method_signature(cppyy_scope_t scope, cppyy_index_t idx);
 
+    RPY_EXTERN
     int cppyy_method_is_template(cppyy_scope_t scope, cppyy_index_t idx);
+    RPY_EXTERN
     int cppyy_method_num_template_args(cppyy_scope_t scope, cppyy_index_t idx);
+    RPY_EXTERN
     char* cppyy_method_template_arg_name(cppyy_scope_t scope, cppyy_index_t idx, cppyy_index_t iarg);
 
+    RPY_EXTERN
     cppyy_method_t cppyy_get_method(cppyy_scope_t scope, cppyy_index_t idx);
+    RPY_EXTERN
     cppyy_index_t cppyy_get_global_operator(
         cppyy_scope_t scope, cppyy_scope_t lc, cppyy_scope_t rc, const char* op);
 
     /* method properties ------------------------------------------------------ */
+    RPY_EXTERN
     int cppyy_is_constructor(cppyy_type_t type, cppyy_index_t idx);
+    RPY_EXTERN
     int cppyy_is_staticmethod(cppyy_type_t type, cppyy_index_t idx);
 
     /* data member reflection information ------------------------------------- */
+    RPY_EXTERN
     int cppyy_num_datamembers(cppyy_scope_t scope);
+    RPY_EXTERN
     char* cppyy_datamember_name(cppyy_scope_t scope, int datamember_index);
+    RPY_EXTERN
     char* cppyy_datamember_type(cppyy_scope_t scope, int datamember_index);
+    RPY_EXTERN
     ptrdiff_t cppyy_datamember_offset(cppyy_scope_t scope, int datamember_index);
 
+    RPY_EXTERN
     int cppyy_datamember_index(cppyy_scope_t scope, const char* name);
 
     /* data member properties ------------------------------------------------- */
+    RPY_EXTERN
     int cppyy_is_publicdata(cppyy_type_t type, int datamember_index);
+    RPY_EXTERN
     int cppyy_is_staticdata(cppyy_type_t type, int datamember_index);
 
     /* misc helpers ----------------------------------------------------------- */
+    RPY_EXTERN
     long long cppyy_strtoll(const char* str);
+    RPY_EXTERN
     unsigned long long cppyy_strtoull(const char* str);
+    RPY_EXTERN
     void cppyy_free(void* ptr);
 
+    RPY_EXTERN
     cppyy_object_t cppyy_charp2stdstring(const char* str);
+    RPY_EXTERN
     cppyy_object_t cppyy_stdstring2stdstring(cppyy_object_t ptr);
 
 #ifdef __cplusplus
diff --git a/pypy/module/cppyy/src/dummy_backend.cxx b/pypy/module/cppyy/src/dummy_backend.cxx
--- a/pypy/module/cppyy/src/dummy_backend.cxx
+++ b/pypy/module/cppyy/src/dummy_backend.cxx
@@ -1,4 +1,3 @@
-#include "src/precommondefs.h"
 #include "cppyy.h"
 #include "capi.h"
 
@@ -349,29 +348,24 @@
 
 
 /* name to opaque C++ scope representation -------------------------------- */
-RPY_EXPORTED_FOR_TESTS
 int cppyy_num_scopes(cppyy_scope_t handle) {
     return 0;
 }
 
-RPY_EXPORTED_FOR_TESTS
 char* cppyy_resolve_name(const char* cppitem_name) {
     return cppstring_to_cstring(cppitem_name);
 }
 
-RPY_EXPORTED_FOR_TESTS
 cppyy_scope_t cppyy_get_scope(const char* scope_name) {
     return s_handles[scope_name];  // lookup failure will return 0 (== error)
 }
 
-RPY_EXPORTED_FOR_TESTS
 cppyy_type_t cppyy_actual_class(cppyy_type_t klass, cppyy_object_t /* obj */) {
     return klass;
 }
 
 
 /* memory management ------------------------------------------------------ */
-RPY_EXPORTED_FOR_TESTS
 void cppyy_destruct(cppyy_type_t handle, cppyy_object_t self) {
     if (handle == s_handles["example01"])
        delete (dummy::example01*)self;
@@ -379,7 +373,6 @@
 
 
 /* method/function dispatching -------------------------------------------- */
-RPY_EXPORTED_FOR_TESTS
 void cppyy_call_v(cppyy_method_t method, cppyy_object_t self, int nargs, void* args) {
     long idx = (long)method;
     if (idx == s_methods["static_example01::staticSetPayload_payload*_double"]) {
@@ -469,7 +462,6 @@
     }
 }
 
-RPY_EXPORTED_FOR_TESTS
 unsigned char cppyy_call_b(cppyy_method_t method, cppyy_object_t self, int nargs, void* args) {
     unsigned char result = 0;
     const long idx = (long)method;
@@ -482,7 +474,6 @@
     return result;
 }
 
-RPY_EXPORTED_FOR_TESTS
 char cppyy_call_c(cppyy_method_t method, cppyy_object_t self, int nargs, void* args) {
     char result = 0;
     const long idx = (long)method;
@@ -498,7 +489,6 @@
     return result;
 }
 
-RPY_EXPORTED_FOR_TESTS
 short cppyy_call_h(cppyy_method_t method, cppyy_object_t self, int nargs, void* args) {
     short result = 0;
     const long idx = (long)method; 
@@ -514,7 +504,6 @@
     return result;
 }
 
-RPY_EXPORTED_FOR_TESTS
 int cppyy_call_i(cppyy_method_t method, cppyy_object_t self, int nargs, void* args) {
     int result = 0;
     const long idx = (long)method;
@@ -547,7 +536,6 @@
     return result;
 }
 
-RPY_EXPORTED_FOR_TESTS
 long cppyy_call_l(cppyy_method_t method, cppyy_object_t self, int nargs, void* args) {
     long result = 0;
     const long idx = (long)method;
@@ -689,7 +677,6 @@
     return result;
 }
 
-RPY_EXPORTED_FOR_TESTS
 long long cppyy_call_ll(cppyy_method_t method, cppyy_object_t self, int nargs, void* args) {
     long long result = 0;
     const long idx = (long)method;
@@ -705,7 +692,6 @@
     return result;
 }   
 
-RPY_EXPORTED_FOR_TESTS
 float cppyy_call_f(cppyy_method_t method, cppyy_object_t self, int nargs, void* args) {
     float result = 0;
     const long idx = (long)method;
@@ -718,7 +704,6 @@
     return result;
 }   
 
-RPY_EXPORTED_FOR_TESTS
 double cppyy_call_d(cppyy_method_t method, cppyy_object_t self, int nargs, void* args) {
     double result = 0.;
     const long idx = (long)method;
@@ -740,7 +725,6 @@
     return result;
 }
 
-RPY_EXPORTED_FOR_TESTS
 char* cppyy_call_s(cppyy_method_t method, cppyy_object_t self, int nargs, void* args) {
     char* result = 0;
     const long idx = (long)method;
@@ -753,7 +737,6 @@
     return result;
 }
 
-RPY_EXPORTED_FOR_TESTS
 cppyy_object_t cppyy_constructor(cppyy_method_t method, cppyy_type_t handle, int nargs, void* args) {
     void* result = 0;
     const long idx = (long)method;
@@ -776,14 +759,12 @@
     return (cppyy_object_t)result;
 }
 
-RPY_EXPORTED_FOR_TESTS
 cppyy_methptrgetter_t cppyy_get_methptr_getter(cppyy_type_t /* handle */, cppyy_index_t /* method_index */) {
     return (cppyy_methptrgetter_t)0;
 }
 
 
 /* handling of function argument buffer ----------------------------------- */
-RPY_EXPORTED_FOR_TESTS
 void* cppyy_allocate_function_args(int nargs) {
     CPPYY_G__value* args = (CPPYY_G__value*)malloc(nargs*sizeof(CPPYY_G__value));
     for (int i = 0; i < nargs; ++i)
@@ -793,36 +774,30 @@
 
 
 /* handling of function argument buffer ----------------------------------- */
-RPY_EXPORTED_FOR_TESTS
 void cppyy_deallocate_function_args(void* args) {
     free(args);
 }
 
-RPY_EXPORTED_FOR_TESTS
 size_t cppyy_function_arg_sizeof() {
     return sizeof(CPPYY_G__value);
 }
 
-RPY_EXPORTED_FOR_TESTS
 size_t cppyy_function_arg_typeoffset() {
     return offsetof(CPPYY_G__value, type);
 }
 
 
 /* scope reflection information ------------------------------------------- */
-RPY_EXPORTED_FOR_TESTS
 int cppyy_is_namespace(cppyy_scope_t /* handle */) {
     return 0;
 }   
 
-RPY_EXPORTED_FOR_TESTS
 int cppyy_is_enum(const char* /* type_name */) {
     return 0;
 }
     
     
 /* class reflection information ------------------------------------------- */
-RPY_EXPORTED_FOR_TESTS
 char* cppyy_final_name(cppyy_type_t handle) {
     for (Handles_t::iterator isp = s_handles.begin(); isp != s_handles.end(); ++isp) {
         if (isp->second == handle)
@@ -831,75 +806,61 @@
     return cppstring_to_cstring("");
 }
 
-RPY_EXPORTED_FOR_TESTS
 char* cppyy_scoped_final_name(cppyy_type_t handle) {
     return cppyy_final_name(handle);
 }   
 
-RPY_EXPORTED_FOR_TESTS
 int cppyy_has_complex_hierarchy(cppyy_type_t /* handle */) {
     return 0;
 }
 
-RPY_EXPORTED_FOR_TESTS
 int cppyy_num_bases(cppyy_type_t /*handle*/) {
    return 0;
 }
 
 
 /* method/function reflection information --------------------------------- */
-RPY_EXPORTED_FOR_TESTS
 int cppyy_num_methods(cppyy_scope_t handle) {
     return s_scopes[handle].m_methods.size();
 }
 
-RPY_EXPORTED_FOR_TESTS
 cppyy_index_t cppyy_method_index_at(cppyy_scope_t /* scope */, int imeth) {
     return (cppyy_index_t)imeth;
 }
 
-RPY_EXPORTED_FOR_TESTS
 char* cppyy_method_name(cppyy_scope_t handle, cppyy_index_t method_index) {
     return cppstring_to_cstring(s_scopes[handle].m_methods[(int)method_index].m_name);
 }
 
-RPY_EXPORTED_FOR_TESTS
 char* cppyy_method_result_type(cppyy_scope_t handle, cppyy_index_t method_index) {
     return cppstring_to_cstring(s_scopes[handle].m_methods[method_index].m_returntype);
 }
     
-RPY_EXPORTED_FOR_TESTS
 int cppyy_method_num_args(cppyy_scope_t handle, cppyy_index_t method_index) {
     return s_scopes[handle].m_methods[method_index].m_argtypes.size();
 }
 
-RPY_EXPORTED_FOR_TESTS
 int cppyy_method_req_args(cppyy_scope_t handle, cppyy_index_t method_index) {
     return cppyy_method_num_args(handle, method_index);
 }
 
-RPY_EXPORTED_FOR_TESTS
 char* cppyy_method_arg_type(cppyy_scope_t handle, cppyy_index_t method_index, int arg_index) {
     return cppstring_to_cstring(s_scopes[handle].m_methods[method_index].m_argtypes[arg_index]);
 }
 
-RPY_EXPORTED_FOR_TESTS
 char* cppyy_method_arg_default(
         cppyy_scope_t /* handle */, cppyy_index_t /* method_index */, int /* arg_index */) {
     return cppstring_to_cstring("");
 }
 
-RPY_EXPORTED_FOR_TESTS
 char* cppyy_method_signature(cppyy_scope_t /* handle */, cppyy_index_t /* method_index */) {
     return cppstring_to_cstring("");
 }
 
-RPY_EXPORTED_FOR_TESTS
 int cppyy_method_is_template(cppyy_scope_t /* handle */, cppyy_index_t /* method_index */) {
     return 0;
 }
     
-RPY_EXPORTED_FOR_TESTS
 cppyy_method_t cppyy_get_method(cppyy_scope_t handle, cppyy_index_t method_index) {
     if (s_scopes.find(handle) != s_scopes.end()) {
         long id = s_scopes[handle].m_method_offset + (long)method_index;
@@ -911,7 +872,6 @@
 
 
 /* method properties -----------------------------------------------------  */
-RPY_EXPORTED_FOR_TESTS
 int cppyy_is_constructor(cppyy_type_t handle, cppyy_index_t method_index) {
     if (s_scopes.find(handle) != s_scopes.end())
         return s_scopes[handle].m_methods[method_index].m_type == kConstructor;
@@ -919,7 +879,6 @@
     return 0;
 }
 
-RPY_EXPORTED_FOR_TESTS
 int cppyy_is_staticmethod(cppyy_type_t handle, cppyy_index_t method_index) {
     if (s_scopes.find(handle) != s_scopes.end())
         return s_scopes[handle].m_methods[method_index].m_type == kStatic;
@@ -929,34 +888,28 @@
 
 
 /* data member reflection information ------------------------------------- */
-RPY_EXPORTED_FOR_TESTS
 int cppyy_num_datamembers(cppyy_scope_t handle) {
     return s_scopes[handle].m_datambrs.size();
 }
 
-RPY_EXPORTED_FOR_TESTS
 char* cppyy_datamember_name(cppyy_scope_t handle, int idatambr) {
     return cppstring_to_cstring(s_scopes[handle].m_datambrs[idatambr].m_name);
 }
 
-RPY_EXPORTED_FOR_TESTS
 char* cppyy_datamember_type(cppyy_scope_t handle, int idatambr) {
     return cppstring_to_cstring(s_scopes[handle].m_datambrs[idatambr].m_type);
 }
 
-RPY_EXPORTED_FOR_TESTS
 ptrdiff_t cppyy_datamember_offset(cppyy_scope_t handle, int idatambr) {
     return s_scopes[handle].m_datambrs[idatambr].m_offset;
 }
 
 
 /* data member properties ------------------------------------------------  */
-RPY_EXPORTED_FOR_TESTS
 int cppyy_is_publicdata(cppyy_scope_t handle, int idatambr) {
     return 1;
 }
 
-RPY_EXPORTED_FOR_TESTS
 int cppyy_is_staticdata(cppyy_scope_t handle, int idatambr) {
     return s_scopes[handle].m_datambrs[idatambr].m_isstatic;
 }
@@ -964,44 +917,37 @@
 
 /* misc helpers ----------------------------------------------------------- */
 #if defined(_MSC_VER)
-RPY_EXPORTED_FOR_TESTS
 long long cppyy_strtoll(const char* str) {
     return _strtoi64(str, NULL, 0);
 }
 
 extern "C" {
-RPY_EXPORTED_FOR_TESTS
 unsigned long long cppyy_strtoull(const char* str) {
     return _strtoui64(str, NULL, 0);
 }
 }
 #else
-RPY_EXPORTED_FOR_TESTS
 long long cppyy_strtoll(const char* str) {
     return strtoll(str, NULL, 0);
 }
 
 extern "C" {
-RPY_EXPORTED_FOR_TESTS
 unsigned long long cppyy_strtoull(const char* str) {
     return strtoull(str, NULL, 0);
 }
 }
 #endif
 
-RPY_EXPORTED_FOR_TESTS
 void cppyy_free(void* ptr) {
     free(ptr);
 }
 
-RPY_EXPORTED_FOR_TESTS
 cppyy_object_t cppyy_charp2stdstring(const char* str) {
     void* arena = new char[sizeof(std::string)];
     new (arena) std::string(str);
     return (cppyy_object_t)arena;
 }
 
-RPY_EXPORTED_FOR_TESTS
 cppyy_object_t cppyy_stdstring2stdstring(cppyy_object_t ptr) {
     void* arena = new char[sizeof(std::string)];
     new (arena) std::string(*(std::string*)ptr);
diff --git a/pypy/module/cppyy/test/conftest.py b/pypy/module/cppyy/test/conftest.py
--- a/pypy/module/cppyy/test/conftest.py
+++ b/pypy/module/cppyy/test/conftest.py
@@ -50,7 +50,7 @@
             eci = ExternalCompilationInfo(
                 separate_module_files=[srcpath.join('dummy_backend.cxx')],
                 include_dirs=[incpath, tstpath, cdir],
-                compile_extra=['-DRPY_EXPORTED_FOR_TESTS=RPY_EXPORTED'],
+                compile_extra=['-DRPY_EXTERN=RPY_EXPORTED'],
                 use_cpp_linker=True,
             )
 
diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -775,8 +775,7 @@
     struct PyPyAPI {
     %(members)s
     } _pypyAPI;
-    RPY_EXPORTED_FOR_TESTS
-    struct PyPyAPI* pypyAPI = &_pypyAPI;
+    RPY_EXTERN struct PyPyAPI* pypyAPI = &_pypyAPI;
     """ % dict(members=structmembers)
 
     functions = generate_decls_and_callbacks(db, export_symbols)
@@ -947,7 +946,7 @@
         name_no_star = process_va_name(name)
         header = ('%s pypy_va_get_%s(va_list* vp)' %
                   (name, name_no_star))
-        pypy_decls.append('RPY_EXPORTED_FOR_TESTS ' + header + ';')
+        pypy_decls.append('RPY_EXTERN ' + header + ';')
         functions.append(header + '\n{return va_arg(*vp, %s);}\n' % name)
 
     for name, (typ, expr) in GLOBALS.iteritems():
@@ -1007,7 +1006,7 @@
     if sys.platform == 'win32':
         get_pythonapi_source = '''
         #include 
-        RPY_EXPORTED_FOR_TESTS
+        RPY_EXTERN
         HANDLE pypy_get_pythonapi_handle() {
             MEMORY_BASIC_INFORMATION  mi;
             memset(&mi, 0, sizeof(mi));
diff --git a/pypy/module/cpyext/include/modsupport.h b/pypy/module/cpyext/include/modsupport.h
--- a/pypy/module/cpyext/include/modsupport.h
+++ b/pypy/module/cpyext/include/modsupport.h
@@ -82,11 +82,20 @@
 /*
  * This is from pyport.h.  Perhaps it belongs elsewhere.
  */
+#ifdef _WIN32
+/* explicitly export since PyAPI_FUNC is usually dllimport */
+#ifdef __cplusplus
+#define PyMODINIT_FUNC extern "C" __declspec(dllexport) void
+#else
+#define PyMODINIT_FUNC __declspec(dllexport) void
+#endif
+#else
 #ifdef __cplusplus
 #define PyMODINIT_FUNC extern "C" PyAPI_FUNC(PyObject *)
 #else
 #define PyMODINIT_FUNC PyAPI_FUNC(PyObject *)
 #endif
+#endif /* WIN32 */
 
 PyAPI_DATA(char *) _Py_PackageContext;
 
diff --git a/pypy/module/cpyext/test/conftest.py b/pypy/module/cpyext/test/conftest.py
--- a/pypy/module/cpyext/test/conftest.py
+++ b/pypy/module/cpyext/test/conftest.py
@@ -7,7 +7,7 @@
     # it's necessary to run "import time" at least once before any
     # other cpyext test, otherwise the same statement will fail in
     # test_datetime.py.
-    space = gettestobjspace(usemodules=['rctime'])
+    space = gettestobjspace(usemodules=['time'])
     space.getbuiltinmodule("time")
 
 def pytest_ignore_collect(path, config):
diff --git a/pypy/module/cpyext/test/test_cpyext.py b/pypy/module/cpyext/test/test_cpyext.py
--- a/pypy/module/cpyext/test/test_cpyext.py
+++ b/pypy/module/cpyext/test/test_cpyext.py
@@ -102,7 +102,7 @@
 class LeakCheckingTest(object):
     """Base class for all cpyext tests."""
     spaceconfig = dict(usemodules=['cpyext', 'thread', '_rawffi', 'array',
-                                   'itertools', 'rctime', 'binascii', 'micronumpy'])
+                                   'itertools', 'time', 'binascii', 'micronumpy'])
     spaceconfig['std.withmethodcache'] = True
 
     enable_leak_checking = True
diff --git a/pypy/module/fcntl/test/test_fcntl.py b/pypy/module/fcntl/test/test_fcntl.py
--- a/pypy/module/fcntl/test/test_fcntl.py
+++ b/pypy/module/fcntl/test/test_fcntl.py
@@ -12,7 +12,7 @@
 
 class AppTestFcntl:
     spaceconfig = dict(usemodules=('fcntl', 'array', 'struct', 'termios',
-                                   'select', 'rctime'))
+                                   'select', 'time'))
 
     def setup_class(cls):
         tmpprefix = str(udir.ensure('test_fcntl', dir=1).join('tmp_'))
diff --git a/pypy/module/imp/test/test_app.py b/pypy/module/imp/test/test_app.py
--- a/pypy/module/imp/test/test_app.py
+++ b/pypy/module/imp/test/test_app.py
@@ -4,7 +4,7 @@
 
 class AppTestImpModule:
     spaceconfig = {
-        'usemodules': ['binascii', 'imp', 'itertools', 'rctime', 'struct'],
+        'usemodules': ['binascii', 'imp', 'itertools', 'time', 'struct'],
     }
 
     def setup_class(cls):
diff --git a/pypy/module/imp/test/test_import.py b/pypy/module/imp/test/test_import.py
--- a/pypy/module/imp/test/test_import.py
+++ b/pypy/module/imp/test/test_import.py
@@ -194,7 +194,7 @@
 
 class AppTestImport(BaseImportTest):
     spaceconfig = {
-        "usemodules": ['rctime'],
+        "usemodules": ['time'],
     }
 
     def setup_class(cls):
@@ -1199,7 +1199,7 @@
 
 class AppTestImportHooks(object):
     spaceconfig = {
-        "usemodules": ['struct', 'itertools', 'rctime'],
+        "usemodules": ['struct', 'itertools', 'time'],
     }
 
     def setup_class(cls):
@@ -1446,7 +1446,7 @@
 
 
 class AppTestMultithreadedImp(object):
-    spaceconfig = dict(usemodules=['thread', 'rctime'])
+    spaceconfig = dict(usemodules=['thread', 'time'])
 
     def setup_class(cls):
         #if not conftest.option.runappdirect:
diff --git a/pypy/module/math/test/test_math.py b/pypy/module/math/test/test_math.py
--- a/pypy/module/math/test/test_math.py
+++ b/pypy/module/math/test/test_math.py
@@ -7,7 +7,7 @@
 
 class AppTestMath:
     spaceconfig = {
-        "usemodules": ['math', 'struct', 'itertools', 'rctime', 'binascii'],
+        "usemodules": ['math', 'struct', 'itertools', 'time', 'binascii'],
     }
 
     def setup_class(cls):
diff --git a/pypy/module/micronumpy/nditer.py b/pypy/module/micronumpy/nditer.py
--- a/pypy/module/micronumpy/nditer.py
+++ b/pypy/module/micronumpy/nditer.py
@@ -89,10 +89,9 @@
     for w_item in lst:
         if not space.isinstance_w(w_item, space.w_str) and not \
                 space.isinstance_w(w_item, space.w_unicode):
-            typename = space.type(w_item).getname(space)
             raise oefmt(space.w_TypeError,
-                        'expected string or Unicode object, %s found',
-                        typename)
+                        "expected string or Unicode object, %T found",
+                        w_item)
         item = space.str_w(w_item)
         if item == 'external_loop':
             nditer.external_loop = True
diff --git a/pypy/module/mmap/interp_mmap.py b/pypy/module/mmap/interp_mmap.py
--- a/pypy/module/mmap/interp_mmap.py
+++ b/pypy/module/mmap/interp_mmap.py
@@ -5,6 +5,7 @@
 from rpython.rlib import rmmap, rarithmetic
 from rpython.rlib.buffer import Buffer
 from rpython.rlib.rmmap import RValueError, RTypeError, RMMapError
+from rpython.rlib.rstring import StringBuilder
 
 if rmmap.HAVE_LARGEFILE_SUPPORT:
     OFF_T = rarithmetic.r_longlong
@@ -171,17 +172,18 @@
         self.check_valid()
 
         space = self.space
-        start, stop, step = space.decode_index(w_index, self.mmap.size)
+        start, stop, step, length = space.decode_index4(w_index, self.mmap.size)
         if step == 0:  # index only
             return space.wrap(ord(self.mmap.getitem(start)))
         elif step == 1:
             if stop - start < 0:
                 return space.wrapbytes("")
-            return space.wrapbytes(self.mmap.getslice(start, stop - start))
+            return space.wrapbytes(self.mmap.getslice(start, length))
         else:
-            res = "".join([self.mmap.getitem(i)
-                           for i in range(start, stop, step)])
-            return space.wrapbytes(res)
+            b = StringBuilder(length)
+            for i in range(start, stop, step):
+                b.append(self.mmap.getitem(i))
+            return space.wrapbytes(b.build())
 
     def descr_setitem(self, w_index, w_value):
         space = self.space
diff --git a/pypy/module/operator/tscmp.h b/pypy/module/operator/tscmp.h
--- a/pypy/module/operator/tscmp.h
+++ b/pypy/module/operator/tscmp.h
@@ -1,1 +1,1 @@
-RPY_EXPORTED_FOR_TESTS int pypy_tscmp(const char *, const char *, long, long);
+RPY_EXTERN int pypy_tscmp(const char *, const char *, long, long);
diff --git a/pypy/module/posix/test/test_posix2.py b/pypy/module/posix/test/test_posix2.py
--- a/pypy/module/posix/test/test_posix2.py
+++ b/pypy/module/posix/test/test_posix2.py
@@ -14,7 +14,7 @@
 import signal
 
 def setup_module(mod):
-    usemodules = ['binascii', 'posix', 'signal', 'struct', 'rctime']
+    usemodules = ['binascii', 'posix', 'signal', 'struct', 'time']
     # py3k os.open uses subprocess, requiring the following per platform
     if os.name != 'nt':
         usemodules += ['fcntl', 'select']
diff --git a/pypy/module/pyexpat/test/test_parser.py b/pypy/module/pyexpat/test/test_parser.py
--- a/pypy/module/pyexpat/test/test_parser.py
+++ b/pypy/module/pyexpat/test/test_parser.py
@@ -200,7 +200,7 @@
 
 class AppTestPyexpat2:
     spaceconfig = dict(usemodules=['_rawffi', 'pyexpat', 'itertools',
-                                   '_socket', 'rctime', 'struct', 'binascii'])
+                                   '_socket', 'time', 'struct', 'binascii'])
 
     def test_django_bug(self):
         xml_str = ''
diff --git a/pypy/module/pypyjit/test/test_policy.py b/pypy/module/pypyjit/test/test_policy.py
--- a/pypy/module/pypyjit/test/test_policy.py
+++ b/pypy/module/pypyjit/test/test_policy.py
@@ -29,7 +29,7 @@
     assert pypypolicy.look_inside_function(get_ident)
 
 def test_time():
-    from pypy.module.rctime.interp_time import time
+    from pypy.module.time.interp_time import time
     assert pypypolicy.look_inside_function(time)
 
 def test_io():
diff --git a/pypy/module/rctime/__init__.py b/pypy/module/rctime/__init__.py
deleted file mode 100644
--- a/pypy/module/rctime/__init__.py
+++ /dev/null
@@ -1,42 +0,0 @@
-
-from pypy.interpreter.mixedmodule import MixedModule
-import os
-
-_WIN = os.name == "nt"
-
-class Module(MixedModule):
-    applevel_name = 'time'
-
-    interpleveldefs = {
-        'time': 'interp_time.time',
-        'clock': 'interp_time.clock',
-        'ctime': 'interp_time.ctime',
-        'asctime': 'interp_time.asctime',
-        'gmtime': 'interp_time.gmtime',
-        'localtime': 'interp_time.localtime',
-        'mktime': 'interp_time.mktime',
-        'strftime': 'interp_time.strftime',
-        'sleep' : 'interp_time.sleep',
-    }
-
-    if os.name == "posix":
-        interpleveldefs['tzset'] = 'interp_time.tzset'
-
-    appleveldefs = {
-        'struct_time': 'app_time.struct_time',
-        '__doc__': 'app_time.__doc__',
-        'strptime': 'app_time.strptime',
-    }
-
-    def startup(self, space):
-        if _WIN:
-            from pypy.module.rctime.interp_time import State
-            space.fromcache(State).startup(space)
-
-        # this machinery is needed to expose constants
-        # that have to be initialized one time only
-        from pypy.module.rctime import interp_time
-
-        interp_time._init_timezone(space)
-        interp_time._init_accept2dyear(space)
-
diff --git a/pypy/module/rctime/interp_time.py b/pypy/module/rctime/interp_time.py
deleted file mode 100644
--- a/pypy/module/rctime/interp_time.py
+++ /dev/null
@@ -1,687 +0,0 @@
-from rpython.rtyper.tool import rffi_platform as platform
-from rpython.rtyper.lltypesystem import rffi
-from pypy.interpreter.error import OperationError, oefmt
-from pypy.interpreter.gateway import unwrap_spec
-from rpython.rtyper.lltypesystem import lltype
-from rpython.rlib.rarithmetic import intmask
-from rpython.rlib import rposix
-from rpython.translator.tool.cbuild import ExternalCompilationInfo
-import os
-import sys
-import time as pytime
-
-_POSIX = os.name == "posix"
-_WIN = os.name == "nt"
-_CYGWIN = sys.platform == "cygwin"
-
-_time_zones = []
-if _CYGWIN:
-    _time_zones = ["GMT-12", "GMT-11", "GMT-10", "GMT-9", "GMT-8", "GMT-7",
-                   "GMT-6", "GMT-5", "GMT-4", "GMT-3", "GMT-2", "GMT-1",
-                   "GMT",  "GMT+1", "GMT+2", "GMT+3", "GMT+4", "GMT+5",
-                   "GMT+6",  "GMT+7", "GMT+8", "GMT+9", "GMT+10", "GMT+11",
-                   "GMT+12",  "GMT+13", "GMT+14"]
-
-if _WIN:
-    # Interruptible sleeps on Windows:
-    # We install a specific Console Ctrl Handler which sets an 'event'.
-    # time.sleep() will actually call WaitForSingleObject with the desired
-    # timeout.  On Ctrl-C, the signal handler is called, the event is set,
-    # and the wait function exits.
-    from rpython.rlib import rwin32
-    from pypy.interpreter.error import wrap_windowserror, wrap_oserror
-    from rpython.rlib import rthread as thread
-
-    eci = ExternalCompilationInfo(
-        includes = ['windows.h'],
-        post_include_bits = [
-            "RPY_EXPORTED_FOR_TESTS\n"
-            "BOOL pypy_timemodule_setCtrlHandler(HANDLE event);"],
-        separate_module_sources=['''
-            static HANDLE interrupt_event;
-
-            static BOOL WINAPI CtrlHandlerRoutine(
-              DWORD dwCtrlType)
-            {
-                SetEvent(interrupt_event);
-                /* allow other default handlers to be called.
-                 * Default Python handler will setup the
-                 * KeyboardInterrupt exception.
-                 */
-                return 0;
-            }
-
-            BOOL pypy_timemodule_setCtrlHandler(HANDLE event)
-            {
-                interrupt_event = event;
-                return SetConsoleCtrlHandler(CtrlHandlerRoutine, TRUE);
-            }
-
-        '''],
-        )
-    _setCtrlHandlerRoutine = rffi.llexternal(
-        'pypy_timemodule_setCtrlHandler',
-        [rwin32.HANDLE], rwin32.BOOL,
-        compilation_info=eci)
-
-    class GlobalState:
-        def __init__(self):
-            self.init()
-
-        def init(self):
-            self.interrupt_event = rwin32.NULL_HANDLE
-
-        def startup(self, space):
-            # Initialize the event handle used to signal Ctrl-C
-            try:
-                globalState.interrupt_event = rwin32.CreateEvent(
-                    rffi.NULL, True, False, rffi.NULL)
-            except WindowsError, e:
-                raise wrap_windowserror(space, e)
-            if not _setCtrlHandlerRoutine(globalState.interrupt_event):
-                raise wrap_windowserror(
-                    space, rwin32.lastWindowsError("SetConsoleCtrlHandler"))
-
-    globalState = GlobalState()
-
-    class State:
-        def __init__(self, space):
-            self.main_thread = 0
-
-        def _cleanup_(self):
-            self.main_thread = 0
-            globalState.init()
-
-        def startup(self, space):
-            self.main_thread = thread.get_ident()
-            globalState.startup(space)
-
-        def get_interrupt_event(self):
-            return globalState.interrupt_event
-
-
-_includes = ["time.h"]
-if _POSIX:
-    _includes.append('sys/time.h')
-
-class CConfig:
-    _compilation_info_ = ExternalCompilationInfo(
-        includes = _includes
-    )
-    CLOCKS_PER_SEC = platform.ConstantInteger("CLOCKS_PER_SEC")
-    clock_t = platform.SimpleType("clock_t", rffi.ULONG)
-    has_gettimeofday = platform.Has('gettimeofday')
-
-if _POSIX:
-    calling_conv = 'c'
-    CConfig.timeval = platform.Struct("struct timeval",
-                                      [("tv_sec", rffi.INT),
-                                       ("tv_usec", rffi.INT)])
-    if _CYGWIN:
-        CConfig.tm = platform.Struct("struct tm", [("tm_sec", rffi.INT),
-            ("tm_min", rffi.INT), ("tm_hour", rffi.INT), ("tm_mday", rffi.INT),
-            ("tm_mon", rffi.INT), ("tm_year", rffi.INT), ("tm_wday", rffi.INT),
-            ("tm_yday", rffi.INT), ("tm_isdst", rffi.INT)])
-    else:
-        CConfig.tm = platform.Struct("struct tm", [("tm_sec", rffi.INT),
-            ("tm_min", rffi.INT), ("tm_hour", rffi.INT), ("tm_mday", rffi.INT),
-            ("tm_mon", rffi.INT), ("tm_year", rffi.INT), ("tm_wday", rffi.INT),
-            ("tm_yday", rffi.INT), ("tm_isdst", rffi.INT), ("tm_gmtoff", rffi.LONG),
-            ("tm_zone", rffi.CCHARP)])
-elif _WIN:
-    calling_conv = 'win'
-    CConfig.tm = platform.Struct("struct tm", [("tm_sec", rffi.INT),
-        ("tm_min", rffi.INT), ("tm_hour", rffi.INT), ("tm_mday", rffi.INT),
-        ("tm_mon", rffi.INT), ("tm_year", rffi.INT), ("tm_wday", rffi.INT),
-        ("tm_yday", rffi.INT), ("tm_isdst", rffi.INT)])
-
-class cConfig:
-    pass
-
-for k, v in platform.configure(CConfig).items():
-    setattr(cConfig, k, v)
-cConfig.tm.__name__ = "_tm"
-
-def external(name, args, result, eci=CConfig._compilation_info_):
-    if _WIN and rffi.sizeof(rffi.TIME_T) == 8:
-        # Recent Microsoft compilers use 64bit time_t and
-        # the corresponding functions are named differently
-        if (rffi.TIME_T in args or rffi.TIME_TP in args
-            or result in (rffi.TIME_T, rffi.TIME_TP)):
-            name = '_' + name + '64'
-    return rffi.llexternal(name, args, result,
-                           compilation_info=eci,
-                           calling_conv=calling_conv,
-                           releasegil=False)
-
-if _POSIX:
-    cConfig.timeval.__name__ = "_timeval"
-    timeval = cConfig.timeval
-
-CLOCKS_PER_SEC = cConfig.CLOCKS_PER_SEC
-clock_t = cConfig.clock_t
-tm = cConfig.tm
-glob_buf = lltype.malloc(tm, flavor='raw', zero=True, immortal=True)
-
-if cConfig.has_gettimeofday:
-    c_gettimeofday = external('gettimeofday', [rffi.VOIDP, rffi.VOIDP], rffi.INT)
-TM_P = lltype.Ptr(tm)
-c_clock = external('clock', [rffi.TIME_TP], clock_t)
-c_time = external('time', [rffi.TIME_TP], rffi.TIME_T)
-c_gmtime = external('gmtime', [rffi.TIME_TP], TM_P)
-c_mktime = external('mktime', [TM_P], rffi.TIME_T)
-c_localtime = external('localtime', [rffi.TIME_TP], TM_P)
-if _POSIX:
-    c_tzset = external('tzset', [], lltype.Void)
-if _WIN:
-    win_eci = ExternalCompilationInfo(
-        includes = ["time.h"],
-        post_include_bits = ["RPY_EXPORTED_FOR_TESTS\n"
-                             "long pypy_get_timezone();\n"
-                             "RPY_EXPORTED_FOR_TESTS\n"
-                             "int pypy_get_daylight();\n"
-                             "RPY_EXPORTED_FOR_TESTS\n"
-                             "char** pypy_get_tzname();\n"
-                             "RPY_EXPORTED_FOR_TESTS\n"
-                             "void* pypy__tzset();"],
-        separate_module_sources = ["""
-        long pypy_get_timezone() { return timezone; }
-        int pypy_get_daylight() { return daylight; }
-        char** pypy_get_tzname() { return tzname; }
-        void pypy__tzset() { return _tzset(); }
-        """])
-    # Ensure sure that we use _tzset() and timezone from the same C Runtime.
-    c_tzset = external('pypy__tzset', [], lltype.Void, win_eci)
-    c_get_timezone = external('pypy_get_timezone', [], rffi.LONG, win_eci)
-    c_get_daylight = external('pypy_get_daylight', [], rffi.INT, win_eci)
-    c_get_tzname = external('pypy_get_tzname', [], rffi.CCHARPP, win_eci)
-
-c_strftime = external('strftime', [rffi.CCHARP, rffi.SIZE_T, rffi.CCHARP, TM_P],
-                      rffi.SIZE_T)
-
-def _init_accept2dyear(space):
-    if os.environ.get("PYTHONY2K"):
-        accept2dyear = 0
-    else:
-        accept2dyear = 1
-    _set_module_object(space, "accept2dyear", space.wrap(accept2dyear))
-
-def _init_timezone(space):
-    timezone = daylight = altzone = 0
-    tzname = ["", ""]
-
-    if _WIN:
-        c_tzset()
-        timezone = c_get_timezone()
-        altzone = timezone - 3600
-        daylight = c_get_daylight()
-        tzname_ptr = c_get_tzname()
-        tzname = rffi.charp2str(tzname_ptr[0]), rffi.charp2str(tzname_ptr[1])
-
-    if _POSIX:
-        if _CYGWIN:
-            YEAR = (365 * 24 + 6) * 3600
-
-            # about January 11th
-            t = (((c_time(lltype.nullptr(rffi.TIME_TP.TO))) / YEAR) * YEAR + 10 * 24 * 3600)
-            # we cannot have reference to stack variable, put it on the heap
-            t_ref = lltype.malloc(rffi.TIME_TP.TO, 1, flavor='raw')
-            t_ref[0] = rffi.cast(rffi.TIME_T, t)
-            p = c_localtime(t_ref)
-            q = c_gmtime(t_ref)
-            janzone = (p.c_tm_hour + 24 * p.c_tm_mday) - (q.c_tm_hour + 24 * q.c_tm_mday)
-            if janzone < -12:
-                janname = "   "
-            elif janzone > 14:
-                janname = "   "
-            else:
-                janname = _time_zones[janzone - 12]
-            janzone = janzone * 3600
-            # about July 11th
-            tt = t + YEAR / 2
-            t_ref[0] = rffi.cast(rffi.TIME_T, tt)
-            p = c_localtime(t_ref)
-            q = c_gmtime(t_ref)
-            julyzone = (p.c_tm_hour + 24 * p.c_tm_mday) - (q.c_tm_hour + 24 * q.c_tm_mday)
-            if julyzone < -12:
-                julyname = "   "
-            elif julyzone > 14:
-                julyname = "   "
-            else:
-                julyname = _time_zones[julyzone - 12]
-            julyzone = julyzone * 3600
-            lltype.free(t_ref, flavor='raw')
-
-            if janzone < julyzone:
-                # DST is reversed in the southern hemisphere
-                timezone = julyzone
-                altzone = janzone
-                daylight = int(janzone != julyzone)
-                tzname = [julyname, janname]
-            else:
-                timezone = janzone
-                altzone = julyzone
-                daylight = int(janzone != julyzone)
-                tzname = [janname, julyname]
-
-        else:
-            YEAR = (365 * 24 + 6) * 3600
-
-            t = (((c_time(lltype.nullptr(rffi.TIME_TP.TO))) / YEAR) * YEAR)
-            # we cannot have reference to stack variable, put it on the heap
-            t_ref = lltype.malloc(rffi.TIME_TP.TO, 1, flavor='raw')
-            t_ref[0] = rffi.cast(rffi.TIME_T, t)
-            p = c_localtime(t_ref)
-            janzone = -p.c_tm_gmtoff
-            tm_zone = rffi.charp2str(p.c_tm_zone)
-            janname = ["   ", tm_zone][bool(tm_zone)]
-            tt = t + YEAR / 2
-            t_ref[0] = rffi.cast(rffi.TIME_T, tt)
-            p = c_localtime(t_ref)
-            lltype.free(t_ref, flavor='raw')
-            tm_zone = rffi.charp2str(p.c_tm_zone)
-            julyzone = -p.c_tm_gmtoff
-            julyname = ["   ", tm_zone][bool(tm_zone)]
-
-            if janzone < julyzone:
-                # DST is reversed in the southern hemisphere
-                timezone = julyzone
-                altzone = janzone
-                daylight = int(janzone != julyzone)
-                tzname = [julyname, janname]
-            else:
-                timezone = janzone
-                altzone = julyzone
-                daylight = int(janzone != julyzone)
-                tzname = [janname, julyname]
-
-    _set_module_object(space, "timezone", space.wrap(timezone))
-    _set_module_object(space, 'daylight', space.wrap(daylight))
-    tzname_w = [space.wrap(tzname[0]), space.wrap(tzname[1])]
-    _set_module_object(space, 'tzname', space.newtuple(tzname_w))
-    _set_module_object(space, 'altzone', space.wrap(altzone))
-
-def _get_error_msg():
-    errno = rposix.get_errno()
-    return os.strerror(errno)
-
-if sys.platform != 'win32':
-    @unwrap_spec(secs=float)
-    def sleep(space, secs):
-        if secs < 0:
-            raise OperationError(space.w_IOError,
-                                 space.wrap("Invalid argument: negative time in sleep"))
-        pytime.sleep(secs)
-else:
-    from rpython.rlib import rwin32
-    from errno import EINTR
-    def _simple_sleep(space, secs, interruptible):
-        if secs == 0.0 or not interruptible:
-            pytime.sleep(secs)
-        else:
-            millisecs = int(secs * 1000)
-            interrupt_event = space.fromcache(State).get_interrupt_event()
-            rwin32.ResetEvent(interrupt_event)
-            rc = rwin32.WaitForSingleObject(interrupt_event, millisecs)
-            if rc == rwin32.WAIT_OBJECT_0:
-                # Yield to make sure real Python signal handler
-                # called.
-                pytime.sleep(0.001)
-                raise wrap_oserror(space,
-                                   OSError(EINTR, "sleep() interrupted"))
-    @unwrap_spec(secs=float)
-    def sleep(space, secs):
-        if secs < 0:
-            raise OperationError(space.w_IOError,
-                                 space.wrap("Invalid argument: negative time in sleep"))
-        # as decreed by Guido, only the main thread can be
-        # interrupted.
-        main_thread = space.fromcache(State).main_thread
-        interruptible = (main_thread == thread.get_ident())
-        MAX = sys.maxint / 1000.0 # > 24 days
-        while secs > MAX:
-            _simple_sleep(space, MAX, interruptible)
-            secs -= MAX
-        _simple_sleep(space, secs, interruptible)
-
-def _get_module_object(space, obj_name):
-    w_module = space.getbuiltinmodule('time')
-    w_obj = space.getattr(w_module, space.wrap(obj_name))
-    return w_obj
-
-def _set_module_object(space, obj_name, w_obj_value):
-    w_module = space.getbuiltinmodule('time')
-    space.setattr(w_module, space.wrap(obj_name), w_obj_value)
-
-def _get_inttime(space, w_seconds):
-    # w_seconds can be a wrapped None (it will be automatically wrapped
-    # in the callers, so we never get a real None here).
-    if space.is_none(w_seconds):
-        seconds = pytime.time()
-    else:
-        seconds = space.float_w(w_seconds)
-    #
-    t = rffi.cast(rffi.TIME_T, seconds)
-    #
-    # Logic from CPython: How much info did we lose?  We assume that
-    # time_t is an integral type.  If we lost a second or more, the
-    # input doesn't fit in a time_t; call it an error.
-    diff = seconds - rffi.cast(lltype.Float, t)
-    if diff <= -1.0 or diff >= 1.0:
-        raise OperationError(space.w_ValueError,
-                      space.wrap("timestamp out of range for platform time_t"))
-    return t
-
-def _tm_to_tuple(space, t):
-    time_tuple = [
-        space.wrap(rffi.getintfield(t, 'c_tm_year') + 1900),
-        space.wrap(rffi.getintfield(t, 'c_tm_mon') + 1), # want january == 1
-        space.wrap(rffi.getintfield(t, 'c_tm_mday')),
-        space.wrap(rffi.getintfield(t, 'c_tm_hour')),
-        space.wrap(rffi.getintfield(t, 'c_tm_min')),
-        space.wrap(rffi.getintfield(t, 'c_tm_sec')),
-        space.wrap((rffi.getintfield(t, 'c_tm_wday') + 6) % 7), # want monday == 0
-        space.wrap(rffi.getintfield(t, 'c_tm_yday') + 1), # want january, 1 == 1
-        space.wrap(rffi.getintfield(t, 'c_tm_isdst'))]
-
-    w_struct_time = _get_module_object(space, 'struct_time')
-    w_time_tuple = space.newtuple(time_tuple)
-    return space.call_function(w_struct_time, w_time_tuple)
-
-def _gettmarg(space, w_tup, allowNone=True):
-    if space.is_none(w_tup):
-        if not allowNone:
-            raise OperationError(space.w_TypeError,
-                                 space.wrap("tuple expected"))
-        # default to the current local time
-        tt = rffi.r_time_t(int(pytime.time()))
-        t_ref = lltype.malloc(rffi.TIME_TP.TO, 1, flavor='raw')
-        t_ref[0] = tt
-        pbuf = c_localtime(t_ref)
-        lltype.free(t_ref, flavor='raw')
-        if not pbuf:
-            raise OperationError(space.w_ValueError,
-                space.wrap(_get_error_msg()))
-        return pbuf
-
-    tup_w = space.fixedview(w_tup)
-    if len(tup_w) != 9:
-        raise oefmt(space.w_TypeError,
-                    "argument must be sequence of length 9, not %d",
-                    len(tup_w))
-
-    y = space.c_int_w(tup_w[0])
-    tm_mon = space.c_int_w(tup_w[1])
-    if tm_mon == 0:
-        tm_mon = 1
-    tm_mday = space.c_int_w(tup_w[2])
-    if tm_mday == 0:
-        tm_mday = 1
-    tm_yday = space.c_int_w(tup_w[7])
-    if tm_yday == 0:
-        tm_yday = 1
-    rffi.setintfield(glob_buf, 'c_tm_mon', tm_mon)
-    rffi.setintfield(glob_buf, 'c_tm_mday', tm_mday)
-    rffi.setintfield(glob_buf, 'c_tm_hour', space.c_int_w(tup_w[3]))
-    rffi.setintfield(glob_buf, 'c_tm_min', space.c_int_w(tup_w[4]))
-    rffi.setintfield(glob_buf, 'c_tm_sec', space.c_int_w(tup_w[5]))
-    rffi.setintfield(glob_buf, 'c_tm_wday', space.c_int_w(tup_w[6]))
-    rffi.setintfield(glob_buf, 'c_tm_yday', tm_yday)
-    rffi.setintfield(glob_buf, 'c_tm_isdst', space.c_int_w(tup_w[8]))
-    if _POSIX:
-        if _CYGWIN:
-            pass
-        else:
-            # actually never happens, but makes annotator happy
-            glob_buf.c_tm_zone = lltype.nullptr(rffi.CCHARP.TO)
-            rffi.setintfield(glob_buf, 'c_tm_gmtoff', 0)
-
-    if y < 1000:
-        w_accept2dyear = _get_module_object(space, "accept2dyear")
-        accept2dyear = space.is_true(w_accept2dyear)
-
-        if accept2dyear:
-            if 69 <= y <= 99:
-                y += 1900
-            elif 0 <= y <= 68:
-                y += 2000
-            else:
-                raise OperationError(space.w_ValueError,
-                                     space.wrap("year out of range"))
-            space.warn(space.wrap("Century info guessed for a 2-digit year."),
-                       space.w_DeprecationWarning)
-
-    # tm_wday does not need checking of its upper-bound since taking "%
-    #  7" in _gettmarg() automatically restricts the range.
-    if rffi.getintfield(glob_buf, 'c_tm_wday') < -1:
-        raise OperationError(space.w_ValueError,
-                             space.wrap("day of week out of range"))
-
-    rffi.setintfield(glob_buf, 'c_tm_year', y - 1900)
-    rffi.setintfield(glob_buf, 'c_tm_mon',
-                     rffi.getintfield(glob_buf, 'c_tm_mon') - 1)
-    rffi.setintfield(glob_buf, 'c_tm_wday',
-                     (rffi.getintfield(glob_buf, 'c_tm_wday') + 1) % 7)
-    rffi.setintfield(glob_buf, 'c_tm_yday',
-                     rffi.getintfield(glob_buf, 'c_tm_yday') - 1)
-
-    return glob_buf
-
-def _checktm(space, t_ref):
-    """Checks added to make sure strftime() and asctime() do not crash
-    Python by indexing blindly into some array for a textual
-    representation by some bad index (fixes bug #897625).  No check for
-    year or wday since handled in _gettmarg()."""
-    if not 0 <= rffi.getintfield(t_ref, 'c_tm_mon') <= 11:
-        raise OperationError(space.w_ValueError,
-                             space.wrap("month out of range"))
-    if not 1 <= rffi.getintfield(t_ref, 'c_tm_mday') <= 31:
-        raise OperationError(space.w_ValueError,
-                             space.wrap("day of month out of range"))
-    if not 0 <= rffi.getintfield(t_ref, 'c_tm_hour') <= 23:
-        raise OperationError(space.w_ValueError,
-                             space.wrap("hour out of range"))
-    if not 0 <= rffi.getintfield(t_ref, 'c_tm_min') <= 59:
-        raise OperationError(space.w_ValueError,
-                             space.wrap("minute out of range"))
-    if not 0 <= rffi.getintfield(t_ref, 'c_tm_sec') <= 61:
-        raise OperationError(space.w_ValueError,
-                             space.wrap("seconds out of range"))
-    # tm_wday does not need checking: "% 7" in _gettmarg() automatically
-    # restricts the range
-    if not 0 <= rffi.getintfield(t_ref, 'c_tm_yday') <= 365:
-        raise OperationError(space.w_ValueError,
-                             space.wrap("day of year out of range"))
-
-def time(space):
-    """time() -> floating point number
-
-    Return the current time in seconds since the Epoch.
-    Fractions of a second may be present if the system clock provides them."""
-
-    secs = pytime.time()
-    return space.wrap(secs)
-
-if _WIN:
-    class PCCache:
-        pass
-    pccache = PCCache()
-    pccache.divisor = 0.0
-    pccache.ctrStart = 0
-
-def clock(space):
-    """clock() -> floating point number
-
-    Return the CPU time or real time since the start of the process or since
-    the first call to clock().  This has as much precision as the system
-    records."""
-
-    return space.wrap(pytime.clock())
-
-def ctime(space, w_seconds=None):
-    """ctime([seconds]) -> string
-
-    Convert a time in seconds since the Epoch to a string in local time.
-    This is equivalent to asctime(localtime(seconds)). When the time tuple is
-    not present, current time as returned by localtime() is used."""
-
-    seconds = _get_inttime(space, w_seconds)
-    with lltype.scoped_alloc(rffi.TIME_TP.TO, 1) as t_ref:
-        t_ref[0] = seconds
-        p = c_localtime(t_ref)
-    if not p:
-        raise OperationError(space.w_ValueError,
-                             space.wrap("unconvertible time"))
-    return _asctime(space, p)
-
-# by now w_tup is an optional argument (and not *args)
-# because of the ext. compiler bugs in handling such arguments (*args, **kwds)
-def asctime(space, w_tup=None):
-    """asctime([tuple]) -> string
-
-    Convert a time tuple to a string, e.g. 'Sat Jun 06 16:26:11 1998'.
-    When the time tuple is not present, current time as returned by localtime()
-    is used."""
-    buf_value = _gettmarg(space, w_tup)
-    _checktm(space, buf_value)
-    return _asctime(space, buf_value)
-
-_wday_names = ["Sun", "Mon", "Tue", "Wed", "Thu", "Fri", "Sat"]
-_mon_names = ["Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sep",
-              "Oct", "Nov", "Dec"]
-
-def _asctime(space, t_ref):
-    # Inspired by Open Group reference implementation available at
-    # http://pubs.opengroup.org/onlinepubs/009695399/functions/asctime.html
-    w, getif = space.wrap, rffi.getintfield
-    args = [w(_wday_names[getif(t_ref, 'c_tm_wday')]),
-            w(_mon_names[getif(t_ref, 'c_tm_mon')]),
-            w(getif(t_ref, 'c_tm_mday')),
-            w(getif(t_ref, 'c_tm_hour')),
-            w(getif(t_ref, 'c_tm_min')),
-            w(getif(t_ref, 'c_tm_sec')),
-            w(getif(t_ref, 'c_tm_year') + 1900)]
-    return space.mod(w("%.3s %.3s%3d %.2d:%.2d:%.2d %d"),
-                     space.newtuple(args))
-
-def gmtime(space, w_seconds=None):
-    """gmtime([seconds]) -> (tm_year, tm_mon, tm_day, tm_hour, tm_min,
-                          tm_sec, tm_wday, tm_yday, tm_isdst)
-
-    Convert seconds since the Epoch to a time tuple expressing UTC (a.k.a.
-    GMT).  When 'seconds' is not passed in, convert the current time instead.
-    """
-
-    # rpython does not support that a variable has two incompatible builtins
-    # as value so we have to duplicate the code. NOT GOOD! see localtime() too
-    seconds = _get_inttime(space, w_seconds)
-    t_ref = lltype.malloc(rffi.TIME_TP.TO, 1, flavor='raw')
-    t_ref[0] = seconds
-    p = c_gmtime(t_ref)
-    lltype.free(t_ref, flavor='raw')
-
-    if not p:
-        raise OperationError(space.w_ValueError, space.wrap(_get_error_msg()))
-    return _tm_to_tuple(space, p)
-
-def localtime(space, w_seconds=None):
-    """localtime([seconds]) -> (tm_year, tm_mon, tm_day, tm_hour, tm_min,
-                             tm_sec, tm_wday, tm_yday, tm_isdst)
-
-    Convert seconds since the Epoch to a time tuple expressing local time.
-    When 'seconds' is not passed in, convert the current time instead."""
-
-    seconds = _get_inttime(space, w_seconds)
-    t_ref = lltype.malloc(rffi.TIME_TP.TO, 1, flavor='raw')
-    t_ref[0] = seconds
-    p = c_localtime(t_ref)
-    lltype.free(t_ref, flavor='raw')
-
-    if not p:
-        raise OperationError(space.w_ValueError, space.wrap(_get_error_msg()))
-    return _tm_to_tuple(space, p)
-
-def mktime(space, w_tup):
-    """mktime(tuple) -> floating point number
-
-    Convert a time tuple in local time to seconds since the Epoch."""
-
-    buf = _gettmarg(space, w_tup, allowNone=False)
-    rffi.setintfield(buf, "c_tm_wday", -1)
-    tt = c_mktime(buf)
-    # A return value of -1 does not necessarily mean an error, but tm_wday
-    # cannot remain set to -1 if mktime succeeds.
-    if tt == -1 and rffi.getintfield(buf, "c_tm_wday") == -1:
-        raise OperationError(space.w_OverflowError,
-            space.wrap("mktime argument out of range"))
-
-    return space.wrap(float(tt))
-
-if _POSIX:
-    def tzset(space):
-        """tzset()
-
-        Initialize, or reinitialize, the local timezone to the value stored in
-        os.environ['TZ']. The TZ environment variable should be specified in
-        standard Unix timezone format as documented in the tzset man page
-        (eg. 'US/Eastern', 'Europe/Amsterdam'). Unknown timezones will silently
-        fall back to UTC. If the TZ environment variable is not set, the local
-        timezone is set to the systems best guess of wallclock time.
-        Changing the TZ environment variable without calling tzset *may* change
-        the local timezone used by methods such as localtime, but this behaviour
-        should not be relied on"""
-
-        c_tzset()
-
-        # reset timezone, altzone, daylight and tzname
-        _init_timezone(space)
-
- at unwrap_spec(format=str)
-def strftime(space, format, w_tup=None):
-    """strftime(format[, tuple]) -> string
-
-    Convert a time tuple to a string according to a format specification.
-    See the library reference manual for formatting codes. When the time tuple
-    is not present, current time as returned by localtime() is used."""
-    buf_value = _gettmarg(space, w_tup)
-    _checktm(space, buf_value)
-
-    # Normalize tm_isdst just in case someone foolishly implements %Z
-    # based on the assumption that tm_isdst falls within the range of
-    # [-1, 1]
-    if rffi.getintfield(buf_value, 'c_tm_isdst') < -1:
-        rffi.setintfield(buf_value, 'c_tm_isdst', -1)
-    elif rffi.getintfield(buf_value, 'c_tm_isdst') > 1:
-        rffi.setintfield(buf_value, 'c_tm_isdst', 1)
-
-    if _WIN:
-        # check that the format string contains only valid directives
-        length = len(format)
-        i = 0
-        while i < length:
-            if format[i] == '%':
-                i += 1
-                if i < length and format[i] == '#':
-                    # not documented by python
-                    i += 1
-                if i >= length or format[i] not in "aAbBcdHIjmMpSUwWxXyYzZ%":
-                    raise OperationError(space.w_ValueError,
-                                         space.wrap("invalid format string"))
-            i += 1
-
-    i = 1024
-    while True:
-        outbuf = lltype.malloc(rffi.CCHARP.TO, i, flavor='raw')
-        try:
-            buflen = c_strftime(outbuf, i, format, buf_value)
-            if buflen > 0 or i >= 256 * len(format):
-                # if the buffer is 256 times as long as the format,
-                # it's probably not failing for lack of room!
-                # More likely, the format yields an empty result,
-                # e.g. an empty format, or %Z when the timezone
-                # is unknown.
-                result = rffi.charp2strn(outbuf, intmask(buflen))
-                return space.wrap(result)
-        finally:
-            lltype.free(outbuf, flavor='raw')
-        i += i
diff --git a/pypy/module/rctime/test/test_rctime.py b/pypy/module/rctime/test/test_rctime.py
deleted file mode 100644
--- a/pypy/module/rctime/test/test_rctime.py
+++ /dev/null
@@ -1,368 +0,0 @@
-class AppTestRCTime:
-    spaceconfig = {
-        "usemodules": ['rctime', 'struct', 'binascii'],
-    }
-
-    def test_attributes(self):
-        import time as rctime
-        assert isinstance(rctime.accept2dyear, int)
-        assert isinstance(rctime.altzone, int)
-        assert isinstance(rctime.daylight, int)
-        assert isinstance(rctime.timezone, int)
-        assert isinstance(rctime.tzname, tuple)
-        assert isinstance(rctime.__doc__, str)
-
-    def test_sleep(self):
-        import time as rctime
-        import sys
-        import os
-        raises(TypeError, rctime.sleep, "foo")
-        rctime.sleep(0.12345)
-        raises(IOError, rctime.sleep, -1.0)
-
-    def test_clock(self):
-        import time as rctime
-        rctime.clock()
-        assert isinstance(rctime.clock(), float)
-
-    def test_time(self):
-        import time as rctime
-        t1 = rctime.time()
-        assert isinstance(rctime.time(), float)
-        assert rctime.time() != 0.0 # 0.0 means failure
-        rctime.sleep(0.02)
-        t2 = rctime.time()
-        assert t1 != t2       # the resolution should be at least 0.01 secs
-
-    def test_ctime(self):
-        import time as rctime
-        raises(TypeError, rctime.ctime, "foo")
-        rctime.ctime(None)
-        rctime.ctime()
-        res = rctime.ctime(0)
-        assert isinstance(res, str)
-        rctime.ctime(rctime.time())
-        raises(ValueError, rctime.ctime, 1E200)
-        raises(OverflowError, rctime.ctime, 10**900)
-        for year in [-100, 100, 1000, 2000, 10000]:
-            try:
-                testval = rctime.mktime((year, 1, 10) + (0,)*6)
-            except (ValueError, OverflowError):
-                # If mktime fails, ctime will fail too.  This may happen
-                # on some platforms.
-                pass
-            else:
-                assert rctime.ctime(testval)[20:] == str(year)
-
-    def test_gmtime(self):
-        import time as rctime
-        raises(TypeError, rctime.gmtime, "foo")
-        rctime.gmtime()
-        rctime.gmtime(None)
-        rctime.gmtime(0)
-        res = rctime.gmtime(rctime.time())
-        assert isinstance(res, rctime.struct_time)
-        assert res[-1] == 0 # DST is always zero in gmtime()
-        t0 = rctime.mktime(rctime.gmtime())
-        t1 = rctime.mktime(rctime.gmtime(None))
-        assert 0 <= (t1 - t0) < 1.2
-        t = rctime.time()
-        assert rctime.gmtime(t) == rctime.gmtime(t)
-        raises(ValueError, rctime.gmtime, 2**64)
-        raises(ValueError, rctime.gmtime, -2**64)
-
-    def test_localtime(self):
-        import time as rctime
-        import os
-        raises(TypeError, rctime.localtime, "foo")
-        rctime.localtime()
-        rctime.localtime(None)
-        rctime.localtime(0)
-        res = rctime.localtime(rctime.time())
-        assert isinstance(res, rctime.struct_time)
-        t0 = rctime.mktime(rctime.localtime())
-        t1 = rctime.mktime(rctime.localtime(None))
-        assert 0 <= (t1 - t0) < 1.2
-        t = rctime.time()
-        assert rctime.localtime(t) == rctime.localtime(t)
-        if os.name == 'nt':
-            raises(ValueError, rctime.localtime, -1)
-        else:
-            rctime.localtime(-1)
-
-    def test_mktime(self):
-        import time as rctime
-        import os, sys
-        raises(TypeError, rctime.mktime, "foo")
-        raises(TypeError, rctime.mktime, None)
-        raises(TypeError, rctime.mktime, (1, 2))
-        raises(TypeError, rctime.mktime, (1, 2, 3, 4, 5, 6, 'f', 8, 9))
-        res = rctime.mktime(rctime.localtime())
-        assert isinstance(res, float)
-
-        ltime = rctime.localtime()
-        rctime.accept2dyear == 0
-        ltime = list(ltime)
-        ltime[0] = -1
-        raises(ValueError, rctime.mktime, tuple(ltime))
-        rctime.accept2dyear == 1
-
-        ltime = list(ltime)
-        ltime[0] = 67
-        ltime = tuple(ltime)
-        if os.name != "nt" and sys.maxsize < 1<<32:   # time_t may be 64bit
-            raises(OverflowError, rctime.mktime, ltime)
-
-        ltime = list(ltime)
-        ltime[0] = 100
-        raises(ValueError, rctime.mktime, tuple(ltime))
-
-        t = rctime.time()
-        assert int(rctime.mktime(rctime.localtime(t))) == int(t)
-        assert int(rctime.mktime(rctime.gmtime(t))) - rctime.timezone == int(t)
-        ltime = rctime.localtime()
-        assert rctime.mktime(tuple(ltime)) == rctime.mktime(ltime)
-        if os.name != 'nt':
-            assert rctime.mktime(rctime.localtime(-1)) == -1
-
-        res = rctime.mktime((2000, 1, 1, 0, 0, 0, -1, -1, -1))
-        if os.name == 'nt':
-            assert rctime.ctime(res) == 'Sat Jan 01 00:00:00 2000'
-        else:
-            assert rctime.ctime(res) == 'Sat Jan  1 00:00:00 2000'
-
-    def test_asctime(self):
-        import time as rctime
-        rctime.asctime()
-        # raises(TypeError, rctime.asctime, None)
-        raises(TypeError, rctime.asctime, ())
-        raises(TypeError, rctime.asctime, (1,))
-        raises(TypeError, rctime.asctime, range(8))
-        raises(TypeError, rctime.asctime, (1, 2))
-        raises(TypeError, rctime.asctime, (1, 2, 3, 4, 5, 6, 'f', 8, 9))
-        raises(TypeError, rctime.asctime, "foo")
-        raises(ValueError, rctime.asctime, (1900, -1, 1, 0, 0, 0, 0, 1, -1))
-        res = rctime.asctime()
-        assert isinstance(res, str)
-        rctime.asctime(rctime.localtime())
-        t = rctime.time()
-        assert rctime.ctime(t) == rctime.asctime(rctime.localtime(t))
-        if rctime.timezone:
-            assert rctime.ctime(t) != rctime.asctime(rctime.gmtime(t))
-        ltime = rctime.localtime()
-        assert rctime.asctime(tuple(ltime)) == rctime.asctime(ltime)
-        try:
-            rctime.asctime((12345,) + (0,) * 8)  # assert this doesn't crash
-        except ValueError:
-            pass  # some OS (ie POSIXes besides Linux) reject year > 9999
-
-    def test_asctime_large_year(self):
-        import time as rctime
-        assert rctime.asctime((12345,) +
-                              (0,) * 8) == 'Mon Jan  1 00:00:00 12345'
-        assert rctime.asctime((123456789,) +
-                              (0,) * 8) == 'Mon Jan  1 00:00:00 123456789'
-        sizeof_int = 4
-        bigyear = (1 << 8 * sizeof_int - 1) - 1
-        asc = rctime.asctime((bigyear, 6, 1) + (0,)*6)
-        assert asc[-len(str(bigyear)):] == str(bigyear)
-        raises(OverflowError, rctime.asctime, (bigyear + 1,) + (0,)*8)
-
-    def test_accept2dyear_access(self):
-        import time as rctime
-
-        accept2dyear = rctime.accept2dyear
-        del rctime.accept2dyear
-        try:
-            # with year >= 1900 this shouldn't need to access accept2dyear
-            assert rctime.asctime((2000,) + (0,) * 8).split()[-1] == '2000'
-        finally:
-            rctime.accept2dyear = accept2dyear
-
-    def test_accept2dyear_bad(self):
-        import time as rctime
-        class X:
-            def __bool__(self):
-                raise RuntimeError('boo')
-        orig, rctime.accept2dyear = rctime.accept2dyear, X()
-        try:
-            raises(RuntimeError, rctime.asctime, (200,)  + (0,) * 8)
-        finally:
-            rctime.accept2dyear = orig
-
-    def test_struct_time(self):
-        import time as rctime
-        raises(TypeError, rctime.struct_time)
-        raises(TypeError, rctime.struct_time, "foo")
-        raises(TypeError, rctime.struct_time, (1, 2, 3))
-        tup = (1, 2, 3, 4, 5, 6, 7, 8, 9)
-        st_time = rctime.struct_time(tup)
-        assert str(st_time).startswith('time.struct_time(tm_year=1, ')
-        assert len(st_time) == len(tup)
-
-    def test_tzset(self):
-        import time as rctime
-        import os
-
-        if not os.name == "posix":
-            skip("tzset available only under Unix")
-
-        # epoch time of midnight Dec 25th 2002. Never DST in northern
-        # hemisphere.
-        xmas2002 = 1040774400.0
-
-        # these formats are correct for 2002, and possibly future years
-        # this format is the 'standard' as documented at:
-        # http://www.opengroup.org/onlinepubs/007904975/basedefs/xbd_chap08.html
-        # They are also documented in the tzset(3) man page on most Unix

From noreply at buildbot.pypy.org  Mon Nov 17 19:06:55 2014
From: noreply at buildbot.pypy.org (pjenvey)
Date: Mon, 17 Nov 2014 19:06:55 +0100 (CET)
Subject: [pypy-commit] pypy py3k: kill rctime reference
Message-ID: <20141117180655.97B491C104D@cobra.cs.uni-duesseldorf.de>

Author: Philip Jenvey 
Branch: py3k
Changeset: r74550:5051ad28ea99
Date: 2014-11-17 10:06 -0800
http://bitbucket.org/pypy/pypy/changeset/5051ad28ea99/

Log:	kill rctime reference

diff --git a/pypy/tool/pytest/apptest.py b/pypy/tool/pytest/apptest.py
--- a/pypy/tool/pytest/apptest.py
+++ b/pypy/tool/pytest/apptest.py
@@ -24,7 +24,6 @@
 RENAMED_USEMODULES = dict(
     _winreg='winreg',
     exceptions='builtins',
-    rctime='time',
     struct='_struct',
     thread='_thread',
     )

From noreply at buildbot.pypy.org  Mon Nov 17 19:59:09 2014
From: noreply at buildbot.pypy.org (rlamy)
Date: Mon, 17 Nov 2014 19:59:09 +0100 (CET)
Subject: [pypy-commit] pypy expressions-2: Create expression object V_Type
 and use it during flowing
Message-ID: <20141117185909.28FD71C1147@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy 
Branch: expressions-2
Changeset: r74551:2bcbfeca1b48
Date: 2014-11-08 01:57 +0000
http://bitbucket.org/pypy/pypy/changeset/2bcbfeca1b48/

Log:	Create expression object V_Type and use it during flowing

diff --git a/rpython/annotator/expression.py b/rpython/annotator/expression.py
new file mode 100644
--- /dev/null
+++ b/rpython/annotator/expression.py
@@ -0,0 +1,17 @@
+from rpython.flowspace.model import Variable
+from rpython.flowspace.operation import op
+from rpython.annotator.model import SomeType
+
+class V_Type(Variable):
+    def __init__(self, v_obj):
+        Variable.__init__(self)
+        self.arg = v_obj
+        s = SomeType()
+        s.is_type_of = [v_obj]
+        self.annotation = s
+
+    def as_operation(self):
+        return op.type(self.arg)
+
+    def __eq__(self, other):
+        return isinstance(other, V_Type) and other.arg == self.arg
diff --git a/rpython/annotator/unaryop.py b/rpython/annotator/unaryop.py
--- a/rpython/annotator/unaryop.py
+++ b/rpython/annotator/unaryop.py
@@ -8,7 +8,7 @@
 from rpython.annotator.model import (SomeObject, SomeInteger, SomeBool,
     SomeString, SomeChar, SomeList, SomeDict, SomeTuple, SomeImpossibleValue,
     SomeUnicodeCodePoint, SomeInstance, SomeBuiltin, SomeBuiltinMethod,
-    SomeFloat, SomeIterator, SomePBC, SomeNone, SomeType, s_ImpossibleValue,
+    SomeFloat, SomeIterator, SomePBC, SomeNone, s_ImpossibleValue,
     s_Bool, s_None, unionof, add_knowntypedata,
     HarmlesslyBlocked, SomeWeakRef, SomeUnicodeString, SomeByteArray)
 from rpython.annotator.bookkeeper import getbookkeeper, immutablevalue
@@ -21,11 +21,9 @@
                         if oper.dispatch == 1])
 UNARY_OPERATIONS.remove('contains')
 
- at op.type.register(SomeObject)
-def type_SomeObject(annotator, arg):
-    r = SomeType()
-    r.is_type_of = [arg]
-    return r
+ at op.assign.register(SomeObject)
+def assign(annotator, v_obj):
+    return annotator.annotation(v_obj)
 
 @op.bool.register(SomeObject)
 def bool_SomeObject(annotator, obj):
diff --git a/rpython/flowspace/model.py b/rpython/flowspace/model.py
--- a/rpython/flowspace/model.py
+++ b/rpython/flowspace/model.py
@@ -248,7 +248,7 @@
 
 
 class Variable(object):
-    __slots__ = ["_name", "_nr", "annotation", "concretetype"]
+    __slots__ = ["_name", "_nr", "annotation", "concretetype", "equals"]
 
     dummyname = 'v'
     namesdict = {dummyname: (dummyname, 0)}
@@ -576,7 +576,8 @@
                 for v in op.args:
                     assert isinstance(v, (Constant, Variable))
                     if isinstance(v, Variable):
-                        usevar(v)
+                        if type(v) is Variable:
+                            usevar(v)
                     else:
                         assert v.value is not last_exception
                         #assert v.value != last_exc_value
@@ -646,14 +647,10 @@
                     assert link.last_exc_value is None
                 for v in link.args:
                     assert isinstance(v, (Constant, Variable))
-                    if isinstance(v, Variable):
+                    if type(v) is Variable:
                         usevar(v, in_link=link)
                         if exc_link:
                             assert v != block.operations[-1].result
-                    #else:
-                    #    if not exc_link:
-                    #        assert v.value is not last_exception
-                    #        #assert v.value != last_exc_value
                 allexitcases[link.exitcase] = True
             assert len(allexitcases) == len(block.exits)
             vars_previous_blocks.update(vars)
diff --git a/rpython/flowspace/operation.py b/rpython/flowspace/operation.py
--- a/rpython/flowspace/operation.py
+++ b/rpython/flowspace/operation.py
@@ -356,7 +356,7 @@
 
 add_operator('is_', 2, dispatch=2, pure=True)
 add_operator('id', 1, dispatch=1, pyfunc=id)
-add_operator('type', 1, dispatch=1, pyfunc=new_style_type, pure=True)
+#add_operator('type', 1, dispatch=1, pyfunc=new_style_type, pure=True)
 add_operator('issubtype', 2, dispatch=1, pyfunc=issubclass, pure=True)  # not for old-style classes
 add_operator('repr', 1, dispatch=1, pyfunc=repr, pure=True)
 add_operator('str', 1, dispatch=1, pyfunc=str, pure=True)
@@ -431,6 +431,7 @@
 add_operator('yield_', 1)
 add_operator('newslice', 3)
 add_operator('hint', None, dispatch=1)
+add_operator('assign', 1, dispatch=1)
 
 class Contains(SingleDispatchMixin, PureOperation):
     opname = 'contains'
@@ -442,6 +443,21 @@
     def get_specialization(cls, s_seq, s_elem):
         return cls._dispatch(type(s_seq))
 
+class Type(SingleDispatchMixin, PureOperation):
+    opname = 'type'
+    arity = 1
+    canraise = []
+    pyfunc = staticmethod(new_style_type)
+
+    def eval(self, ctx):
+        result = self.constfold()
+        if result is not None:
+            return result
+        from rpython.annotator.expression import V_Type
+        v_instance, = self.args
+        result = V_Type(v_instance)
+        return ctx.do_op(op.assign(result))
+
 
 class NewDict(HLOperation):
     opname = 'newdict'
diff --git a/rpython/translator/transform.py b/rpython/translator/transform.py
--- a/rpython/translator/transform.py
+++ b/rpython/translator/transform.py
@@ -134,6 +134,16 @@
                     s_dict = self.annotation(op.args[0])
                     s_dict.dictdef.generalize_key(self.binding(op.args[1]))
 
+def remove_assign(ann, block_subset):
+    for block in block_subset:
+        for i in range(len(block.operations)):
+            op = block.operations[i]
+            if op.opname == 'assign':
+                new_op = op.args[0].as_operation()
+                new_op.result = op.result
+                block.operations[i] = new_op
+
+
 
 def transform_dead_op_vars(self, block_subset):
     # we redo the same simplification from simplify.py,
@@ -249,6 +259,7 @@
     transform_extend_with_str_slice,
     transform_extend_with_char_count,
     transform_list_contains,
+    remove_assign,
     ]
 
 def transform_graph(ann, extra_passes=None, block_subset=None):

From noreply at buildbot.pypy.org  Mon Nov 17 19:59:10 2014
From: noreply at buildbot.pypy.org (rlamy)
Date: Mon, 17 Nov 2014 19:59:10 +0100 (CET)
Subject: [pypy-commit] pypy expressions-2: apply variable renaming to V_Type
Message-ID: <20141117185910.6F1B91C1147@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy 
Branch: expressions-2
Changeset: r74552:51bd29ba916d
Date: 2014-11-08 18:31 +0000
http://bitbucket.org/pypy/pypy/changeset/51bd29ba916d/

Log:	apply variable renaming to V_Type

diff --git a/rpython/annotator/expression.py b/rpython/annotator/expression.py
--- a/rpython/annotator/expression.py
+++ b/rpython/annotator/expression.py
@@ -15,3 +15,9 @@
 
     def __eq__(self, other):
         return isinstance(other, V_Type) and other.arg == self.arg
+
+    def replace(self, mapping):
+        if self.arg in mapping:
+            return V_Type(mapping[self.arg])
+        else:
+            return self
diff --git a/rpython/flowspace/flowcontext.py b/rpython/flowspace/flowcontext.py
--- a/rpython/flowspace/flowcontext.py
+++ b/rpython/flowspace/flowcontext.py
@@ -440,7 +440,7 @@
             msg = "implicit %s shouldn't occur" % exc_cls.__name__
             w_type = Constant(AssertionError)
             w_value = Constant(AssertionError(msg))
-            link = Link([w_type, w_value], self.graph.exceptblock)
+            link = self.graph.new_raise_link(w_value, w_type)
             self.recorder.crnt_block.closeblock(link)
 
         except Raise as e:
@@ -448,7 +448,7 @@
             if w_exc.w_type == const(ImportError):
                 msg = 'import statement always raises %s' % e
                 raise ImportError(msg)
-            link = Link([w_exc.w_type, w_exc.w_value], self.graph.exceptblock)
+            link = self.graph.new_raise_link(w_exc.w_value, w_exc.w_type)
             self.recorder.crnt_block.closeblock(link)
 
         except StopFlowing:
diff --git a/rpython/flowspace/model.py b/rpython/flowspace/model.py
--- a/rpython/flowspace/model.py
+++ b/rpython/flowspace/model.py
@@ -31,6 +31,9 @@
     def getreturnvar(self):
         return self.returnblock.inputargs[0]
 
+    def new_raise_link(self, v_exception, v_exc_type):
+        return Link([v_exc_type, v_exception], self.exceptblock)
+
     @property
     def source(self):
         if hasattr(self, "_source"):
@@ -206,13 +209,12 @@
         return uniqueitems([w for w in result if isinstance(w, Constant)])
 
     def renamevariables(self, mapping):
-        self.inputargs = [mapping.get(a, a) for a in self.inputargs]
-        for op in self.operations:
-            op.args = [mapping.get(a, a) for a in op.args]
-            op.result = mapping.get(op.result, op.result)
-        self.exitswitch = mapping.get(self.exitswitch, self.exitswitch)
+        self.inputargs = [a.replace(mapping) for a in self.inputargs]
+        self.operations = [op.replace(mapping) for op in self.operations]
+        if self.exitswitch is not None:
+            self.exitswitch = self.exitswitch.replace(mapping)
         for link in self.exits:
-            link.args = [mapping.get(a, a) for a in link.args]
+            link.args = [a.replace(mapping) for a in link.args]
 
     def closeblock(self, *exits):
         assert self.exits == [], "block already closed"
@@ -318,6 +320,8 @@
             newvar.concretetype = self.concretetype
         return newvar
 
+    def replace(self, mapping):
+        return mapping.get(self, self)
 
 
 class Constant(Hashable):
@@ -347,6 +351,9 @@
             # cannot count on it not mutating at runtime!
             return False
 
+    def replace(self, mapping):
+        return self
+
 
 class FSException(object):
     def __init__(self, w_type, w_value):
@@ -422,8 +429,8 @@
                                 ", ".join(map(repr, self.args)))
 
     def replace(self, mapping):
-        newargs = [mapping.get(arg, arg) for arg in self.args]
-        newresult = mapping.get(self.result, self.result)
+        newargs = [arg.replace(mapping) for arg in self.args]
+        newresult = self.result.replace(mapping)
         return type(self)(self.opname, newargs, newresult, self.offset)
 
 class Atom(object):
@@ -578,6 +585,8 @@
                     if isinstance(v, Variable):
                         if type(v) is Variable:
                             usevar(v)
+                        else:
+                            usevar(v.arg)
                     else:
                         assert v.value is not last_exception
                         #assert v.value != last_exc_value
diff --git a/rpython/flowspace/operation.py b/rpython/flowspace/operation.py
--- a/rpython/flowspace/operation.py
+++ b/rpython/flowspace/operation.py
@@ -74,8 +74,8 @@
         self.offset = -1
 
     def replace(self, mapping):
-        newargs = [mapping.get(arg, arg) for arg in self.args]
-        newresult = mapping.get(self.result, self.result)
+        newargs = [arg.replace(mapping) for arg in self.args]
+        newresult = self.result.replace(mapping)
         newop = type(self)(*newargs)
         newop.result = newresult
         newop.offset = self.offset
diff --git a/rpython/translator/simplify.py b/rpython/translator/simplify.py
--- a/rpython/translator/simplify.py
+++ b/rpython/translator/simplify.py
@@ -629,7 +629,7 @@
         assert len(links) == len(new_args)
         for link, args in zip(links, new_args):
             link.args = args
-    for block in graph.iterblocks():
+    for block in entrymap:
         block.renamevariables(renaming)
 
 def remove_identical_vars(graph):

From noreply at buildbot.pypy.org  Mon Nov 17 19:59:11 2014
From: noreply at buildbot.pypy.org (rlamy)
Date: Mon, 17 Nov 2014 19:59:11 +0100 (CET)
Subject: [pypy-commit] pypy expressions-2: Allow expressions to tell which
 variables they depend on
Message-ID: <20141117185911.B6E4E1C1147@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy 
Branch: expressions-2
Changeset: r74553:0642cf37c18a
Date: 2014-11-09 01:11 +0000
http://bitbucket.org/pypy/pypy/changeset/0642cf37c18a/

Log:	Allow expressions to tell which variables they depend on

diff --git a/rpython/annotator/expression.py b/rpython/annotator/expression.py
--- a/rpython/annotator/expression.py
+++ b/rpython/annotator/expression.py
@@ -21,3 +21,7 @@
             return V_Type(mapping[self.arg])
         else:
             return self
+
+    @property
+    def dependencies(self):
+        return self.arg.dependencies
diff --git a/rpython/flowspace/model.py b/rpython/flowspace/model.py
--- a/rpython/flowspace/model.py
+++ b/rpython/flowspace/model.py
@@ -323,6 +323,10 @@
     def replace(self, mapping):
         return mapping.get(self, self)
 
+    @property
+    def dependencies(self):
+        return set([self])
+
 
 class Constant(Hashable):
     __slots__ = ["concretetype"]
@@ -354,6 +358,10 @@
     def replace(self, mapping):
         return self
 
+    @property
+    def dependencies(self):
+        return set()
+
 
 class FSException(object):
     def __init__(self, w_type, w_value):
diff --git a/rpython/translator/backendopt/ssa.py b/rpython/translator/backendopt/ssa.py
--- a/rpython/translator/backendopt/ssa.py
+++ b/rpython/translator/backendopt/ssa.py
@@ -151,21 +151,23 @@
         variables_created = variables_created_in(block)
         seen = set(variables_created)
         variables_used = []
-        def record_used_var(v):
-            if v not in seen:
-                variables_used.append(v)
-                seen.add(v)
+        def record_dependencies(arg):
+            if arg is None:
+                return
+            for v in arg.dependencies:
+                if v not in seen:
+                    variables_used.append(v)
+                    seen.add(v)
         for op in block.operations:
             for arg in op.args:
-                record_used_var(arg)
-        record_used_var(block.exitswitch)
+                record_dependencies(arg)
+        record_dependencies(block.exitswitch)
         for link in block.exits:
             for arg in link.args:
-                record_used_var(arg)
+                record_dependencies(arg)
 
         for v in variables_used:
-            if (isinstance(v, Variable) and
-                    v._name not in ('last_exception_', 'last_exc_value_')):
+            if (v._name not in ('last_exception_', 'last_exc_value_')):
                 pending.append((block, v))
 
     while pending:
diff --git a/rpython/translator/simplify.py b/rpython/translator/simplify.py
--- a/rpython/translator/simplify.py
+++ b/rpython/translator/simplify.py
@@ -477,9 +477,11 @@
         # figure out which variables are ever read
         for op in block.operations:
             if not canremove(op, block):   # the inputs are always needed
-                read_vars.update(op.args)
+                for arg in op.args:
+                    read_vars.update(arg.dependencies)
             else:
-                dependencies[op.result].update(op.args)
+                for arg in op.args:
+                    dependencies[op.result].update(arg.dependencies)
 
         if isinstance(block.exitswitch, Variable):
             read_vars.add(block.exitswitch)

From noreply at buildbot.pypy.org  Mon Nov 17 19:59:12 2014
From: noreply at buildbot.pypy.org (rlamy)
Date: Mon, 17 Nov 2014 19:59:12 +0100 (CET)
Subject: [pypy-commit] pypy expressions-2: op.assign is removable
Message-ID: <20141117185912.F34D31C1147@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy 
Branch: expressions-2
Changeset: r74554:a1073aa369c8
Date: 2014-11-17 03:02 +0000
http://bitbucket.org/pypy/pypy/changeset/a1073aa369c8/

Log:	op.assign is removable

diff --git a/rpython/translator/simplify.py b/rpython/translator/simplify.py
--- a/rpython/translator/simplify.py
+++ b/rpython/translator/simplify.py
@@ -434,7 +434,7 @@
 # (they have no side effects, at least in R-Python)
 CanRemove = {}
 for _op in '''
-        newtuple newlist newdict bool
+        newtuple newlist newdict bool assign
         is_ id type issubtype repr str len hash getattr getitem
         pos neg abs hex oct ord invert add sub mul
         truediv floordiv div mod divmod pow lshift rshift and_ or_

From noreply at buildbot.pypy.org  Mon Nov 17 19:59:14 2014
From: noreply at buildbot.pypy.org (rlamy)
Date: Mon, 17 Nov 2014 19:59:14 +0100 (CET)
Subject: [pypy-commit] pypy expressions-2: fix generators
Message-ID: <20141117185914.4C8A41C1147@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy 
Branch: expressions-2
Changeset: r74555:02c004e892a3
Date: 2014-11-09 01:48 +0000
http://bitbucket.org/pypy/pypy/changeset/02c004e892a3/

Log:	fix generators

diff --git a/rpython/flowspace/generator.py b/rpython/flowspace/generator.py
--- a/rpython/flowspace/generator.py
+++ b/rpython/flowspace/generator.py
@@ -106,6 +106,7 @@
 def tweak_generator_body_graph(Entry, graph):
     # First, always run simplify_graph in order to reduce the number of
     # variables passed around
+    from rpython.annotator.expression import V_Type
     simplify_graph(graph)
     insert_empty_startblock(None, graph)
     _insert_reads(graph.startblock, Entry.varnames)
@@ -115,7 +116,7 @@
     #
     stopblock = Block([])
     op0 = op.simple_call(const(StopIteration))
-    op1 = op.type(op0.result)
+    op1 = op.assign(V_Type(op0.result))
     stopblock.operations = [op0, op1]
     stopblock.closeblock(Link([op1.result, op0.result], graph.exceptblock))
     #

From noreply at buildbot.pypy.org  Mon Nov 17 19:59:15 2014
From: noreply at buildbot.pypy.org (rlamy)
Date: Mon, 17 Nov 2014 19:59:15 +0100 (CET)
Subject: [pypy-commit] pypy expressions-2: fix checkgraph()
Message-ID: <20141117185915.8B0971C1147@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy 
Branch: expressions-2
Changeset: r74556:42077fcf55a4
Date: 2014-11-09 03:04 +0000
http://bitbucket.org/pypy/pypy/changeset/42077fcf55a4/

Log:	fix checkgraph()

diff --git a/rpython/flowspace/model.py b/rpython/flowspace/model.py
--- a/rpython/flowspace/model.py
+++ b/rpython/flowspace/model.py
@@ -570,10 +570,11 @@
                 "variable %r used in more than one block" % (v,))
             vars[v] = only_in_link
 
-        def usevar(v, in_link=None):
-            assert v in vars
-            if in_link is not None:
-                assert vars[v] is None or vars[v] is in_link
+        def usevar(var, in_link=None):
+            for v in var.dependencies:
+                assert v in vars
+                if in_link is not None:
+                    assert vars[v] is None or vars[v] is in_link
 
 
         for block in graph.iterblocks():
@@ -591,10 +592,7 @@
                 for v in op.args:
                     assert isinstance(v, (Constant, Variable))
                     if isinstance(v, Variable):
-                        if type(v) is Variable:
-                            usevar(v)
-                        else:
-                            usevar(v.arg)
+                        usevar(v)
                     else:
                         assert v.value is not last_exception
                         #assert v.value != last_exc_value
@@ -664,7 +662,7 @@
                     assert link.last_exc_value is None
                 for v in link.args:
                     assert isinstance(v, (Constant, Variable))
-                    if type(v) is Variable:
+                    if isinstance(v, Variable):
                         usevar(v, in_link=link)
                         if exc_link:
                             assert v != block.operations[-1].result

From noreply at buildbot.pypy.org  Mon Nov 17 19:59:16 2014
From: noreply at buildbot.pypy.org (rlamy)
Date: Mon, 17 Nov 2014 19:59:16 +0100 (CET)
Subject: [pypy-commit] pypy expressions-2: Make sure that inputargs contains
	only Variables
Message-ID: <20141117185916.C10F71C1147@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy 
Branch: expressions-2
Changeset: r74557:214a3579014f
Date: 2014-11-09 05:27 +0000
http://bitbucket.org/pypy/pypy/changeset/214a3579014f/

Log:	Make sure that inputargs contains only Variables

diff --git a/rpython/flowspace/model.py b/rpython/flowspace/model.py
--- a/rpython/flowspace/model.py
+++ b/rpython/flowspace/model.py
@@ -163,7 +163,7 @@
                 exits blockcolor""".split()
 
     def __init__(self, inputargs):
-        self.inputargs = list(inputargs)  # mixed list of variable/const XXX
+        self.inputargs = list(inputargs)  # list of Variable
         self.operations = []              # list of SpaceOperation(s)
         self.exitswitch = None            # a variable or
                                           #  Constant(last_exception), see below
@@ -197,7 +197,8 @@
         "Return all variables mentioned in this Block."
         result = self.inputargs[:]
         for op in self.operations:
-            result += op.args
+            for arg in op.args:
+                result.extend(arg.dependencies)
             result.append(op.result)
         return uniqueitems([w for w in result if isinstance(w, Variable)])
 
@@ -564,7 +565,7 @@
             assert block.exits == ()
 
         def definevar(v, only_in_link=None):
-            assert isinstance(v, Variable)
+            assert type(v) is Variable
             assert v not in vars, "duplicate variable %r" % (v,)
             assert v not in vars_previous_blocks, (
                 "variable %r used in more than one block" % (v,))

From noreply at buildbot.pypy.org  Mon Nov 17 19:59:17 2014
From: noreply at buildbot.pypy.org (rlamy)
Date: Mon, 17 Nov 2014 19:59:17 +0100 (CET)
Subject: [pypy-commit] pypy expressions-2: create ExceptBlock class
Message-ID: <20141117185917.E7E4E1C1147@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy 
Branch: expressions-2
Changeset: r74558:125a6ee9aeb3
Date: 2014-11-10 01:23 +0000
http://bitbucket.org/pypy/pypy/changeset/125a6ee9aeb3/

Log:	create ExceptBlock class

diff --git a/rpython/flowspace/model.py b/rpython/flowspace/model.py
--- a/rpython/flowspace/model.py
+++ b/rpython/flowspace/model.py
@@ -18,11 +18,7 @@
         self.returnblock = Block([return_var or Variable()])
         self.returnblock.operations = ()
         self.returnblock.exits = ()
-        # block corresponding to exception results
-        self.exceptblock = Block([Variable('etype'),   # exception class
-                                  Variable('evalue')])  # exception value
-        self.exceptblock.operations = ()
-        self.exceptblock.exits = ()
+        self.exceptblock = ExceptBlock()
         self.tag = None
 
     def getargs(self):
@@ -181,8 +177,6 @@
         else:
             if (not self.exits) and len(self.inputargs) == 1:
                 txt = "return block"
-            elif (not self.exits) and len(self.inputargs) == 2:
-                txt = "raise block"
             else:
                 txt = "codeless block"
         return txt
@@ -249,6 +243,17 @@
 
     view = show
 
+class ExceptBlock(Block):
+    def __init__(self):
+        self.inputargs = [Variable('etype'),   # exception class
+                          Variable('evalue')]  # exception value
+        self.operations = ()
+        self.exits = ()
+        self.exitswitch = None
+
+    def __str__(self):
+        return "raise block"
+
 
 class Variable(object):
     __slots__ = ["_name", "_nr", "annotation", "concretetype", "equals"]

From noreply at buildbot.pypy.org  Mon Nov 17 19:59:19 2014
From: noreply at buildbot.pypy.org (rlamy)
Date: Mon, 17 Nov 2014 19:59:19 +0100 (CET)
Subject: [pypy-commit] pypy expressions-2: refactor join_blocks()
Message-ID: <20141117185919.1F8DC1C1147@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy 
Branch: expressions-2
Changeset: r74559:e9895438b860
Date: 2014-11-10 18:51 +0000
http://bitbucket.org/pypy/pypy/changeset/e9895438b860/

Log:	refactor join_blocks()

diff --git a/rpython/translator/simplify.py b/rpython/translator/simplify.py
--- a/rpython/translator/simplify.py
+++ b/rpython/translator/simplify.py
@@ -65,28 +65,19 @@
     When this happens, we need to replace the preceeding link with the
     following link.  Arguments of the links should be updated."""
     for link in list(graph.iterlinks()):
-            while not link.target.operations:
-                block1 = link.target
-                if block1.exitswitch is not None:
-                    break
-                if not block1.exits:
-                    break
-                exit = block1.exits[0]
-                assert block1 is not exit.target, (
-                    "the graph contains an empty infinite loop")
-                outputargs = []
-                for v in exit.args:
-                    if isinstance(v, Variable):
-                        try:
-                            i = block1.inputargs.index(v)
-                            v = link.args[i]
-                        except ValueError:
-                            # the variable was passed implicitly to block1
-                            pass
-                    outputargs.append(v)
-                link.args = outputargs
-                link.target = exit.target
-                # the while loop above will simplify recursively the new link
+        while not link.target.operations:
+            block1 = link.target
+            if block1.exitswitch is not None:
+                break
+            if not block1.exits:
+                break
+            exit = block1.exits[0]
+            assert block1 is not exit.target, (
+                "the graph contains an empty infinite loop")
+            subst = dict(zip(block1.inputargs, link.args))
+            link.args = [v.replace(subst) for v in exit.args]
+            link.target = exit.target
+            # the while loop above will simplify recursively the new link
 
 def transform_ovfcheck(graph):
     """The special function calls ovfcheck needs to

From noreply at buildbot.pypy.org  Mon Nov 17 19:59:20 2014
From: noreply at buildbot.pypy.org (rlamy)
Date: Mon, 17 Nov 2014 19:59:20 +0100 (CET)
Subject: [pypy-commit] pypy expressions-2: remove op.assign hack
Message-ID: <20141117185920.635761C1147@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy 
Branch: expressions-2
Changeset: r74560:4f0ff483dbfe
Date: 2014-11-11 00:51 +0000
http://bitbucket.org/pypy/pypy/changeset/4f0ff483dbfe/

Log:	remove op.assign hack

diff --git a/rpython/annotator/unaryop.py b/rpython/annotator/unaryop.py
--- a/rpython/annotator/unaryop.py
+++ b/rpython/annotator/unaryop.py
@@ -21,10 +21,6 @@
                         if oper.dispatch == 1])
 UNARY_OPERATIONS.remove('contains')
 
- at op.assign.register(SomeObject)
-def assign(annotator, v_obj):
-    return annotator.annotation(v_obj)
-
 @op.bool.register(SomeObject)
 def bool_SomeObject(annotator, obj):
     r = SomeBool()
diff --git a/rpython/flowspace/flowcontext.py b/rpython/flowspace/flowcontext.py
--- a/rpython/flowspace/flowcontext.py
+++ b/rpython/flowspace/flowcontext.py
@@ -388,15 +388,19 @@
             return w_condition.value
         return self.recorder.guessbool(self, w_condition)
 
-    def record(self, spaceop):
+    def merge_point(self):
         recorder = self.recorder
         if getattr(recorder, 'final_state', None) is not None:
             self.mergeblock(recorder.crnt_block, recorder.final_state)
             raise StopFlowing
+
+    def record(self, spaceop):
+        recorder = self.recorder
         spaceop.offset = self.last_instr
         recorder.append(spaceop)
 
     def do_op(self, op):
+        self.merge_point()
         self.record(op)
         self.guessexception(op.canraise)
         return op.result
diff --git a/rpython/flowspace/generator.py b/rpython/flowspace/generator.py
--- a/rpython/flowspace/generator.py
+++ b/rpython/flowspace/generator.py
@@ -116,9 +116,8 @@
     #
     stopblock = Block([])
     op0 = op.simple_call(const(StopIteration))
-    op1 = op.assign(V_Type(op0.result))
-    stopblock.operations = [op0, op1]
-    stopblock.closeblock(Link([op1.result, op0.result], graph.exceptblock))
+    stopblock.operations = [op0]
+    stopblock.closeblock(Link([V_Type(op0.result), op0.result], graph.exceptblock))
     #
     for block in list(graph.iterblocks()):
         for exit in block.exits:
diff --git a/rpython/flowspace/operation.py b/rpython/flowspace/operation.py
--- a/rpython/flowspace/operation.py
+++ b/rpython/flowspace/operation.py
@@ -431,7 +431,6 @@
 add_operator('yield_', 1)
 add_operator('newslice', 3)
 add_operator('hint', None, dispatch=1)
-add_operator('assign', 1, dispatch=1)
 
 class Contains(SingleDispatchMixin, PureOperation):
     opname = 'contains'
@@ -453,10 +452,10 @@
         result = self.constfold()
         if result is not None:
             return result
+        ctx.merge_point()
         from rpython.annotator.expression import V_Type
         v_instance, = self.args
-        result = V_Type(v_instance)
-        return ctx.do_op(op.assign(result))
+        return V_Type(v_instance)
 
 
 class NewDict(HLOperation):
diff --git a/rpython/translator/simplify.py b/rpython/translator/simplify.py
--- a/rpython/translator/simplify.py
+++ b/rpython/translator/simplify.py
@@ -11,6 +11,7 @@
 from rpython.flowspace.model import (Variable, Constant,
                                      c_last_exception, checkgraph, mkentrymap)
 from rpython.flowspace.operation import OverflowingOperation, op
+from rpython.annotator.expression import V_Type
 from rpython.rlib import rarithmetic
 from rpython.translator import unsimplify
 from rpython.rtyper.lltypesystem import lloperation, lltype
@@ -72,6 +73,8 @@
             if not block1.exits:
                 break
             exit = block1.exits[0]
+            if any(isinstance(arg, V_Type) for arg in exit.args):
+                break
             assert block1 is not exit.target, (
                 "the graph contains an empty infinite loop")
             subst = dict(zip(block1.inputargs, link.args))
@@ -311,7 +314,8 @@
             for vprev, vtarg in zip(link.args, link.target.inputargs):
                 renaming[vtarg] = vprev
             def rename(v):
-                return renaming.get(v, v)
+                if v is not None:
+                    return v.replace(renaming)
             def rename_op(op):
                 op = op.replace(renaming)
                 # special case...
@@ -425,7 +429,7 @@
 # (they have no side effects, at least in R-Python)
 CanRemove = {}
 for _op in '''
-        newtuple newlist newdict bool assign
+        newtuple newlist newdict bool
         is_ id type issubtype repr str len hash getattr getitem
         pos neg abs hex oct ord invert add sub mul
         truediv floordiv div mod divmod pow lshift rshift and_ or_
diff --git a/rpython/translator/transform.py b/rpython/translator/transform.py
--- a/rpython/translator/transform.py
+++ b/rpython/translator/transform.py
@@ -8,6 +8,7 @@
 from rpython.flowspace.model import SpaceOperation
 from rpython.flowspace.model import Variable, Constant, Link
 from rpython.flowspace.model import c_last_exception, checkgraph
+from rpython.annotator.expression import V_Type
 from rpython.annotator import model as annmodel
 from rpython.rtyper.lltypesystem import lltype
 
@@ -134,15 +135,43 @@
                     s_dict = self.annotation(op.args[0])
                     s_dict.dictdef.generalize_key(self.binding(op.args[1]))
 
-def remove_assign(ann, block_subset):
+def remove_expr(ann, block_subset):
     for block in block_subset:
-        for i in range(len(block.operations)):
+        i = 0
+        while i < len(block.operations):
             op = block.operations[i]
-            if op.opname == 'assign':
-                new_op = op.args[0].as_operation()
-                new_op.result = op.result
-                block.operations[i] = new_op
-
+            if any(isinstance(arg, V_Type) for arg in op.args):
+                new_args = []
+                new_ops = []
+                for arg in op.args:
+                    if isinstance(arg, V_Type):
+                        new_op = arg.as_operation()
+                        new_op.result.annotation = arg.annotation
+                        new_ops.append(new_op)
+                        new_args.append(new_op.result)
+                    else:
+                        new_args.append(arg)
+                op.args = new_args
+                block.operations[i:i] = new_ops
+            else:
+                i += 1
+        for exit in block.exits:
+            if any(isinstance(arg, V_Type) for arg in exit.args):
+                new_args = []
+                new_ops = []
+                for arg in exit.args:
+                    if isinstance(arg, V_Type):
+                        new_op = arg.as_operation()
+                        new_op.result.annotation = arg.annotation
+                        new_ops.append(new_op)
+                        new_args.append(new_op.result)
+                    else:
+                        new_args.append(arg)
+                exit.args = new_args
+                if block.exitswitch == c_last_exception:
+                    block.operations[-1:-1] = new_ops
+                else:
+                    block.operations.extend(new_ops)
 
 
 def transform_dead_op_vars(self, block_subset):
@@ -259,7 +288,7 @@
     transform_extend_with_str_slice,
     transform_extend_with_char_count,
     transform_list_contains,
-    remove_assign,
+    remove_expr,
     ]
 
 def transform_graph(ann, extra_passes=None, block_subset=None):

From noreply at buildbot.pypy.org  Mon Nov 17 19:59:21 2014
From: noreply at buildbot.pypy.org (rlamy)
Date: Mon, 17 Nov 2014 19:59:21 +0100 (CET)
Subject: [pypy-commit] pypy expressions-2: move V_Type to flowspace/
Message-ID: <20141117185921.A481B1C1147@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy 
Branch: expressions-2
Changeset: r74561:929d4201c3b7
Date: 2014-11-11 19:53 +0000
http://bitbucket.org/pypy/pypy/changeset/929d4201c3b7/

Log:	move V_Type to flowspace/

diff --git a/rpython/annotator/expression.py b/rpython/flowspace/expression.py
rename from rpython/annotator/expression.py
rename to rpython/flowspace/expression.py
diff --git a/rpython/flowspace/generator.py b/rpython/flowspace/generator.py
--- a/rpython/flowspace/generator.py
+++ b/rpython/flowspace/generator.py
@@ -106,7 +106,7 @@
 def tweak_generator_body_graph(Entry, graph):
     # First, always run simplify_graph in order to reduce the number of
     # variables passed around
-    from rpython.annotator.expression import V_Type
+    from rpython.flowspace.expression import V_Type
     simplify_graph(graph)
     insert_empty_startblock(None, graph)
     _insert_reads(graph.startblock, Entry.varnames)
diff --git a/rpython/flowspace/operation.py b/rpython/flowspace/operation.py
--- a/rpython/flowspace/operation.py
+++ b/rpython/flowspace/operation.py
@@ -453,7 +453,7 @@
         if result is not None:
             return result
         ctx.merge_point()
-        from rpython.annotator.expression import V_Type
+        from rpython.flowspace.expression import V_Type
         v_instance, = self.args
         return V_Type(v_instance)
 
diff --git a/rpython/translator/simplify.py b/rpython/translator/simplify.py
--- a/rpython/translator/simplify.py
+++ b/rpython/translator/simplify.py
@@ -11,7 +11,7 @@
 from rpython.flowspace.model import (Variable, Constant,
                                      c_last_exception, checkgraph, mkentrymap)
 from rpython.flowspace.operation import OverflowingOperation, op
-from rpython.annotator.expression import V_Type
+from rpython.flowspace.expression import V_Type
 from rpython.rlib import rarithmetic
 from rpython.translator import unsimplify
 from rpython.rtyper.lltypesystem import lloperation, lltype
diff --git a/rpython/translator/transform.py b/rpython/translator/transform.py
--- a/rpython/translator/transform.py
+++ b/rpython/translator/transform.py
@@ -8,7 +8,7 @@
 from rpython.flowspace.model import SpaceOperation
 from rpython.flowspace.model import Variable, Constant, Link
 from rpython.flowspace.model import c_last_exception, checkgraph
-from rpython.annotator.expression import V_Type
+from rpython.flowspace.expression import V_Type
 from rpython.annotator import model as annmodel
 from rpython.rtyper.lltypesystem import lltype
 

From noreply at buildbot.pypy.org  Mon Nov 17 19:59:22 2014
From: noreply at buildbot.pypy.org (rlamy)
Date: Mon, 17 Nov 2014 19:59:22 +0100 (CET)
Subject: [pypy-commit] pypy expressions-2: Clean up V_Type code, add tests
Message-ID: <20141117185922.DF2531C1147@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy 
Branch: expressions-2
Changeset: r74562:45cea07d4528
Date: 2014-11-12 00:32 +0000
http://bitbucket.org/pypy/pypy/changeset/45cea07d4528/

Log:	Clean up V_Type code, add tests

diff --git a/rpython/flowspace/expression.py b/rpython/flowspace/expression.py
--- a/rpython/flowspace/expression.py
+++ b/rpython/flowspace/expression.py
@@ -1,9 +1,8 @@
 from rpython.flowspace.model import Variable
-from rpython.flowspace.operation import op
-from rpython.annotator.model import SomeType
 
 class V_Type(Variable):
     def __init__(self, v_obj):
+        from rpython.annotator.model import SomeType
         Variable.__init__(self)
         self.arg = v_obj
         s = SomeType()
@@ -11,11 +10,18 @@
         self.annotation = s
 
     def as_operation(self):
+        from rpython.flowspace.operation import op
         return op.type(self.arg)
 
     def __eq__(self, other):
         return isinstance(other, V_Type) and other.arg == self.arg
 
+    def __hash__(self):
+        return hash((type(self), self.arg))
+
+    def __repr__(self):
+        return 'type(%s)' % self.arg
+
     def replace(self, mapping):
         if self.arg in mapping:
             return V_Type(mapping[self.arg])
diff --git a/rpython/flowspace/generator.py b/rpython/flowspace/generator.py
--- a/rpython/flowspace/generator.py
+++ b/rpython/flowspace/generator.py
@@ -5,6 +5,7 @@
 from rpython.flowspace.pygraph import PyGraph
 from rpython.flowspace.model import (Block, Link, Variable,
     Constant, checkgraph, const)
+from rpython.flowspace.expression import V_Type
 from rpython.flowspace.operation import op
 from rpython.translator.unsimplify import insert_empty_startblock, split_block
 from rpython.translator.simplify import eliminate_empty_blocks, simplify_graph
@@ -106,7 +107,6 @@
 def tweak_generator_body_graph(Entry, graph):
     # First, always run simplify_graph in order to reduce the number of
     # variables passed around
-    from rpython.flowspace.expression import V_Type
     simplify_graph(graph)
     insert_empty_startblock(None, graph)
     _insert_reads(graph.startblock, Entry.varnames)
diff --git a/rpython/flowspace/operation.py b/rpython/flowspace/operation.py
--- a/rpython/flowspace/operation.py
+++ b/rpython/flowspace/operation.py
@@ -12,6 +12,7 @@
 from rpython.tool.sourcetools import compile2
 from rpython.flowspace.model import (Constant, WrapException, const, Variable,
                                      SpaceOperation)
+from rpython.flowspace.expression import V_Type
 from rpython.flowspace.specialcase import register_flow_sc
 from rpython.annotator.model import (
     SomeTuple, AnnotatorError, read_can_only_throw)
@@ -453,7 +454,6 @@
         if result is not None:
             return result
         ctx.merge_point()
-        from rpython.flowspace.expression import V_Type
         v_instance, = self.args
         return V_Type(v_instance)
 
diff --git a/rpython/flowspace/test/test_expression.py b/rpython/flowspace/test/test_expression.py
new file mode 100644
--- /dev/null
+++ b/rpython/flowspace/test/test_expression.py
@@ -0,0 +1,15 @@
+from rpython.flowspace.model import Variable, const
+from rpython.flowspace.expression import V_Type
+
+def test_type():
+    v1, v2 = Variable(), Variable()
+    assert V_Type(v1) == V_Type(v1)
+    assert V_Type(v1) != V_Type(v2)
+    assert hash(V_Type(v1)) == hash(V_Type(v1))
+    assert hash(V_Type(v1)) != hash(V_Type(v2))
+
+def test_type_replace():
+    v1, v2 = Variable(), Variable()
+    assert V_Type(v1).replace({v1: v2}) == V_Type(v2)
+    assert V_Type(const(1)).replace({v1: v2}) == V_Type(const(1))
+    assert V_Type(v1).replace({v2: v1}) == V_Type(v1)

From noreply at buildbot.pypy.org  Mon Nov 17 19:59:24 2014
From: noreply at buildbot.pypy.org (rlamy)
Date: Mon, 17 Nov 2014 19:59:24 +0100 (CET)
Subject: [pypy-commit] pypy expressions-2: fix simplify.py to handle
	expressions
Message-ID: <20141117185924.2C0CB1C1147@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy 
Branch: expressions-2
Changeset: r74563:df2f684b9b71
Date: 2014-11-17 18:49 +0000
http://bitbucket.org/pypy/pypy/changeset/df2f684b9b71/

Log:	fix simplify.py to handle expressions

diff --git a/rpython/translator/simplify.py b/rpython/translator/simplify.py
--- a/rpython/translator/simplify.py
+++ b/rpython/translator/simplify.py
@@ -485,11 +485,11 @@
             for link in block.exits:
                 if link.target not in set_of_blocks:
                     for arg, targetarg in zip(link.args, link.target.inputargs):
-                        read_vars.add(arg)
+                        read_vars.update(arg.dependencies)
                         read_vars.add(targetarg)
                 else:
                     for arg, targetarg in zip(link.args, link.target.inputargs):
-                        dependencies[targetarg].add(arg)
+                        dependencies[targetarg].update(arg.dependencies)
         else:
             # return and except blocks implicitely use their input variable(s)
             for arg in block.inputargs:
@@ -591,7 +591,18 @@
         phis = inputs[block]
         to_remove = []
         unique_phis = {}
-        for i, (input, phi_args) in enumerate(phis):
+        i = 0
+        while i < len(phis):
+            input, phi_args = phis[i]
+            if all(isinstance(arg, V_Type) for arg in phi_args):
+                # replace phi(type(v1), type(v2)) with type(phi(v1, v2))
+                new_args = [arg.arg for arg in phi_args]
+                new_v = Variable()
+                uf.union(V_Type(new_v), input)
+                to_remove.append(i)
+                phis.append((new_v, new_args))
+                i += 1
+                continue
             new_args = [uf.find_rep(arg) for arg in phi_args]
             if all_equal(new_args) and not isspecialvar(new_args[0]):
                 uf.union(new_args[0], input)
@@ -603,6 +614,7 @@
                     to_remove.append(i)
                 else:
                     unique_phis[t] = input
+            i += 1
         for i in reversed(to_remove):
             del phis[i]
         return bool(to_remove)
@@ -614,7 +626,15 @@
             if simplify_phis(block):
                 progress = True
 
+    # if uf is {v1: type(v2}, v2: v3}, we need to add {type(v2): type(v3)}
+    for family in uf.infos():
+        v = family.rep
+        while isinstance(v, V_Type) and v.arg in uf:
+            new_v = V_Type(uf[v.arg].rep)
+            uf.union(new_v, v)
+            v = new_v.arg
     renaming = {key: uf[key].rep for key in uf}
+
     for block, links in entrymap.items():
         if inputs[block]:
             new_inputs, new_args = zip(*inputs[block])

From noreply at buildbot.pypy.org  Mon Nov 17 20:00:58 2014
From: noreply at buildbot.pypy.org (rlamy)
Date: Mon, 17 Nov 2014 20:00:58 +0100 (CET)
Subject: [pypy-commit] pypy expressions: branch superseded by expressions-2
Message-ID: <20141117190058.EA3B41C1147@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy 
Branch: expressions
Changeset: r74564:c3e9466adc06
Date: 2014-11-17 19:00 +0000
http://bitbucket.org/pypy/pypy/changeset/c3e9466adc06/

Log:	branch superseded by expressions-2


From noreply at buildbot.pypy.org  Mon Nov 17 22:06:55 2014
From: noreply at buildbot.pypy.org (mattip)
Date: Mon, 17 Nov 2014 22:06:55 +0100 (CET)
Subject: [pypy-commit] pypy ufuncapi: almost complete program flow,
	refactoring needed
Message-ID: <20141117210655.3BE081C1147@cobra.cs.uni-duesseldorf.de>

Author: mattip 
Branch: ufuncapi
Changeset: r74565:250491ca11a1
Date: 2014-11-17 23:05 +0200
http://bitbucket.org/pypy/pypy/changeset/250491ca11a1/

Log:	almost complete program flow, refactoring needed

diff --git a/pypy/module/micronumpy/nditer.py b/pypy/module/micronumpy/nditer.py
--- a/pypy/module/micronumpy/nditer.py
+++ b/pypy/module/micronumpy/nditer.py
@@ -289,7 +289,7 @@
 class W_NDIter(W_Root):
     _immutable_fields_ = ['ndim', ]
     def __init__(self, space, w_seq, w_flags, w_op_flags, w_op_dtypes,
-                 w_casting, w_op_axes, w_itershape, w_buffersize, order):
+                 w_casting, w_op_axes, w_itershape, buffersize=0, order='K'):
         from pypy.module.micronumpy.ufuncs import find_binop_result_dtype
         
         self.order = order
@@ -578,14 +578,14 @@
 @unwrap_spec(w_flags=WrappedDefault(None), w_op_flags=WrappedDefault(None),
              w_op_dtypes=WrappedDefault(None), order=str,
              w_casting=WrappedDefault(None), w_op_axes=WrappedDefault(None),
-             w_itershape=WrappedDefault(None), w_buffersize=WrappedDefault(None))
-def descr__new__(space, w_subtype, w_seq, w_flags, w_op_flags, w_op_dtypes,
-                 w_casting, w_op_axes, w_itershape, w_buffersize, order='K'):
+             w_itershape=WrappedDefault(None), buffersize=int)
+def descr_new_nditer(space, w_subtype, w_seq, w_flags, w_op_flags, w_op_dtypes,
+                 w_casting, w_op_axes, w_itershape, buffersize=0, order='K'):
     return W_NDIter(space, w_seq, w_flags, w_op_flags, w_op_dtypes, w_casting, w_op_axes,
-                    w_itershape, w_buffersize, order)
+                    w_itershape, buffersize, order)
 
 W_NDIter.typedef = TypeDef('numpy.nditer',
-    __new__ = interp2app(descr__new__),
+    __new__ = interp2app(descr_new_nditer),
 
     __iter__ = interp2app(W_NDIter.descr_iter),
     __getitem__ = interp2app(W_NDIter.descr_getitem),
diff --git a/pypy/module/micronumpy/support.py b/pypy/module/micronumpy/support.py
--- a/pypy/module/micronumpy/support.py
+++ b/pypy/module/micronumpy/support.py
@@ -112,7 +112,7 @@
             if var_name not in var_names:
                 var_names[var_name] = ufunc.core_num_dim_ix
                 ufunc.core_num_dim_ix += 1
-            ufunc.core_dim_ixs[cur_core_dim] = var_names[var_name]
+            ufunc.core_dim_ixs.append(var_names[var_name])
             cur_core_dim += 1
             nd += 1
             i = next_comma
@@ -144,7 +144,6 @@
     if cur_arg != ufunc.nargs:
         raise oefmt(space.w_ValueError, '%s at %d in "%s"',
             "incomplete signature: not all arguments found", i, signature)
-    ufunc.core_dim_ixs = ufunc.core_dim_ixs[:cur_core_dim]
     if cur_core_dim == 0:
-        ufunc.core_enabled = 0
+        ufunc.core_enabled = False
     return 0 # for historical reasons, any failures will raise
diff --git a/pypy/module/micronumpy/ufuncs.py b/pypy/module/micronumpy/ufuncs.py
--- a/pypy/module/micronumpy/ufuncs.py
+++ b/pypy/module/micronumpy/ufuncs.py
@@ -542,10 +542,10 @@
         #These will be filled in by _parse_signature
         self.core_enabled = True    # False for scalar ufunc, True for generalized ufunc
         self.stack_inputs = stack_inputs
-        self.core_num_dim_ix = 0 # number of distinct dimention names in signature
+        self.core_num_dim_ix = 0 # number of distinct dimension names in signature
         self.core_num_dims = [0] * self.nargs  # number of core dimensions of each nargs
         self.core_offsets = [0] * self.nargs
-        self.core_dim_ixs = [0] * len(signature)
+        self.core_dim_ixs = [] # indices into unique shapes for each arg
 
     def reduce(self, space, w_obj, w_axis, keepdims=False, out=None, dtype=None,
                cumulative=False):
@@ -559,8 +559,8 @@
                  self.name, self.nin, len(args_w))
         for i in range(self.nin):
             inargs[i] = convert_to_array(space, args_w[i])
-        outargs = [None] * min(self.nout, len(args_w)-self.nin)
-        for i in range(len(outargs)):
+        outargs = [None] * self.nout
+        for i in range(len(args_w)-self.nin):
             out = args_w[i+self.nin]
             if space.is_w(out, space.w_None) or out is None:
                 continue
@@ -698,19 +698,35 @@
 
         # TODO parse and handle subok
         # TODO handle flags, op_flags
-        flags = ['external_loop']
-        op_flags = None
+        w_flags = space.w_None #space.newlist([space.wrap('external_loop')])
+        w_op_flags = space.w_None
+        w_op_dtypes = space.w_None
+        w_casting = space.w_None
+        w_itershape = space.newlist([space.wrap(i) for i in iter_shape]) 
+        w_op_axes = space.w_None
 
         # mimic NpyIter_AdvancedNew with a nditer
 
         if self.stack_inputs:
             inargs = inargs + outargs
-            it = W_NDIter(space, space.newlist(inargs + outargs),
-                            flags=flags, op_flags=op_flags)
-            for args in it:
-                space.call_args(func, Arguments.frompacked(space, args))
+            nd_it = W_NDIter(space, space.newlist(inargs + outargs), w_flags,
+                          w_op_flags, w_op_dtypes, w_casting, w_op_axes,
+                          w_itershape)
+            # XXX coalesce each iterators, according to inner_dimensions
+            while not nd_it.done:
+                for it, st in nd_it.iters:
+                    if not it.done(st):
+                        break
+                else:
+                    nd_it.done = True
+                    break
+                args = []
+                for i, (it, st) in enumerate(nd_it.iters):
+                    args.append(nd_it.getitem(it, st))
+                    nd_it.iters[i] = (it, it.next(st))
+                space.call_args(func, Arguments.frompacked(space, space.newlist(args)))
             if len(outargs) > 1:
-                return outargs
+                return space.newtuple([convert_to_array(space, o) for o in outargs])
             return outargs[0]
         inargs0 = inargs[0]
         assert isinstance(inargs0, W_NDimArray)
@@ -1147,9 +1163,9 @@
     name: str, default=''
     doc: str, default=''
     stack_inputs*: boolean, whether the function is of the form
-            out = func(*in)     False
+            out = func(*in)  False
             or
-            func(*in_out)       True
+            func(*in_out)    True (forces use of a nditer with 'external_loop')
 
     only one of out_dtype or signature may be specified
 
@@ -1218,6 +1234,9 @@
     if len(signature) == 0:
         # cpython compatability, func is of the form (i),(i)->(i)
         signature = ','.join(['(i)'] * nin) + '->' + ','.join(['(i)'] * nout)
+    else:
+        #stack_inputs = True
+        pass
 
     w_ret = W_UfuncGeneric(space, func, name, identity, nin, nout, dtypes,
                            signature, match_dtypes=match_dtypes,

From noreply at buildbot.pypy.org  Mon Nov 17 22:22:44 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 17 Nov 2014 22:22:44 +0100 (CET)
Subject: [pypy-commit] creflect default: Kill stuff,
	and start to pass some tests.
Message-ID: <20141117212244.83EF11C0F1B@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r53:313a8c6c6a9d
Date: 2014-11-17 22:23 +0100
http://bitbucket.org/cffi/creflect/changeset/313a8c6c6a9d/

Log:	Kill stuff, and start to pass some tests.

diff --git a/creflect/codegen.py b/creflect/codegen.py
--- a/creflect/codegen.py
+++ b/creflect/codegen.py
@@ -1,21 +1,10 @@
 
 class CodeBlock(object):
 
-    def __init__(self, tr):
-        self.tr = tr
-        self.blocknum = tr.next_block_num()
-        self.skip_label_active = False
-        self.includes = []
+    def __init__(self, parent):
         self.decllines = []
         self.codelines = []
-        self.sprintfleft = ""
-        self.sprintfright = ""
-        self.sprintfright_extra = ("", 0)
-        self.sprintfmax = []
-
-    def add_include(self, includename):
-        if includename not in self.includes:
-            self.includes.append(includename)
+        self.crx_type_vars = [] if parent is None else parent.crx_type_vars
 
     def writedecl(self, line):
         self.decllines.append(line)
@@ -23,137 +12,31 @@
     def writeline(self, line):
         self.codelines.append(line)
 
-    def sprintf_add_left(self, msg):
-        assert '&' not in msg
-        self.sprintfleft = self.sprintfleft + msg
-
-    def sprintf_add_right(self, msg, extra="", extralength=None):
-        assert '&' not in msg
-        self.sprintfright = msg + self.sprintfright
-        if extra:
-            assert extralength is not None, "specify 'extralength' explicitly"
-            if self.sprintfright_extra[0]:
-                extra += ', '
-            self.sprintfright_extra = (
-                extra + self.sprintfright_extra[0],
-                extralength + self.sprintfright_extra[1])
-
-    def sprintf_add_both_sides(self, marker):
-        i = marker.index('&')
-        self.sprintf_add_left(marker[:i])
-        self.sprintf_add_right(marker[i+1:])
-
-    def sprintf(self, buf, extra="", extralength=None, indent=0):
-        if extralength is None:
-            assert not extra, "specify 'extralength' explicitly"
-            extralength = 0
-        if extra:
-            extra = ', ' + extra
-        self.writeline('%sr += sprintf(r, "%s"%s);' %
-                       (' ' * indent, buf, extra))
-        self.sprintfmax.append(len(buf) + extralength)
-
-    def flushleft(self):
-        buf = self.sprintfleft
-        if buf:
-            self.sprintfleft = ""
-            self.sprintf(buf)
-        return len(self.sprintfmax)
-
-    def end_disjoint_cases(self, startindex):
-        if startindex == len(self.sprintfmax):
-            return
-        assert startindex < len(self.sprintfmax)
-        m = max(self.sprintfmax[startindex:])
-        self.sprintfmax[startindex:] = [m]
-
-    def flush(self):
-        buf = self.sprintfleft + self.sprintfright
-        if buf:
-            self.sprintfleft = ""
-            self.sprintfright = ""
-            extra, extralength = self.sprintfright_extra
-            self.sprintfright_extra = ("", 0)
-            self.sprintf(buf, extra=extra, extralength=extralength)
-
-    def get_skip_label(self):
-        if not self.skip_label_active:
-            self.skip_label_active = True
-            self.writedecl('char *r0 = r;')
-        return 'f%d' % (self.blocknum,)
-
-    def close(self):
-        self.flush()
-        if self.skip_label_active:
-            self.writeline('%s:;' % (self.get_skip_label(),))
-            self.skip_label_active = False
-
     def write_subblock(self, subblock):
-        subblock.close()
-        for incl in subblock.includes:
-            self.add_include(incl)
         self.writeline("{")
         for line in subblock.decllines + subblock.codelines:
             self.writeline("    " + line)
         self.writeline("}")
-        self.sprintfmax.extend(subblock.sprintfmax)
 
+    def add_crx_type_var(self):
+        name = 't%d' % (len(self.crx_type_vars) + 1)
+        self.crx_type_vars.append('*%s' % name)
+        return name
 
-class TransformCDef(object):
 
-    def __init__(self, csource):
-        self.csource = csource
-        self.skip_label_counter = 0
-        self.skip_label_active = False
-        self.block_counter = 0
-        self.return_value = False
+def transform_cdef(csource, creflect_func_name):
+    outerblock = CodeBlock(parent=None)
+    outerblock.writeline('void %s(crx_builder_t *cb)' % (creflect_func_name,))
 
-    def next_block_num(self):
-        r = self.block_counter
-        self.block_counter = r + 1
-        return r
+    funcblock = CodeBlock(outerblock)
 
-    def need_return_value(self):
-        if not self.return_value:
-            self.return_value = True
-            self.funcblock.writedecl("int r1 = 0;")
+    for decl in csource.get_declarations():
+        decl.write_declaration(funcblock)
 
-    def generate(self, funcname):
-        outerblock = CodeBlock(self)
-        outerblock.add_include('stdio.h')
-        outerblock.writeline('int %s(char *r)' % (funcname,))
+    if funcblock.crx_type_vars:
+        funcblock.writedecl("crx_type_t %s;" % (
+            ', '.join(funcblock.crx_type_vars),))
 
-        funcblock = CodeBlock(self)
-        self.funcblock = funcblock
-
-        for decl in self.csource.get_declarations():
-            decl.write_declaration(funcblock)
-
-        funcblock.writedecl('if (!r)')
-        if funcblock.sprintfmax:
-            funcblock.writedecl('    return %s;' %
-                                (' + '.join(map(str, funcblock.sprintfmax)),))
-        else:
-            funcblock.writedecl('    return 1;')
-            funcblock.writedecl('*r = 0;')
-        if self.return_value:
-            funcblock.writeline('return r1;')
-        else:
-            funcblock.writeline('return 0;')
-
-        outerblock.write_subblock(funcblock)
-        outerblock.writeline('')
-        #
-        if outerblock.includes:
-            extra = ['#include <%s>' % includename
-                     for includename in outerblock.includes]
-            extra.append('')
-        else:
-            extra = []
-        return extra + outerblock.codelines
-
-
-def transform_cdef(csource, creflect_func_num=1):
-    transform = TransformCDef(csource)
-    outputlines = transform.generate(creflect_func_num)
-    return '\n'.join(outputlines)
+    outerblock.write_subblock(funcblock)
+    outerblock.writeline('')     # for the final \n below
+    return '\n'.join(outerblock.codelines)
diff --git a/creflect/model.py b/creflect/model.py
--- a/creflect/model.py
+++ b/creflect/model.py
@@ -15,9 +15,7 @@
                 for name in (self._attrs_ + 'const')]
 
     def inspect_type(self, block, inspect):
-        if inspect.started:
-            inspect.assign_to_p1('0')
-        block.sprintf_add_both_sides(self.c_name_with_marker)
+        raise NotImplementedError
 
     def __eq__(self, other):
         return (self.__class__ == other.__class__ and
@@ -36,8 +34,13 @@
     def __init__(self, const):
         self.const = const
         self.c_name_with_marker = 'void &'
-        if const:
-            self.c_name_with_marker = 'const ' + self.c_name_with_marker
+
+    def inspect_type(self, block, inspect):
+        if inspect.started:
+            inspect.assign_to_p1('0')
+        t1 = block.add_crx_type_var()
+        block.writeline('%s = cb->get_void_type(cb);' % (t1,))
+        return t1
 
 void_type = VoidType(const=False)
 const_void_type = VoidType(const=True)
@@ -89,51 +92,20 @@
         block.writeline("(void)(%s << 1);%s" % (star_p1, comment2))
         if self.const:
             block.sprintf_add_left("const ")
-        disjunction = block.flushleft()
-        if self.const:
-            block.add_include("string.h")
             block.writeline("memset(b, -1, sizeof(b));")
         else:
             block.writeline("%s = -1;%s" % (star_p1, comment3))
-        block.writeline("if (%s > 0) {" % (star_p1,))
-        block.writeline("    if (sizeof(%s) == 1 && %s == 1)" %
-                        (star_p1, star_p1))
-        block.sprintf("_Bool", indent=8)
-        #
-        checktypes = ["char", "short", "int", "long", "long long"]
-        if self.name in checktypes:
-            i = checktypes.index(self.name)
-        elif self.name.startswith("unsigned ") and self.name[9:] in checktypes:
-            i = checktypes.index(self.name[9:])
+        t1 = block.add_crx_type_var()
+        if self.is_integer_type():
+            block.writeline('%s = CRX_INT_TYPE(cb, %s, "%s");' % (
+                t1, star_p1, self.name))
+        elif self.is_char_type():
+            block.writeline('%s = cb->get_char_type(cb);' % (t1,))
+        elif self.is_float_type():
+            xxx
         else:
-            i = 0
-        checktypes = checktypes[i::-1] + checktypes[i+1:]
-        for t in checktypes:
-            ut = "unsigned " + t
-            if self.name == t == "char":
-                ut = "char"
-            block.writeline("    else if (sizeof(%s) == sizeof(%s))" %
-                            (star_p1, ut))
-            block.sprintf(ut, indent=8)
-        block.writeline("    else")
-        block.sprintf("uint%u_t", "(int)sizeof(%s) * 8" % (star_p1,),
-                      extralength=10, indent=8)
-        block.writeline("}")
-        block.writeline("else {")
-        elsecase = ""
-        for t in checktypes:
-            if t == "char" and self.name != "char":
-                t = "signed char"
-            block.writeline("    %sif (sizeof(%s) == sizeof(%s))" %
-                            (elsecase, star_p1, t))
-            block.sprintf("%s" % (t,), indent=8)
-            elsecase = "else "
-        block.writeline("    else")
-        block.sprintf("int%u_t", "(int)sizeof(%s) * 8" % (star_p1,),
-                      extralength=10, indent=8)
-        block.writeline("}")
-        block.end_disjoint_cases(disjunction)
-        block.sprintf_add_left(" ")
+            raise AssertionError
+        return t1
 
 
 class BaseFunctionType(BaseType):
@@ -185,7 +157,7 @@
         self.totype.inspect_type(block, inspect)
         #
         extra = self._base_pattern[self.totype.is_array_type, self.const]
-        block.sprintf_add_both_sides(extra)
+        #block.sprintf_add_both_sides(extra)
 
 
 class ArrayType(BaseType):
@@ -419,11 +391,9 @@
         self.type = type
 
     def write_declaration(self, funcblock):
-        block = CodeBlock(funcblock.tr)
-        block.sprintf_add_left('typedef ')
-        block.sprintf_add_right(r';\n')
+        block = CodeBlock(funcblock)
         inspect = TypeInspector(block, self.name)
-        self.type.inspect_type(block, inspect)
+        t1 = self.type.inspect_type(block, inspect)
         inspect.stop()
-        block.sprintf_add_left(self.name)
+        block.writeline('cb->define_type(cb, "%s", %s);' % (self.name, t1))
         funcblock.write_subblock(block)
diff --git a/test/test_codegen.py b/test/test_codegen.py
--- a/test/test_codegen.py
+++ b/test/test_codegen.py
@@ -1,7 +1,7 @@
 import py
 import os
 from creflect.cparser import CSource
-from creflect.codegen import TransformCDef
+from creflect.codegen import transform_cdef
 
 
 def _trimlines(lines):
@@ -37,12 +37,12 @@
     f.close()
     print
     #
+    csource = CSource(''.join(cdefblock))
     funcname = 'test' + os.path.splitext(filename)[0].replace('-','_')
-    transform = TransformCDef(CSource(''.join(cdefblock)))
-    got = transform.generate(funcname)
-    print '\n'.join(got)
+    text = transform_cdef(csource, funcname)
+    print text
     #
-    got = _trimlines(got)
+    got = _trimlines(text.splitlines())
     expected = _trimlines(expected)
     compare_lists(got, expected)
 

From noreply at buildbot.pypy.org  Mon Nov 17 22:28:46 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 17 Nov 2014 22:28:46 +0100 (CET)
Subject: [pypy-commit] creflect default: Fix more tests
Message-ID: <20141117212846.98F961C0F1B@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r54:cbf8158d85b4
Date: 2014-11-17 22:29 +0100
http://bitbucket.org/cffi/creflect/changeset/cbf8158d85b4/

Log:	Fix more tests

diff --git a/creflect/model.py b/creflect/model.py
--- a/creflect/model.py
+++ b/creflect/model.py
@@ -15,6 +15,15 @@
                 for name in (self._attrs_ + 'const')]
 
     def inspect_type(self, block, inspect):
+        t1 = self.inspect_nonconst_type(block, inspect)
+        if self.const:
+            t2 = block.add_crx_type_var()
+            block.writeline('%s = cb->get_const_type(cb, %s);' % (t2, t1))
+            return t2
+        else:
+            return t1
+
+    def inspect_nonconst_type(self, block, inspect):
         raise NotImplementedError
 
     def __eq__(self, other):
@@ -35,7 +44,7 @@
         self.const = const
         self.c_name_with_marker = 'void &'
 
-    def inspect_type(self, block, inspect):
+    def inspect_nonconst_type(self, block, inspect):
         if inspect.started:
             inspect.assign_to_p1('0')
         t1 = block.add_crx_type_var()
@@ -82,7 +91,7 @@
     def is_float_type(self):
         return self.ALL_PRIMITIVE_TYPES[self.name] == 'f'
 
-    def inspect_type(self, block, inspect):
+    def inspect_nonconst_type(self, block, inspect):
         star_p1 = inspect.fetch_star_p1()
         comment1 = inspect.get_comment(0, False, "a valid type", minlevel=1)
         comment2 = inspect.get_comment(0, False, "an integer type")
@@ -116,21 +125,14 @@
 
 class PointerType(BaseType):
     _attrs_ = ('totype',)
-    _base_pattern       = {
-        (False, False): "*&",
-        (True, False): "(*&)",
-        (False, True): "*const &",
-        (True, True): "(*const &)",
-        }
 
     def __init__(self, totype, const):
         self.totype = totype
         self.const = const
         base = self.totype.c_name_with_marker
-        extra = self._base_pattern[self.totype.is_array_type, self.const]
-        self.c_name_with_marker = base.replace('&', extra)
+        self.c_name_with_marker = base.replace('&', '(*&)')
 
-    def inspect_type(self, block, inspect):
+    def inspect_nonconst_type(self, block, inspect):
         star_p1 = inspect.fetch_star_p1()
         new_var = 'p%d' % (len(inspect.levels) + 2,)
         block.writedecl("char *%s;" % (new_var,))
@@ -154,10 +156,10 @@
                 block.writeline("}")
             inspect.after_star_p1_assignment.append(after)
         inspect.levels.append('*')
-        self.totype.inspect_type(block, inspect)
-        #
-        extra = self._base_pattern[self.totype.is_array_type, self.const]
-        #block.sprintf_add_both_sides(extra)
+        t1 = self.totype.inspect_type(block, inspect)
+        t2 = block.add_crx_type_var()
+        block.writeline('%s = cb->get_pointer_type(cb, %s);' % (t2, t1))
+        return t2
 
 
 class ArrayType(BaseType):
@@ -178,7 +180,7 @@
         self.c_name_with_marker = (
             self.item.c_name_with_marker.replace('&', brackets))
 
-    def inspect_type(self, block, inspect):
+    def inspect_nonconst_type(self, block, inspect):
         star_p1 = inspect.fetch_star_p1()
         errmsg = "type '%s' is not an array, but a pointer type" % (
             inspect.get_comment_type(0, False),)

From noreply at buildbot.pypy.org  Mon Nov 17 22:35:46 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 17 Nov 2014 22:35:46 +0100 (CET)
Subject: [pypy-commit] creflect default: progress
Message-ID: <20141117213546.35B341D28AF@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r55:c0cc01d4d272
Date: 2014-11-17 22:36 +0100
http://bitbucket.org/cffi/creflect/changeset/c0cc01d4d272/

Log:	progress

diff --git a/creflect/model.py b/creflect/model.py
--- a/creflect/model.py
+++ b/creflect/model.py
@@ -180,7 +180,9 @@
         self.c_name_with_marker = (
             self.item.c_name_with_marker.replace('&', brackets))
 
-    def inspect_nonconst_type(self, block, inspect):
+    def inspect_type(self, block, inspect):
+        # this class overrides inspect_type() instead of
+        # inspect_nonconst_type(), to avoid the extra call to get_const_type()
         star_p1 = inspect.fetch_star_p1()
         errmsg = "type '%s' is not an array, but a pointer type" % (
             inspect.get_comment_type(0, False),)
@@ -190,19 +192,17 @@
                 ampersand_star_p1 = ampersand_star_p1[2:]
             block.writeline("if ((void *)%s != (void *)%s) {" % (
                 ampersand_star_p1, star_p1))
-            block.writeline("    r = r0;")
-            block.sprintf(r"#error %s\n" % errmsg, indent=4)
-            block.tr.need_return_value()
-            block.writeline("    r1 = -1;")
-            block.writeline("    goto %s;" % block.get_skip_label())
+            block.writeline('    cb->error(cb, "%s");' % (errmsg,))
+            block.writeline("    return;")
             block.writeline("}")
         inspect.levels.append('[]')
         inspect.after_star_p1_assignment.append(after)
-        self.item.inspect_type(block, inspect)
-        block.sprintf_add_right('[%lld]',
-            extra='(long long)(sizeof(%s) / sizeof(*%s))' % (
-                star_p1, star_p1),
-            extralength=20)
+        t1 = self.item.inspect_type(block, inspect)
+        t2 = block.add_crx_type_var()
+        block.writeline('%s = cb->get_array_type(cb, %s, '
+                        'sizeof(%s) / sizeof(*%s));' % (
+            t2, t1, star_p1, star_p1))
+        return t2
 
 
 class StructOrUnionOrEnum(BaseType):
diff --git a/test/codegen/005c.c b/test/codegen/005c.c
--- a/test/codegen/005c.c
+++ b/test/codegen/005c.c
@@ -2,59 +2,25 @@
 
 # ____________________________________________________________
 
-int test005c(char *r)
+void test005c(crx_builder_t *cb)
 {
-    int r1 = 0;
-    if (!r)
-        return 58 + 8 + 18 + 38;
+    crx_type_t *t1, *t2, *t3;
     {
         foo_t *p1;
         char *p2;
         char b[sizeof(***p1)];  /* check that '(*foo_t)[]' is a valid type */
-        char *r0 = r;
         p1 = (void *)&p2;
         *p1 = (void *)b;    /* check that 'foo_t' is a pointer type */
         if ((void *)*p1 != (void *)**p1) {
-            r = r0;
-            r += sprintf(r, "#error type '*foo_t' is not an array, but a pointer type\n");
-            r1 = -1;
-            goto f2;
+            cb->error(cb, "type '*foo_t' is not an array, but a pointer type");
+            return;
         }
         (void)(***p1 << 1);  /* check that '(*foo_t)[]' is an integer type */
-        r += sprintf(r, "typedef ");
         ***p1 = -1;  /* check that '(*foo_t)[]' is not declared 'const' */
-        if (***p1 > 0) {
-            if (sizeof(***p1) == 1 && ***p1 == 1)
-                r += sprintf(r, "_Bool");
-            else if (sizeof(***p1) == sizeof(unsigned int))
-                r += sprintf(r, "unsigned int");
-            else if (sizeof(***p1) == sizeof(unsigned short))
-                r += sprintf(r, "unsigned short");
-            else if (sizeof(***p1) == sizeof(unsigned char))
-                r += sprintf(r, "unsigned char");
-            else if (sizeof(***p1) == sizeof(unsigned long))
-                r += sprintf(r, "unsigned long");
-            else if (sizeof(***p1) == sizeof(unsigned long long))
-                r += sprintf(r, "unsigned long long");
-            else
-                r += sprintf(r, "uint%u_t", (int)sizeof(***p1) * 8);
-        }
-        else {
-            if (sizeof(***p1) == sizeof(int))
-                r += sprintf(r, "int");
-            else if (sizeof(***p1) == sizeof(short))
-                r += sprintf(r, "short");
-            else if (sizeof(***p1) == sizeof(signed char))
-                r += sprintf(r, "signed char");
-            else if (sizeof(***p1) == sizeof(long))
-                r += sprintf(r, "long");
-            else if (sizeof(***p1) == sizeof(long long))
-                r += sprintf(r, "long long");
-            else
-                r += sprintf(r, "int%u_t", (int)sizeof(***p1) * 8);
-        }
-        r += sprintf(r, " (*foo_t)[%lld];\n", (long long)(sizeof(**p1) / sizeof(***p1)));
-        f2:;
+        t1 = CRX_INT_TYPE(cb, ***p1, "int");
+        t2 = cb->get_array_type(cb, t1, sizeof(**p1) / sizeof(***p1));
+        t3 = cb->get_pointer_type(cb, t2);
+        cb->define_type(cb, "foo_t", t3);
+#expect TYPEDEF foo_t = PTR ARRAY[27] int
     }
-    return r1;
 }

From noreply at buildbot.pypy.org  Mon Nov 17 22:45:03 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 17 Nov 2014 22:45:03 +0100 (CET)
Subject: [pypy-commit] creflect default: fixes
Message-ID: <20141117214503.3C0181D3616@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r56:7911f7d90ae1
Date: 2014-11-17 22:45 +0100
http://bitbucket.org/cffi/creflect/changeset/7911f7d90ae1/

Log:	fixes

diff --git a/test/codegen/005d.c b/test/codegen/005d.c
--- a/test/codegen/005d.c
+++ b/test/codegen/005d.c
@@ -2,15 +2,12 @@
 
 # ____________________________________________________________
 
-int test005d(char *r)
+void test005d(crx_builder_t *cb)
 {
-    int r1 = 0;
-    if (!r)
-        return 57 + 59 + 64 + 68 + 8 + 18 + 120;
+    crx_type_t *t1, *t2, *t3, *t4, *t5, *t6, *t7, *t8, *t9, *t10;
     {
         foo_t *p1;
         char *p4;
-        char *r0 = r;
         char *p5;
         char *p6;
         char *p9;
@@ -18,69 +15,39 @@
         char b[sizeof(**********p1)];  /* check that '**(***foo_t[][])[][]' is a valid type */
         p1 = (void *)&p4;
         if ((void *)p1 != (void *)*p1) {
-            r = r0;
-            r += sprintf(r, "#error type 'foo_t' is not an array, but a pointer type\n");
-            r1 = -1;
-            goto f2;
+            cb->error(cb, "type 'foo_t' is not an array, but a pointer type");
+            return;
         }
         if ((void *)*p1 != (void *)**p1) {
-            r = r0;
-            r += sprintf(r, "#error type 'foo_t[]' is not an array, but a pointer type\n");
-            r1 = -1;
-            goto f2;
+            cb->error(cb, "type 'foo_t[]' is not an array, but a pointer type");
+            return;
         }
         ***p1 = (void *)&p5;  /* check that 'foo_t[][]' is a pointer type */
         ****p1 = (void *)&p6;  /* check that '*foo_t[][]' is a pointer type */
         *****p1 = (void *)&p9;  /* check that '**foo_t[][]' is a pointer type */
         if ((void *)*****p1 != (void *)******p1) {
-            r = r0;
-            r += sprintf(r, "#error type '***foo_t[][]' is not an array, but a pointer type\n");
-            r1 = -1;
-            goto f2;
+            cb->error(cb, "type '***foo_t[][]' is not an array, but a pointer type");
+            return;
         }
         if ((void *)******p1 != (void *)*******p1) {
-            r = r0;
-            r += sprintf(r, "#error type '(***foo_t[][])[]' is not an array, but a pointer type\n");
-            r1 = -1;
-            goto f2;
+            cb->error(cb, "type '(***foo_t[][])[]' is not an array, but a pointer type");
+            return;
         }
         ********p1 = (void *)&p10;  /* check that '(***foo_t[][])[][]' is a pointer type */
         *********p1 = (void *)b;    /* check that '*(***foo_t[][])[][]' is a pointer type */
         (void)(**********p1 << 1);  /* check that '**(***foo_t[][])[][]' is an integer type */
-        r += sprintf(r, "typedef ");
         **********p1 = -1;  /* check that '**(***foo_t[][])[][]' is not declared 'const' */
-        if (**********p1 > 0) {
-            if (sizeof(**********p1) == 1 && **********p1 == 1)
-                r += sprintf(r, "_Bool");
-            else if (sizeof(**********p1) == sizeof(unsigned int))
-                r += sprintf(r, "unsigned int");
-            else if (sizeof(**********p1) == sizeof(unsigned short))
-                r += sprintf(r, "unsigned short");
-            else if (sizeof(**********p1) == sizeof(unsigned char))
-                r += sprintf(r, "unsigned char");
-            else if (sizeof(**********p1) == sizeof(unsigned long))
-                r += sprintf(r, "unsigned long");
-            else if (sizeof(**********p1) == sizeof(unsigned long long))
-                r += sprintf(r, "unsigned long long");
-            else
-                r += sprintf(r, "uint%u_t", (int)sizeof(**********p1) * 8);
-        }
-        else {
-            if (sizeof(**********p1) == sizeof(int))
-                r += sprintf(r, "int");
-            else if (sizeof(**********p1) == sizeof(short))
-                r += sprintf(r, "short");
-            else if (sizeof(**********p1) == sizeof(signed char))
-                r += sprintf(r, "signed char");
-            else if (sizeof(**********p1) == sizeof(long))
-                r += sprintf(r, "long");
-            else if (sizeof(**********p1) == sizeof(long long))
-                r += sprintf(r, "long long");
-            else
-                r += sprintf(r, "int%u_t", (int)sizeof(**********p1) * 8);
-        }
-        r += sprintf(r, " **(***foo_t[%lld][%lld])[%lld][%lld];\n", (long long)(sizeof(*p1) / sizeof(**p1)), (long long)(sizeof(**p1) / sizeof(***p1)), (long long)(sizeof(******p1) / sizeof(*******p1)), (long long)(sizeof(*******p1) / sizeof(********p1)));
-        f2:;
+        t1 = CRX_INT_TYPE(cb, **********p1, "int");
+        t2 = cb->get_pointer_type(cb, t1);
+        t3 = cb->get_pointer_type(cb, t2);
+        t4 = cb->get_array_type(cb, t3, sizeof(*******p1) / sizeof(********p1));
+        t5 = cb->get_array_type(cb, t4, sizeof(******p1) / sizeof(*******p1));
+        t6 = cb->get_pointer_type(cb, t5);
+        t7 = cb->get_pointer_type(cb, t6);
+        t8 = cb->get_pointer_type(cb, t7);
+        t9 = cb->get_array_type(cb, t8, sizeof(**p1) / sizeof(***p1));
+        t10 = cb->get_array_type(cb, t9, sizeof(*p1) / sizeof(**p1));
+        cb->define_type(cb, "foo_t", t10);
+#expect TYPEDEF foo_t = ARRAY[4] ARRAY[3] PTR PTR PTR ARRAY[27] ARRAY[91] PTR PTR int
     }
-    return r1;
 }
diff --git a/test/codegen/005e.c b/test/codegen/005e.c
deleted file mode 100644
--- a/test/codegen/005e.c
+++ /dev/null
@@ -1,106 +0,0 @@
-typedef int foo_t[55];
-typedef int bar_t[66];
-
-# ____________________________________________________________
-
-int test005e(char *r)
-{
-    int r1 = 0;
-    if (!r)
-        return 57 + 8 + 18 + 35 + 57 + 8 + 18 + 35;
-    {
-        foo_t *p1;
-        char b[sizeof(**p1)];  /* check that 'foo_t[]' is a valid type */
-        char *r0 = r;
-        p1 = (void *)b;
-        if ((void *)p1 != (void *)*p1) {
-            r = r0;
-            r += sprintf(r, "#error type 'foo_t' is not an array, but a pointer type\n");
-            r1 = -1;
-            goto f2;
-        }
-        (void)(**p1 << 1);  /* check that 'foo_t[]' is an integer type */
-        r += sprintf(r, "typedef ");
-        **p1 = -1;  /* check that 'foo_t[]' is not declared 'const' */
-        if (**p1 > 0) {
-            if (sizeof(**p1) == 1 && **p1 == 1)
-                r += sprintf(r, "_Bool");
-            else if (sizeof(**p1) == sizeof(unsigned int))
-                r += sprintf(r, "unsigned int");
-            else if (sizeof(**p1) == sizeof(unsigned short))
-                r += sprintf(r, "unsigned short");
-            else if (sizeof(**p1) == sizeof(unsigned char))
-                r += sprintf(r, "unsigned char");
-            else if (sizeof(**p1) == sizeof(unsigned long))
-                r += sprintf(r, "unsigned long");
-            else if (sizeof(**p1) == sizeof(unsigned long long))
-                r += sprintf(r, "unsigned long long");
-            else
-                r += sprintf(r, "uint%u_t", (int)sizeof(**p1) * 8);
-        }
-        else {
-            if (sizeof(**p1) == sizeof(int))
-                r += sprintf(r, "int");
-            else if (sizeof(**p1) == sizeof(short))
-                r += sprintf(r, "short");
-            else if (sizeof(**p1) == sizeof(signed char))
-                r += sprintf(r, "signed char");
-            else if (sizeof(**p1) == sizeof(long))
-                r += sprintf(r, "long");
-            else if (sizeof(**p1) == sizeof(long long))
-                r += sprintf(r, "long long");
-            else
-                r += sprintf(r, "int%u_t", (int)sizeof(**p1) * 8);
-        }
-        r += sprintf(r, " foo_t[%lld];\n", (long long)(sizeof(*p1) / sizeof(**p1)));
-        f2:;
-    }
-    {
-        bar_t *p1;
-        char b[sizeof(**p1)];  /* check that 'bar_t[]' is a valid type */
-        char *r0 = r;
-        p1 = (void *)b;
-        if ((void *)p1 != (void *)*p1) {
-            r = r0;
-            r += sprintf(r, "#error type 'bar_t' is not an array, but a pointer type\n");
-            r1 = -1;
-            goto f3;
-        }
-        (void)(**p1 << 1);  /* check that 'bar_t[]' is an integer type */
-        r += sprintf(r, "typedef ");
-        **p1 = -1;  /* check that 'bar_t[]' is not declared 'const' */
-        if (**p1 > 0) {
-            if (sizeof(**p1) == 1 && **p1 == 1)
-                r += sprintf(r, "_Bool");
-            else if (sizeof(**p1) == sizeof(unsigned int))
-                r += sprintf(r, "unsigned int");
-            else if (sizeof(**p1) == sizeof(unsigned short))
-                r += sprintf(r, "unsigned short");
-            else if (sizeof(**p1) == sizeof(unsigned char))
-                r += sprintf(r, "unsigned char");
-            else if (sizeof(**p1) == sizeof(unsigned long))
-                r += sprintf(r, "unsigned long");
-            else if (sizeof(**p1) == sizeof(unsigned long long))
-                r += sprintf(r, "unsigned long long");
-            else
-                r += sprintf(r, "uint%u_t", (int)sizeof(**p1) * 8);
-        }
-        else {
-            if (sizeof(**p1) == sizeof(int))
-                r += sprintf(r, "int");
-            else if (sizeof(**p1) == sizeof(short))
-                r += sprintf(r, "short");
-            else if (sizeof(**p1) == sizeof(signed char))
-                r += sprintf(r, "signed char");
-            else if (sizeof(**p1) == sizeof(long))
-                r += sprintf(r, "long");
-            else if (sizeof(**p1) == sizeof(long long))
-                r += sprintf(r, "long long");
-            else
-                r += sprintf(r, "int%u_t", (int)sizeof(**p1) * 8);
-        }
-        r += sprintf(r, " bar_t[%lld];\n", (long long)(sizeof(*p1) / sizeof(**p1)));
-        f3:;
-    }
-    return r1;
-}

From noreply at buildbot.pypy.org  Mon Nov 17 22:50:56 2014
From: noreply at buildbot.pypy.org (pjenvey)
Date: Mon, 17 Nov 2014 22:50:56 +0100 (CET)
Subject: [pypy-commit] pypy py3k: RPY_EXPORTED_FOR_TESTS -> RPY_EXTERN
Message-ID: <20141117215056.BDE631D3616@cobra.cs.uni-duesseldorf.de>

Author: Philip Jenvey 
Branch: py3k
Changeset: r74566:a9d6e26dcb96
Date: 2014-11-17 13:11 -0800
http://bitbucket.org/pypy/pypy/changeset/a9d6e26dcb96/

Log:	RPY_EXPORTED_FOR_TESTS -> RPY_EXTERN

diff --git a/pypy/module/_codecs/locale_codec.h b/pypy/module/_codecs/locale_codec.h
--- a/pypy/module/_codecs/locale_codec.h
+++ b/pypy/module/_codecs/locale_codec.h
@@ -2,7 +2,7 @@
 #include 
 #include "src/precommondefs.h"
 
-RPY_EXPORTED_FOR_TESTS wchar_t* pypy_char2wchar(const char* arg, size_t *size);
-RPY_EXPORTED_FOR_TESTS void pypy_char2wchar_free(wchar_t *text);
-RPY_EXPORTED_FOR_TESTS char* pypy_wchar2char(const wchar_t *text, size_t *error_pos);
-RPY_EXPORTED_FOR_TESTS void pypy_wchar2char_free(char *bytes);
+RPY_EXTERN wchar_t* pypy_char2wchar(const char* arg, size_t *size);
+RPY_EXTERN void pypy_char2wchar_free(wchar_t *text);
+RPY_EXTERN char* pypy_wchar2char(const wchar_t *text, size_t *error_pos);
+RPY_EXTERN void pypy_wchar2char_free(char *bytes);
diff --git a/pypy/module/_posixsubprocess/_posixsubprocess.h b/pypy/module/_posixsubprocess/_posixsubprocess.h
--- a/pypy/module/_posixsubprocess/_posixsubprocess.h
+++ b/pypy/module/_posixsubprocess/_posixsubprocess.h
@@ -1,6 +1,6 @@
 #include "src/precommondefs.h"
 
-RPY_EXPORTED_FOR_TESTS void
+RPY_EXTERN void
 pypy_subprocess_child_exec(
            char *const exec_array[],
            char *const argv[],
@@ -17,8 +17,8 @@
            int (*preexec_fn)(void*),
            void *preexec_fn_arg);
 
-RPY_EXPORTED_FOR_TESTS int
+RPY_EXTERN int
 pypy_subprocess_cloexec_pipe(int *fds);
 
-RPY_EXPORTED_FOR_TESTS void
+RPY_EXTERN void
 pypy_subprocess_init(void);
diff --git a/pypy/module/posix/interp_nt.py b/pypy/module/posix/interp_nt.py
--- a/pypy/module/posix/interp_nt.py
+++ b/pypy/module/posix/interp_nt.py
@@ -24,7 +24,7 @@
     includes=['windows.h'],
     include_dirs=[cdir],
     post_include_bits=[
-        "RPY_EXPORTED_FOR_TESTS DWORD "
+        "RPY_EXTERN DWORD "
         "pypy_GetFinalPathNameByHandle(FARPROC, HANDLE, LPTSTR, DWORD, DWORD);"],
     separate_module_sources=[separate_module_source])
 pypy_GetFinalPathNameByHandle = rffi.llexternal(

From noreply at buildbot.pypy.org  Mon Nov 17 22:51:42 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 17 Nov 2014 22:51:42 +0100 (CET)
Subject: [pypy-commit] creflect default: progress
Message-ID: <20141117215142.A63B51D3616@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r57:9dd40ca6f5b6
Date: 2014-11-17 22:52 +0100
http://bitbucket.org/cffi/creflect/changeset/9dd40ca6f5b6/

Log:	progress

diff --git a/creflect/creflect.h b/creflect/creflect.h
--- a/creflect/creflect.h
+++ b/creflect/creflect.h
@@ -1,4 +1,5 @@
 #include 
+#include 
 
 
 typedef struct {
diff --git a/creflect/model.py b/creflect/model.py
--- a/creflect/model.py
+++ b/creflect/model.py
@@ -97,12 +97,11 @@
         comment2 = inspect.get_comment(0, False, "an integer type")
         comment3 = inspect.get_comment(0, False, "not declared 'const'")
         block.writedecl("char b[sizeof(%s)];%s" % (star_p1, comment1))
+        if self.const:
+            block.writeline("memset(b, -1, sizeof(b));")
         inspect.assign_to_p1("b")
         block.writeline("(void)(%s << 1);%s" % (star_p1, comment2))
-        if self.const:
-            block.sprintf_add_left("const ")
-            block.writeline("memset(b, -1, sizeof(b));")
-        else:
+        if not self.const:
             block.writeline("%s = -1;%s" % (star_p1, comment3))
         t1 = block.add_crx_type_var()
         if self.is_integer_type():
@@ -148,11 +147,8 @@
                     ampersand_star_p1 = ampersand_star_p1[2:]
                 block.writeline("if ((void *)%s == (void *)%s) {" % (
                     ampersand_star_p1, star_p1))
-                block.writeline("    r = r0;")
-                block.sprintf(r"#error %s\n" % errmsg, indent=4)
-                block.tr.need_return_value()
-                block.writeline("    r1 = -1;")
-                block.writeline("    goto %s;" % block.get_skip_label())
+                block.writeline('    cb->error(cb, "%s");' % (errmsg,))
+                block.writeline("    return;")
                 block.writeline("}")
             inspect.after_star_p1_assignment.append(after)
         inspect.levels.append('*')
diff --git a/test/codegen/006.c b/test/codegen/006.c
--- a/test/codegen/006.c
+++ b/test/codegen/006.c
@@ -1,50 +1,19 @@
 typedef const unsigned int num_t;
 
 # ____________________________________________________________
-#include 
 
-int test006(char *r)
+void test006(crx_builder_t *cb)
 {
-    if (!r)
-        return 14 + 18 + 9;
+    crx_type_t *t1, *t2;
     {
         num_t *p1;
         char b[sizeof(*p1)];
         memset(b, -1, sizeof(b));
         p1 = (void *)b;
         (void)(*p1 << 1);  /* check that 'num_t' is an integer type */
-        r += sprintf(r, "typedef const ");
-        if (*p1 > 0) {
-            if (sizeof(*p1) == 1 && *p1 == 1)
-                r += sprintf(r, "_Bool");
-            else if (sizeof(*p1) == sizeof(unsigned int))
-                r += sprintf(r, "unsigned int");
-            else if (sizeof(*p1) == sizeof(unsigned short))
-                r += sprintf(r, "unsigned short");
-            else if (sizeof(*p1) == sizeof(unsigned char))
-                r += sprintf(r, "unsigned char");
-            else if (sizeof(*p1) == sizeof(unsigned long))
-                r += sprintf(r, "unsigned long");
-            else if (sizeof(*p1) == sizeof(unsigned long long))
-                r += sprintf(r, "unsigned long long");
-            else
-                r += sprintf(r, "uint%u_t", (int)sizeof(*p1) * 8);
-        }
-        else {
-            if (sizeof(*p1) == sizeof(int))
-                r += sprintf(r, "int");
-            else if (sizeof(*p1) == sizeof(short))
-                r += sprintf(r, "short");
-            else if (sizeof(*p1) == sizeof(signed char))
-                r += sprintf(r, "signed char");
-            else if (sizeof(*p1) == sizeof(long))
-                r += sprintf(r, "long");
-            else if (sizeof(*p1) == sizeof(long long))
-                r += sprintf(r, "long long");
-            else
-                r += sprintf(r, "int%u_t", (int)sizeof(*p1) * 8);
-        }
-        r += sprintf(r, " num_t;\n");
+        t1 = CRX_INT_TYPE(cb, *p1, "unsigned int");
+        t2 = cb->get_const_type(cb, t1);
+        cb->define_type(cb, "num_t", t2);
+#expect TYPEDEF num_t = CONST unsigned int
     }
-    return 0;
 }
diff --git a/test/codegen/006b.c b/test/codegen/006b.c
--- a/test/codegen/006b.c
+++ b/test/codegen/006b.c
@@ -2,59 +2,25 @@
 
 # ____________________________________________________________
 
-int test006b(char *r)
+void test006b(crx_builder_t *cb)
 {
-    int r1 = 0;
-    if (!r)
-        return 57 + 8 + 18 + 16;
+    crx_type_t *t1, *t2, *t3;
     {
         num_t *p1;
         char *p2;
         char b[sizeof(**p1)];  /* check that '*num_t' is a valid type */
-        char *r0 = r;
         p1 = (void *)&p2;
         p2 = (void *)b;
         if ((void *)p1 == (void *)*p1) {
-            r = r0;
-            r += sprintf(r, "#error type 'num_t' is not a pointer, but an array type\n");
-            r1 = -1;
-            goto f2;
+            cb->error(cb, "type 'num_t' is not a pointer, but an array type");
+            return;
         }
         (void)(**p1 << 1);  /* check that '*num_t' is an integer type */
-        r += sprintf(r, "typedef ");
         **p1 = -1;  /* check that '*num_t' is not declared 'const' */
-        if (**p1 > 0) {
-            if (sizeof(**p1) == 1 && **p1 == 1)
-                r += sprintf(r, "_Bool");
-            else if (sizeof(**p1) == sizeof(unsigned int))
-                r += sprintf(r, "unsigned int");
-            else if (sizeof(**p1) == sizeof(unsigned short))
-                r += sprintf(r, "unsigned short");
-            else if (sizeof(**p1) == sizeof(unsigned char))
-                r += sprintf(r, "unsigned char");
-            else if (sizeof(**p1) == sizeof(unsigned long))
-                r += sprintf(r, "unsigned long");
-            else if (sizeof(**p1) == sizeof(unsigned long long))
-                r += sprintf(r, "unsigned long long");
-            else
-                r += sprintf(r, "uint%u_t", (int)sizeof(**p1) * 8);
-        }
-        else {
-            if (sizeof(**p1) == sizeof(int))
-                r += sprintf(r, "int");
-            else if (sizeof(**p1) == sizeof(short))
-                r += sprintf(r, "short");
-            else if (sizeof(**p1) == sizeof(signed char))
-                r += sprintf(r, "signed char");
-            else if (sizeof(**p1) == sizeof(long))
-                r += sprintf(r, "long");
-            else if (sizeof(**p1) == sizeof(long long))
-                r += sprintf(r, "long long");
-            else
-                r += sprintf(r, "int%u_t", (int)sizeof(**p1) * 8);
-        }
-        r += sprintf(r, " *const num_t;\n");
-        f2:;
+        t1 = CRX_INT_TYPE(cb, **p1, "int");
+        t2 = cb->get_pointer_type(cb, t1);
+        t3 = cb->get_const_type(cb, t2);
+        cb->define_type(cb, "num_t", t3);
+#expect TYPEDEF num_t = CONST PTR int
     }
-    return r1;
 }

From noreply at buildbot.pypy.org  Mon Nov 17 22:56:33 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 17 Nov 2014 22:56:33 +0100 (CET)
Subject: [pypy-commit] creflect default: more progress
Message-ID: <20141117215633.2591F1D3616@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r58:f6aaaa694bb1
Date: 2014-11-17 22:56 +0100
http://bitbucket.org/cffi/creflect/changeset/f6aaaa694bb1/

Log:	more progress

diff --git a/test/codegen/006c.c b/test/codegen/006c.c
--- a/test/codegen/006c.c
+++ b/test/codegen/006c.c
@@ -2,65 +2,30 @@
 
 # ____________________________________________________________
 
-int test006c(char *r)
+void test006c(crx_builder_t *cb)
 {
-    int r1 = 0;
-    if (!r)
-        return 57 + 58 + 8 + 18 + 44;
+    crx_type_t *t1, *t2, *t3, *t4;
     {
         foo_t *p1;
         char *p2;
         char b[sizeof(***p1)];  /* check that '(*foo_t)[]' is a valid type */
-        char *r0 = r;
         p1 = (void *)&p2;
         p2 = (void *)b;
         if ((void *)p1 == (void *)*p1) {
-            r = r0;
-            r += sprintf(r, "#error type 'foo_t' is not a pointer, but an array type\n");
-            r1 = -1;
-            goto f2;
+            cb->error(cb, "type 'foo_t' is not a pointer, but an array type");
+            return;
         }
         if ((void *)*p1 != (void *)**p1) {
-            r = r0;
-            r += sprintf(r, "#error type '*foo_t' is not an array, but a pointer type\n");
-            r1 = -1;
-            goto f2;
+            cb->error(cb, "type '*foo_t' is not an array, but a pointer type");
+            return;
         }
         (void)(***p1 << 1);  /* check that '(*foo_t)[]' is an integer type */
-        r += sprintf(r, "typedef ");
         ***p1 = -1;  /* check that '(*foo_t)[]' is not declared 'const' */
-        if (***p1 > 0) {
-            if (sizeof(***p1) == 1 && ***p1 == 1)
-                r += sprintf(r, "_Bool");
-            else if (sizeof(***p1) == sizeof(unsigned int))
-                r += sprintf(r, "unsigned int");
-            else if (sizeof(***p1) == sizeof(unsigned short))
-                r += sprintf(r, "unsigned short");
-            else if (sizeof(***p1) == sizeof(unsigned char))
-                r += sprintf(r, "unsigned char");
-            else if (sizeof(***p1) == sizeof(unsigned long))
-                r += sprintf(r, "unsigned long");
-            else if (sizeof(***p1) == sizeof(unsigned long long))
-                r += sprintf(r, "unsigned long long");
-            else
-                r += sprintf(r, "uint%u_t", (int)sizeof(***p1) * 8);
-        }
-        else {
-            if (sizeof(***p1) == sizeof(int))
-                r += sprintf(r, "int");
-            else if (sizeof(***p1) == sizeof(short))
-                r += sprintf(r, "short");
-            else if (sizeof(***p1) == sizeof(signed char))
-                r += sprintf(r, "signed char");
-            else if (sizeof(***p1) == sizeof(long))
-                r += sprintf(r, "long");
-            else if (sizeof(***p1) == sizeof(long long))
-                r += sprintf(r, "long long");
-            else
-                r += sprintf(r, "int%u_t", (int)sizeof(***p1) * 8);
-        }
-        r += sprintf(r, " (*const foo_t)[%lld];\n", (long long)(sizeof(**p1) / sizeof(***p1)));
-        f2:;
+        t1 = CRX_INT_TYPE(cb, ***p1, "int");
+        t2 = cb->get_array_type(cb, t1, sizeof(**p1) / sizeof(***p1));
+        t3 = cb->get_pointer_type(cb, t2);
+        t4 = cb->get_const_type(cb, t3);
+        cb->define_type(cb, "foo_t", t4);
+#expect TYPEDEF foo_t = CONST PTR ARRAY[27] int
     }
-    return r1;
 }

From noreply at buildbot.pypy.org  Tue Nov 18 00:03:56 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Tue, 18 Nov 2014 00:03:56 +0100 (CET)
Subject: [pypy-commit] creflect default: Kill file, duplicate from func-003.c
Message-ID: <20141117230356.EE2A11C31FC@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r59:9a6ff1e210b0
Date: 2014-11-17 23:00 +0100
http://bitbucket.org/cffi/creflect/changeset/9a6ff1e210b0/

Log:	Kill file, duplicate from func-003.c

diff --git a/test/codegen/007.c b/test/codegen/007.c
deleted file mode 100644
--- a/test/codegen/007.c
+++ /dev/null
@@ -1,22 +0,0 @@
-typedef int (*myfunc_t)(long, unsigned long long);
-
-# ____________________________________________________________
-
-static void __creflect_t_myfunc_t(void *func, void *args[], void *result) {
-    myfunc_t f = func;
-    *(int *)result = f(*(long *)args[0], *(unsigned long long *)args[1]);
-}
-
-int test007(char *r)
-{
-    if (!r)
-        return 65;
-    {
-        myfunc_t *p1;
-        char *p2;
-        p1 = (void *)&p2;
-        *p1 = (void *)0;    /* check that 'myfunc_t' is a pointer type */
-        r += sprintf(r, "typedef int (*myfunc_t)(long, unsigned long long)/*%p*/;\n", &__creflect_t_myfunc_t);
-    }
-    return 0;
-}

From noreply at buildbot.pypy.org  Tue Nov 18 00:03:58 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Tue, 18 Nov 2014 00:03:58 +0100 (CET)
Subject: [pypy-commit] creflect default: Functions!
Message-ID: <20141117230358.3259F1C31FC@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r60:a5c04855d4c6
Date: 2014-11-18 00:04 +0100
http://bitbucket.org/cffi/creflect/changeset/a5c04855d4c6/

Log:	Functions!

diff --git a/creflect/codegen.py b/creflect/codegen.py
--- a/creflect/codegen.py
+++ b/creflect/codegen.py
@@ -4,7 +4,14 @@
     def __init__(self, parent):
         self.decllines = []
         self.codelines = []
-        self.crx_type_vars = [] if parent is None else parent.crx_type_vars
+        if parent is None:
+            self.crx_type_vars = []
+            self.crx_type_cache = {}
+            self.crx_top_level = self
+        else:
+            self.crx_type_vars = parent.crx_type_vars
+            self.crx_type_cache = parent.crx_type_cache
+            self.crx_top_level = parent.crx_top_level
 
     def writedecl(self, line):
         self.decllines.append(line)
@@ -18,15 +25,31 @@
             self.writeline("    " + line)
         self.writeline("}")
 
-    def add_crx_type_var(self):
-        name = 't%d' % (len(self.crx_type_vars) + 1)
-        self.crx_type_vars.append('*%s' % name)
-        return name
+    def _get_next_name(self, prefix):
+        return '%s%d' % (prefix, len(self.crx_type_vars) + 1)
 
+    def write_crx_type_var(self, expr):
+        if expr in self.crx_type_cache:
+            return self.crx_type_cache[expr]
+        #
+        t1 = self._get_next_name('t')
+        self.crx_type_vars.append('*%s' % t1)
+        self.crx_type_cache[expr] = t1
+        self.writeline('%s = %s;' % (t1, expr))
+        return t1
 
-def transform_cdef(csource, creflect_func_name):
+    def add_array_crx_types(self, array_size):
+        if array_size > 0:
+            a1 = self._get_next_name('a')
+            self.crx_type_vars.append('*%s[%d]' % (a1, array_size))
+            return a1
+        else:
+            return '0'
+
+
+def transform_cdef(csource, crx_func_name):
     outerblock = CodeBlock(parent=None)
-    outerblock.writeline('void %s(crx_builder_t *cb)' % (creflect_func_name,))
+    outerblock.crx_func_name = crx_func_name
 
     funcblock = CodeBlock(outerblock)
 
@@ -37,6 +60,7 @@
         funcblock.writedecl("crx_type_t %s;" % (
             ', '.join(funcblock.crx_type_vars),))
 
+    outerblock.writeline('void %s(crx_builder_t *cb)' % (crx_func_name,))
     outerblock.write_subblock(funcblock)
     outerblock.writeline('')     # for the final \n below
     return '\n'.join(outerblock.codelines)
diff --git a/creflect/cparser.py b/creflect/cparser.py
--- a/creflect/cparser.py
+++ b/creflect/cparser.py
@@ -94,8 +94,8 @@
         node = decl.type
         if isinstance(node, pycparser.c_ast.FuncDecl):
             tp = self._get_type(node)
-            assert isinstance(tp, model.RawFunctionType)
-            tp = self._get_type_pointer(tp)
+            assert isinstance(tp, model.FunctionType)
+            tp = model.PointerType(tp)
             self._declare('function ' + decl.name, tp)
         else:
             const = 'const' in decl.quals
@@ -135,11 +135,6 @@
             realtype = self._get_type(decl.type, approx_name = '$' + decl.name)
         self.declarations.append(model.TypeDefDecl(decl.name, realtype))
 
-    def _get_type_pointer(self, type, const):
-        if isinstance(type, model.RawFunctionType):
-            return type.as_function_pointer()
-        return model.PointerType(type, const)
-
     def _get_type(self, typenode, approx_name=None, partial_length_ok=False):
         # first, dereference typedefs, if we have it already parsed, we're good
         if (isinstance(typenode, pycparser.c_ast.TypeDecl) and
@@ -165,7 +160,7 @@
             if approx_name:
                 approx_name = '$' + approx_name
             realtype = self._get_type(typenode.type, approx_name)
-            return self._get_type_pointer(realtype, const)
+            return model.PointerType(realtype, const)
         #
         if isinstance(typenode, pycparser.c_ast.TypeDecl):
             const = 'const' in typenode.quals
@@ -191,7 +186,7 @@
         #
         if isinstance(typenode, pycparser.c_ast.FuncDecl):
             # a function type
-            return self._parse_function_type(typenode, name)
+            return self._parse_function_type(typenode, approx_name)
         #
         # nested anonymous structs or unions end up here
         if isinstance(typenode, pycparser.c_ast.Struct):
@@ -204,6 +199,36 @@
         raise api.FFIError(":%d: bad or unsupported type declaration" %
                 typenode.coord.line)
 
+    def _parse_function_type(self, typenode, funcname=None):
+        params = list(getattr(typenode.args, 'params', []))
+        ellipsis = (
+            len(params) > 0 and
+            isinstance(params[-1].type, pycparser.c_ast.TypeDecl) and
+            isinstance(params[-1].type.type,
+                       pycparser.c_ast.IdentifierType) and
+            params[-1].type.type.names == ['__dotdotdot__'])
+        if ellipsis:
+            params.pop()
+            if not params:
+                raise api.CDefError(
+                    "%s: a function with only '(...)' as argument"
+                    " is not correct C" % (funcname or 'in expression'))
+        elif (len(params) == 1 and
+            isinstance(params[0].type, pycparser.c_ast.TypeDecl) and
+            isinstance(params[0].type.type, pycparser.c_ast.IdentifierType)
+                and list(params[0].type.type.names) == ['void']):
+            del params[0]
+        args = [self._as_func_arg(self._get_type(argdeclnode.type))
+                for argdeclnode in params]
+        result = self._get_type(typenode.type)
+        return model.FunctionType(tuple(args), result, ellipsis)
+
+    def _as_func_arg(self, type):
+        if isinstance(type, (model.ArrayType, model.FunctionType)):
+            return model.PointerType(type.item)
+        else:
+            return type
+
     def _get_struct_union_enum_type(self, kind, type, const, approx_name=None):
         name = type.name or approx_name
         if not name or name.startswith('$$$'):
diff --git a/creflect/creflect.h b/creflect/creflect.h
--- a/creflect/creflect.h
+++ b/creflect/creflect.h
@@ -10,7 +10,7 @@
 } crx_field_t;
 
 typedef struct crx_type_s crx_type_t;   /* opaque */
-typedef void (*crx_trampoline_t)(void *fn, void *args[], void *res);
+typedef void (*crx_trampoline_fn)(void *fn, void *args[], void *res);
 
 #define CRX_SELF struct _crx_builder_s *
 typedef struct _crx_builder_s {
@@ -24,7 +24,7 @@
     crx_type_t *(*get_float_type)(CRX_SELF, size_t,
                                const char *);
     crx_type_t *(*get_function_type)(CRX_SELF, crx_type_t *,
-                                     crx_type_t *[], int, crx_trampoline_t *);
+                                     crx_type_t *[], int, crx_trampoline_fn);
     crx_type_t *(*get_ellipsis_function_type)(CRX_SELF, crx_type_t *,
                                               crx_type_t *[], int);
     crx_type_t *(*get_pointer_type)(CRX_SELF, crx_type_t *);
diff --git a/creflect/model.py b/creflect/model.py
--- a/creflect/model.py
+++ b/creflect/model.py
@@ -2,13 +2,12 @@
 
 
 class BaseType(object):
-    is_array_type = False
 
-    def _get_c_name(self):
-        return self.c_name_with_marker.replace('&', '')
+    def get_c_name(self, replace_with=''):
+        return self.c_name_with_marker.replace('&', replace_with).rstrip()
 
     def __repr__(self):
-        return '<%s>' % (self._get_c_name(),)
+        return '<%s>' % (self.get_c_name(),)
 
     def _get_items(self):
         return [(name, getattr(self, name))
@@ -17,15 +16,17 @@
     def inspect_type(self, block, inspect):
         t1 = self.inspect_nonconst_type(block, inspect)
         if self.const:
-            t2 = block.add_crx_type_var()
-            block.writeline('%s = cb->get_const_type(cb, %s);' % (t2, t1))
-            return t2
+            return block.write_crx_type_var('cb->get_const_type(cb, %s)' % t1)
         else:
             return t1
 
     def inspect_nonconst_type(self, block, inspect):
         raise NotImplementedError
 
+    def _flush_inspect(self, inspect):
+        if inspect is not None and inspect.started:
+            inspect.assign_to_p1('0')
+
     def __eq__(self, other):
         return (self.__class__ == other.__class__ and
                 self._get_items() == other._get_items())
@@ -43,13 +44,12 @@
     def __init__(self, const):
         self.const = const
         self.c_name_with_marker = 'void &'
+        if const:
+            self.c_name_with_marker = 'const ' + self.c_name_with_marker
 
     def inspect_nonconst_type(self, block, inspect):
-        if inspect.started:
-            inspect.assign_to_p1('0')
-        t1 = block.add_crx_type_var()
-        block.writeline('%s = cb->get_void_type(cb);' % (t1,))
-        return t1
+        self._flush_inspect(inspect)
+        return block.write_crx_type_var('cb->get_void_type(cb)')
 
 void_type = VoidType(const=False)
 const_void_type = VoidType(const=True)
@@ -65,15 +65,15 @@
         'long':               'i',
         'long long':          'i',
         'signed char':        'i',
-        'unsigned char':      'i',
-        'unsigned short':     'i',
-        'unsigned int':       'i',
-        'unsigned long':      'i',
-        'unsigned long long': 'i',
+        'unsigned char':      'u',
+        'unsigned short':     'u',
+        'unsigned int':       'u',
+        'unsigned long':      'u',
+        'unsigned long long': 'u',
         'float':              'f',
         'double':             'f',
         'long double':        'f',
-        '_Bool':              'i',
+        '_Bool':              'u',
         }
 
     def __init__(self, name, const):
@@ -87,80 +87,147 @@
     def is_char_type(self):
         return self.ALL_PRIMITIVE_TYPES[self.name] == 'c'
     def is_integer_type(self):
+        return self.ALL_PRIMITIVE_TYPES[self.name] in 'iu'
+    def is_signed_type(self):
         return self.ALL_PRIMITIVE_TYPES[self.name] == 'i'
+    def is_unsigned_type(self):
+        return self.ALL_PRIMITIVE_TYPES[self.name] == 'u'
     def is_float_type(self):
         return self.ALL_PRIMITIVE_TYPES[self.name] == 'f'
 
     def inspect_nonconst_type(self, block, inspect):
-        star_p1 = inspect.fetch_star_p1()
-        comment1 = inspect.get_comment(0, False, "a valid type", minlevel=1)
-        comment2 = inspect.get_comment(0, False, "an integer type")
-        comment3 = inspect.get_comment(0, False, "not declared 'const'")
-        block.writedecl("char b[sizeof(%s)];%s" % (star_p1, comment1))
-        if self.const:
-            block.writeline("memset(b, -1, sizeof(b));")
-        inspect.assign_to_p1("b")
-        block.writeline("(void)(%s << 1);%s" % (star_p1, comment2))
-        if not self.const:
-            block.writeline("%s = -1;%s" % (star_p1, comment3))
-        t1 = block.add_crx_type_var()
-        if self.is_integer_type():
-            block.writeline('%s = CRX_INT_TYPE(cb, %s, "%s");' % (
-                t1, star_p1, self.name))
-        elif self.is_char_type():
-            block.writeline('%s = cb->get_char_type(cb);' % (t1,))
-        elif self.is_float_type():
-            xxx
+        if inspect is None:
+            if self.is_signed_type():
+                expr = 'cb->get_signed_type(cb, sizeof(%s), "%s")' % (
+                    self.name, self.name)
+            elif self.is_unsigned_type():
+                expr = 'cb->get_unsigned_type(cb, sizeof(%s), "%s")' % (
+                    self.name, self.name)
+            elif self.is_char_type():
+                expr = 'cb->get_char_type(cb)'
+            elif self.is_float_type():
+                xxx
+            else:
+                raise AssertionError
         else:
-            raise AssertionError
-        return t1
+            star_p1 = inspect.fetch_star_p1()
+            comment1 = inspect.get_comment(0, False, "a valid type", minlevel=1)
+            comment2 = inspect.get_comment(0, False, "an integer type")
+            comment3 = inspect.get_comment(0, False, "not declared 'const'")
+            block.writedecl("char b[sizeof(%s)];%s" % (star_p1, comment1))
+            if self.const:
+                block.writeline("memset(b, -1, sizeof(b));")
+            inspect.assign_to_p1("b")
+            block.writeline("(void)(%s << 1);%s" % (star_p1, comment2))
+            if not self.const:
+                block.writeline("%s = -1;%s" % (star_p1, comment3))
+            if self.is_integer_type():
+                expr = 'CRX_INT_TYPE(cb, %s, "%s")' % (star_p1, self.name)
+            elif self.is_char_type():
+                expr = 'cb->get_char_type(cb)'
+            elif self.is_float_type():
+                xxx
+            else:
+                raise AssertionError
+        return block.write_crx_type_var(expr)
 
 
-class BaseFunctionType(BaseType):
-    pass
+class FunctionType(BaseType):
+    _attrs_ = ('args', 'result', 'ellipsis')
+    const = True
 
-class RawFunctionType(BaseFunctionType):
-    pass
+    def __init__(self, args, result, ellipsis):
+        self.args = args
+        self.result = result
+        self.ellipsis = ellipsis
+        #
+        reprargs = [arg.get_c_name() for arg in self.args]
+        if self.ellipsis:
+            reprargs.append('...')
+        reprargs = reprargs or ['void']
+        replace_with = '&(%s)' % (', '.join(reprargs),)
+        self.c_name_with_marker = (
+            self.result.c_name_with_marker.replace('&', replace_with))
+
+    def inspect_type(self, block, inspect):
+        # this class overrides inspect_type() instead of
+        # inspect_nonconst_type(), to avoid the extra call to get_const_type()
+        self._flush_inspect(inspect)
+        assert not self.ellipsis, "XXX"
+        t1 = self.result.inspect_type(block, None)
+        call = ['    ']
+        if not isinstance(self.result, VoidType):
+            call.append('*(%s)result = ' % (self.result.get_c_name('*'),))
+        call.append('f(')
+        t_args = [arg.inspect_type(block, None) for arg in self.args]
+        a2 = block.add_array_crx_types(len(t_args))
+        for i, t in enumerate(t_args):
+            block.writeline('%s[%d] = %s;' % (a2, i, t))
+            if i > 0:
+                call.append(', ')
+            call.append('*(%s)args[%d]' % (self.args[i].get_c_name('*'), i))
+        call.append(');')
+        #
+        toplevel = block.crx_top_level
+        crx_func_name = '%s__%s' % (toplevel.crx_func_name,
+                                    block._get_next_name('f'))
+        wrline = toplevel.writeline
+        wrline('static void %s(void *func, void *args[], void *result) {' % (
+            crx_func_name,))
+        wrline('    func_t f = func;')  # XXX!
+        wrline(''.join(call))
+        wrline('}')
+        wrline('')
+        expr = 'cb->get_function_type(cb, %s, %s, %d, &%s)' % (
+            t1, a2, len(t_args), crx_func_name)
+        return block.write_crx_type_var(expr)
+
 
 class PointerType(BaseType):
     _attrs_ = ('totype',)
+    _base_pattern       = {
+        (False, False): "*&",
+        (True, False): "(*&)",
+        (False, True): "*const &",
+        (True, True): "(*const &)",
+        }
 
     def __init__(self, totype, const):
         self.totype = totype
         self.const = const
         base = self.totype.c_name_with_marker
-        self.c_name_with_marker = base.replace('&', '(*&)')
+        parens = isinstance(self.totype, (ArrayType, FunctionType))
+        extra = self._base_pattern[parens, self.const]
+        self.c_name_with_marker = base.replace('&', extra)
 
     def inspect_nonconst_type(self, block, inspect):
-        star_p1 = inspect.fetch_star_p1()
-        new_var = 'p%d' % (len(inspect.levels) + 2,)
-        block.writedecl("char *%s;" % (new_var,))
-        inspect.assign_to_p1("&%s" % (new_var,))
-        #
-        if self.const:
-            errmsg = "type '%s' is not a pointer, but an array type" % (
-                inspect.get_comment_type(0, False),)
-            inspect.assign_target = new_var
-            def after():
-                ampersand_star_p1 = '&' + star_p1
-                if ampersand_star_p1.startswith('&*'):
-                    ampersand_star_p1 = ampersand_star_p1[2:]
-                block.writeline("if ((void *)%s == (void *)%s) {" % (
-                    ampersand_star_p1, star_p1))
-                block.writeline('    cb->error(cb, "%s");' % (errmsg,))
-                block.writeline("    return;")
-                block.writeline("}")
-            inspect.after_star_p1_assignment.append(after)
-        inspect.levels.append('*')
+        if inspect is not None:
+            star_p1 = inspect.fetch_star_p1()
+            new_var = 'p%d' % (len(inspect.levels) + 2,)
+            block.writedecl("char *%s;" % (new_var,))
+            inspect.assign_to_p1("&%s" % (new_var,))
+            #
+            if self.const:
+                errmsg = "type '%s' is not a pointer, but an array type" % (
+                    inspect.get_comment_type(0, False),)
+                inspect.assign_target = new_var
+                def after():
+                    ampersand_star_p1 = '&' + star_p1
+                    if ampersand_star_p1.startswith('&*'):
+                        ampersand_star_p1 = ampersand_star_p1[2:]
+                    block.writeline("if ((void *)%s == (void *)%s) {" % (
+                        ampersand_star_p1, star_p1))
+                    block.writeline('    cb->error(cb, "%s");' % (errmsg,))
+                    block.writeline("    return;")
+                    block.writeline("}")
+                inspect.after_star_p1_assignment.append(after)
+            inspect.levels.append('*')
         t1 = self.totype.inspect_type(block, inspect)
-        t2 = block.add_crx_type_var()
-        block.writeline('%s = cb->get_pointer_type(cb, %s);' % (t2, t1))
-        return t2
+        return block.write_crx_type_var('cb->get_pointer_type(cb, %s)' % t1)
 
 
 class ArrayType(BaseType):
     _attrs_ = ('item', 'length')
-    is_array_type = True
     const = True     # not really applicable, but must have one
 
     def __init__(self, item, length):
@@ -179,26 +246,25 @@
     def inspect_type(self, block, inspect):
         # this class overrides inspect_type() instead of
         # inspect_nonconst_type(), to avoid the extra call to get_const_type()
-        star_p1 = inspect.fetch_star_p1()
-        errmsg = "type '%s' is not an array, but a pointer type" % (
-            inspect.get_comment_type(0, False),)
-        def after():
-            ampersand_star_p1 = '&' + star_p1
-            if ampersand_star_p1.startswith('&*'):
-                ampersand_star_p1 = ampersand_star_p1[2:]
-            block.writeline("if ((void *)%s != (void *)%s) {" % (
-                ampersand_star_p1, star_p1))
-            block.writeline('    cb->error(cb, "%s");' % (errmsg,))
-            block.writeline("    return;")
-            block.writeline("}")
-        inspect.levels.append('[]')
-        inspect.after_star_p1_assignment.append(after)
+        if inspect is not None:
+            star_p1 = inspect.fetch_star_p1()
+            errmsg = "type '%s' is not an array, but a pointer type" % (
+                inspect.get_comment_type(0, False),)
+            def after():
+                ampersand_star_p1 = '&' + star_p1
+                if ampersand_star_p1.startswith('&*'):
+                    ampersand_star_p1 = ampersand_star_p1[2:]
+                block.writeline("if ((void *)%s != (void *)%s) {" % (
+                    ampersand_star_p1, star_p1))
+                block.writeline('    cb->error(cb, "%s");' % (errmsg,))
+                block.writeline("    return;")
+                block.writeline("}")
+            inspect.levels.append('[]')
+            inspect.after_star_p1_assignment.append(after)
         t1 = self.item.inspect_type(block, inspect)
-        t2 = block.add_crx_type_var()
-        block.writeline('%s = cb->get_array_type(cb, %s, '
-                        'sizeof(%s) / sizeof(*%s));' % (
-            t2, t1, star_p1, star_p1))
-        return t2
+        expr = 'cb->get_array_type(cb, %s, sizeof(%s) / sizeof(*%s));' % (
+            t1, star_p1, star_p1)
+        return block.write_crx_type_var(expr)
 
 
 class StructOrUnionOrEnum(BaseType):
diff --git a/test/cgcompile.c b/test/cgcompile.c
--- a/test/cgcompile.c
+++ b/test/cgcompile.c
@@ -80,9 +80,16 @@
 
 static crx_type_t *tst_get_function_type(crx_builder_t *cb, crx_type_t *ret,
                                          crx_type_t *args[], int nargs,
-                                         crx_trampoline_t *trampl)
+                                         crx_trampoline_fn trampl)
 {
-    abort();
+    int i;
+    const char *prev = "FUNC( ";
+    for (i = 0; i < nargs; i++) {
+        prev = newtype2(prev, args[i]->text)->text;
+        prev = newtype2(prev, " -> ")->text;
+    }
+    prev = newtype2(prev, ret->text)->text;
+    return newtype2(prev, " )");
 }
 
 static crx_type_t *tst_get_ellipsis_function_type(crx_builder_t *cb,
diff --git a/test/codegen/func-003.c b/test/codegen/func-003.c
--- a/test/codegen/func-003.c
+++ b/test/codegen/func-003.c
@@ -1,15 +1,15 @@
-typedef int(*func_t)(long, long);
+typedef int(*func_t)(long, long, int);
 
 # ____________________________________________________________
 
-static void __creflect_t3(void *func, void *args[], void *result) {
+static void testfunc_003__f4(void *func, void *args[], void *result) {
     func_t f = func;
-    *(int *)result = f(*(long *)args[0], *(long *)args[1]);
+    *(int *)result = f(*(long *)args[0], *(long *)args[1], *(int *)args[2]);
 }
 
 void testfunc_003(crx_builder_t *cb)
 {
-    crx_type *t1, *t2, *a3[2], *t3;
+    crx_type_t *t1, *t2, *a3[3], *t4, *t5;
     {
         func_t *p1;
         char *p2;
@@ -19,7 +19,10 @@
         t2 = cb->get_signed_type(cb, sizeof(long), "long");
         a3[0] = t2;
         a3[1] = t2;
-        t3 = cb->get_function_type(cb, t1, a3, 2, &__creflect_t3);
-        cb->define_type(cb, "func_t", t3);
+        a3[2] = t1;
+        t4 = cb->get_function_type(cb, t1, a3, 3, &testfunc_003__f4);
+        t5 = cb->get_pointer_type(cb, t4);
+        cb->define_type(cb, "func_t", t5);
+#expect TYPEDEF func_t = PTR FUNC( long -> long -> int -> int )
     }
 }

From noreply at buildbot.pypy.org  Tue Nov 18 00:48:52 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Tue, 18 Nov 2014 00:48:52 +0100 (CET)
Subject: [pypy-commit] creflect default: Properly lay out the current
	limitations
Message-ID: <20141117234852.E89C31C31CF@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r61:e79cbb0feeeb
Date: 2014-11-18 00:14 +0100
http://bitbucket.org/cffi/creflect/changeset/e79cbb0feeeb/

Log:	Properly lay out the current limitations

diff --git a/creflect/model.py b/creflect/model.py
--- a/creflect/model.py
+++ b/creflect/model.py
@@ -153,7 +153,12 @@
         # this class overrides inspect_type() instead of
         # inspect_nonconst_type(), to avoid the extra call to get_const_type()
         self._flush_inspect(inspect)
+
+        # limitations so far:
         assert not self.ellipsis, "XXX"
+        assert inspect is not None, "XXX"
+        assert inspect.levels == ['*'], "XXX"
+
         t1 = self.result.inspect_type(block, None)
         call = ['    ']
         if not isinstance(self.result, VoidType):
@@ -174,7 +179,7 @@
         wrline = toplevel.writeline
         wrline('static void %s(void *func, void *args[], void *result) {' % (
             crx_func_name,))
-        wrline('    func_t f = func;')  # XXX!
+        wrline('    %s f = func;' % inspect.typename)
         wrline(''.join(call))
         wrline('}')
         wrline('')

From noreply at buildbot.pypy.org  Tue Nov 18 00:48:54 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Tue, 18 Nov 2014 00:48:54 +0100 (CET)
Subject: [pypy-commit] creflect default: in-progress
Message-ID: <20141117234854.482DC1C31CF@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r62:3216c4cab225
Date: 2014-11-18 00:40 +0100
http://bitbucket.org/cffi/creflect/changeset/3216c4cab225/

Log:	in-progress

diff --git a/creflect/cparser.py b/creflect/cparser.py
--- a/creflect/cparser.py
+++ b/creflect/cparser.py
@@ -95,9 +95,9 @@
         if isinstance(node, pycparser.c_ast.FuncDecl):
             tp = self._get_type(node)
             assert isinstance(tp, model.FunctionType)
-            tp = model.PointerType(tp)
-            self._declare('function ' + decl.name, tp)
+            self.declarations.append(model.VarDecl(decl.name, tp))
         else:
+            xxxxxxxxxxxxxx
             const = 'const' in decl.quals
             if isinstance(node, pycparser.c_ast.Struct):
                 if node.decls is not None:
diff --git a/creflect/model.py b/creflect/model.py
--- a/creflect/model.py
+++ b/creflect/model.py
@@ -157,13 +157,20 @@
         # limitations so far:
         assert not self.ellipsis, "XXX"
         assert inspect is not None, "XXX"
-        assert inspect.levels == ['*'], "XXX"
+        if inspect.started:
+            assert inspect.levels == ['*'], "XXX"
+
+        if inspect.started:
+            localvar = 'f'
+        else:
+            assert inspect.varname is not None
+            localvar = inspect.varname
 
         t1 = self.result.inspect_type(block, None)
         call = ['    ']
         if not isinstance(self.result, VoidType):
             call.append('*(%s)result = ' % (self.result.get_c_name('*'),))
-        call.append('f(')
+        call.append('%s(' % localvar)
         t_args = [arg.inspect_type(block, None) for arg in self.args]
         a2 = block.add_array_crx_types(len(t_args))
         for i, t in enumerate(t_args):
@@ -174,12 +181,17 @@
         call.append(');')
         #
         toplevel = block.crx_top_level
-        crx_func_name = '%s__%s' % (toplevel.crx_func_name,
-                                    block._get_next_name('f'))
+        if inspect.varname is not None:
+            extraname = 'c_' + inspect.varname
+        else:
+            extraname = block._get_next_name('f')
+        crx_func_name = '%s__%s' % (toplevel.crx_func_name, extraname)
         wrline = toplevel.writeline
         wrline('static void %s(void *func, void *args[], void *result) {' % (
             crx_func_name,))
-        wrline('    %s f = func;' % inspect.typename)
+        if inspect.started:
+            assert inspect.levels == ['*']
+            wrline('    %s %s = func;' % (inspect.typename, localvar))
         wrline(''.join(call))
         wrline('}')
         wrline('')
@@ -288,9 +300,10 @@
 
 
 class TypeInspector(object):
-    def __init__(self, block, typename, structfield=None):
+    def __init__(self, block, typename, structfield=None, varname=None):
         self.block = block
         self.typename = typename
+        self.varname = varname
         self.structfield = structfield
         self.started = False
         if structfield is None:
@@ -466,3 +479,17 @@
         inspect.stop()
         block.writeline('cb->define_type(cb, "%s", %s);' % (self.name, t1))
         funcblock.write_subblock(block)
+
+
+class VarDecl(object):
+    def __init__(self, name, type):
+        self.name = name
+        self.type = type
+
+    def write_declaration(self, funcblock):
+        block = CodeBlock(funcblock)
+        inspect = TypeInspector(block, None, varname=self.name)
+        t1 = self.type.inspect_type(block, inspect)
+        inspect.stop()
+        block.writeline('cb->define_var(cb, "%s", %s, &xxx);' % (self.name, t1))
+        funcblock.write_subblock(block)
diff --git a/test/codegen/func-001.c b/test/codegen/func-001.c
--- a/test/codegen/func-001.c
+++ b/test/codegen/func-001.c
@@ -6,21 +6,21 @@
 
 # ____________________________________________________________
 
-static void __creflect_t2(void *func, void *args[], void *result) {
-    int(*f)(void) = func;
+static void testfunc_001__c_f(void *func, void *args[], void *result) {
     *(int *)result = f();
 }
 
-static int __creflect_d_f(void) {
+static int testfunc_001__d_f(void) {
     return f();
 }
 
 void testfunc_001(crx_builder_t *cb)
 {
-    crx_type *t1, *t2;
+    crx_type_t *t1, *t2;
     {
         t1 = cb->get_signed_type(cb, sizeof(int), "int");
-        t2 = cb->get_function_type(cb, t1, 0, 0, &__creflect_t2);
-        cb->define_var(cb, "f", t2, &__creflect_d_f);
+        t2 = cb->get_function_type(cb, t1, 0, 0, &testfunc_001__c_f);
+        cb->define_var(cb, "f", t2, &testfunc_001__d_f);
+#expect VAR f: FUNC( int )
     }
 }
diff --git a/test/test_codegen.py b/test/test_codegen.py
--- a/test/test_codegen.py
+++ b/test/test_codegen.py
@@ -33,7 +33,11 @@
         cdefblock.append(line)
     else:
         raise ValueError("marker '# _____' not found in %r" % (filename,))
-    expected = [line.rstrip('\n') for line in f]
+    expected = []
+    for line in f:
+        expected.append(line.rstrip('\n'))
+        if line.startswith('# _____'):
+            del expected[:]
     f.close()
     print
     #

From noreply at buildbot.pypy.org  Tue Nov 18 00:48:55 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Tue, 18 Nov 2014 00:48:55 +0100 (CET)
Subject: [pypy-commit] creflect default: Pass the base case of a global
	function declaration.
Message-ID: <20141117234855.71C4B1C31CF@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r63:364e932db77a
Date: 2014-11-18 00:49 +0100
http://bitbucket.org/cffi/creflect/changeset/364e932db77a/

Log:	Pass the base case of a global function declaration.

diff --git a/creflect/model.py b/creflect/model.py
--- a/creflect/model.py
+++ b/creflect/model.py
@@ -27,6 +27,9 @@
         if inspect is not None and inspect.started:
             inspect.assign_to_p1('0')
 
+    def shadow_global_var(self, top_level_block, name):
+        return name
+
     def __eq__(self, other):
         return (self.__class__ == other.__class__ and
                 self._get_items() == other._get_items())
@@ -199,6 +202,20 @@
             t1, a2, len(t_args), crx_func_name)
         return block.write_crx_type_var(expr)
 
+    def shadow_global_var(self, top_level_block, fnname):
+        shadowname = '%s__d_%s' % (top_level_block.crx_func_name, fnname)
+        wrline = top_level_block.writeline
+        args = [arg.get_c_name('a%d' % i) for i, arg in enumerate(self.args)]
+        decl = '%s(%s)' % (shadowname, ', '.join(args) or 'void')
+        wrline('static %s {' % self.result.get_c_name(decl))
+        args = ['a%d' % i for i, arg in enumerate(self.args)]
+        wrline('    %s%s(%s);' % ('' if isinstance(self.result, VoidType)
+                                  else 'return ',
+                                  fnname, ', '.join(args)))
+        wrline('}')
+        wrline('')
+        return shadowname
+
 
 class PointerType(BaseType):
     _attrs_ = ('totype',)
@@ -491,5 +508,7 @@
         inspect = TypeInspector(block, None, varname=self.name)
         t1 = self.type.inspect_type(block, inspect)
         inspect.stop()
-        block.writeline('cb->define_var(cb, "%s", %s, &xxx);' % (self.name, t1))
+        shadow = self.type.shadow_global_var(block.crx_top_level, self.name)
+        block.writeline('cb->define_var(cb, "%s", %s, &%s);' % (
+            self.name, t1, shadow))
         funcblock.write_subblock(block)

From noreply at buildbot.pypy.org  Tue Nov 18 00:49:13 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Tue, 18 Nov 2014 00:49:13 +0100 (CET)
Subject: [pypy-commit] creflect default: fix test
Message-ID: <20141117234913.50AC41C31CF@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r64:0db6ba186923
Date: 2014-11-18 00:49 +0100
http://bitbucket.org/cffi/creflect/changeset/0db6ba186923/

Log:	fix test

diff --git a/creflect/model.py b/creflect/model.py
--- a/creflect/model.py
+++ b/creflect/model.py
@@ -296,7 +296,7 @@
             inspect.levels.append('[]')
             inspect.after_star_p1_assignment.append(after)
         t1 = self.item.inspect_type(block, inspect)
-        expr = 'cb->get_array_type(cb, %s, sizeof(%s) / sizeof(*%s));' % (
+        expr = 'cb->get_array_type(cb, %s, sizeof(%s) / sizeof(*%s))' % (
             t1, star_p1, star_p1)
         return block.write_crx_type_var(expr)
 

From noreply at buildbot.pypy.org  Tue Nov 18 00:57:44 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Tue, 18 Nov 2014 00:57:44 +0100 (CET)
Subject: [pypy-commit] creflect default: Pass the next test
Message-ID: <20141117235744.37E0E1C31FC@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r65:f27c87f990ac
Date: 2014-11-18 00:58 +0100
http://bitbucket.org/cffi/creflect/changeset/f27c87f990ac/

Log:	Pass the next test

diff --git a/test/codegen/func-001b.c b/test/codegen/func-001b.c
--- a/test/codegen/func-001b.c
+++ b/test/codegen/func-001b.c
@@ -1,46 +1,63 @@
-unsigned int f(long long);
+unsigned int f(long);
 long g(int, int);
 long h(int, int);
 
 # ____________________________________________________________
 
-unsigned int f(long long a) { return (unsigned int)(a / 1291); }
+unsigned int f(long a) { return (unsigned int)(a / 1291); }
 long g(int a, int b) { return b - a; }
 long h(int a, int b) { return b * (long)a; }
 
 # ____________________________________________________________
 
-static void __creflect_t1(void *func, void *args[], void *result) {
-    unsigned int(*f)(long long) = func;
-    *(unsigned int *)result = f(*(long long *)args[0]);
+static void testfunc_001b__c_f(void *func, void *args[], void *result) {
+    *(unsigned int *)result = f(*(long *)args[0]);
 }
 
-static unsigned int __creflect_d_f(long long a0) {
+static unsigned int testfunc_001b__d_f(long a0) {
     return f(a0);
 }
 
-static void __creflect_t2(void *func, void *args[], void *result) {
-    long(*f)(int, int) = func;
-    *(long *)result = f(*(int *)args[0], *(int *)args[1]);
+static void testfunc_001b__c_g(void *func, void *args[], void *result) {
+    *(long *)result = g(*(int *)args[0], *(int *)args[1]);
 }
 
-static long __creflect_d_g(int a0, int a1) {
+static long testfunc_001b__d_g(int a0, int a1) {
     return g(a0, a1);
 }
 
-static long __creflect_d_h(int a0, int a1) {
+static void testfunc_001b__c_h(void *func, void *args[], void *result) {
+    *(long *)result = h(*(int *)args[0], *(int *)args[1]);
+}
+
+static long testfunc_001b__d_h(int a0, int a1) {
     return h(a0, a1);
 }
 
-int testfunc_002(char *r)
+void testfunc_001b(crx_builder_t *cb)
 {
-    if (!r)
-        return 3 + 999 + 1;
-    r += sprintf(r, "unsigned int");
-    r += sprintf(r, " /*%p*/f(long long)/*%p*/;\n", &__creflect_d_f, &__creflect_t1);
-    r += sprintf(r, "long");
-    r += sprintf(r, " /*%p*/g(int, int)/*%p*/;\n", &__creflect_d_g, &__creflect_t2);
-    r += sprintf(r, "long");
-    r += sprintf(r, " /*%p*/h(int, int)/*%p*/;\n", &__creflect_d_h, &__creflect_t2);
-    return 0;
+    crx_type_t *t1, *t2, *a3[1], *t4, *t5, *a6[2], *t7, *a8[2], *t9;
+    {
+        t1 = cb->get_unsigned_type(cb, sizeof(unsigned int), "unsigned int");
+        t2 = cb->get_signed_type(cb, sizeof(long), "long");
+        a3[0] = t2;
+        t4 = cb->get_function_type(cb, t1, a3, 1, &testfunc_001b__c_f);
+        cb->define_var(cb, "f", t4, &testfunc_001b__d_f);
+#expect VAR f: FUNC( long -> unsigned int )
+    }
+    {
+        t5 = cb->get_signed_type(cb, sizeof(int), "int");
+        a6[0] = t5;
+        a6[1] = t5;
+        t7 = cb->get_function_type(cb, t2, a6, 2, &testfunc_001b__c_g);
+        cb->define_var(cb, "g", t7, &testfunc_001b__d_g);
+#expect VAR g: FUNC( int -> int -> long )
+    }
+    {
+        a8[0] = t5;
+        a8[1] = t5;
+        t9 = cb->get_function_type(cb, t2, a8, 2, &testfunc_001b__c_h);
+        cb->define_var(cb, "h", t9, &testfunc_001b__d_h);
+#expect VAR h: FUNC( int -> int -> long )
+    }
 }

From noreply at buildbot.pypy.org  Tue Nov 18 01:16:52 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Tue, 18 Nov 2014 01:16:52 +0100 (CET)
Subject: [pypy-commit] creflect default: Streamline the declaration of
	functions.
Message-ID: <20141118001652.188301C0F1B@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r66:9cc3ae087730
Date: 2014-11-18 01:17 +0100
http://bitbucket.org/cffi/creflect/changeset/9cc3ae087730/

Log:	Streamline the declaration of functions.

diff --git a/creflect/cparser.py b/creflect/cparser.py
--- a/creflect/cparser.py
+++ b/creflect/cparser.py
@@ -95,7 +95,7 @@
         if isinstance(node, pycparser.c_ast.FuncDecl):
             tp = self._get_type(node)
             assert isinstance(tp, model.FunctionType)
-            self.declarations.append(model.VarDecl(decl.name, tp))
+            self.declarations.append(model.FuncDecl(decl.name, tp))
         else:
             xxxxxxxxxxxxxx
             const = 'const' in decl.quals
diff --git a/creflect/creflect.h b/creflect/creflect.h
--- a/creflect/creflect.h
+++ b/creflect/creflect.h
@@ -10,7 +10,8 @@
 } crx_field_t;
 
 typedef struct crx_type_s crx_type_t;   /* opaque */
-typedef void (*crx_trampoline_fn)(void *fn, void *args[], void *res);
+typedef void (*crx_trampoline0_fn)(void *args[], void *res);
+typedef void (*crx_trampoline1_fn)(void *fn, void *args[], void *res);
 
 #define CRX_SELF struct _crx_builder_s *
 typedef struct _crx_builder_s {
@@ -24,7 +25,7 @@
     crx_type_t *(*get_float_type)(CRX_SELF, size_t,
                                const char *);
     crx_type_t *(*get_function_type)(CRX_SELF, crx_type_t *,
-                                     crx_type_t *[], int, crx_trampoline_fn);
+                                     crx_type_t *[], int, crx_trampoline1_fn);
     crx_type_t *(*get_ellipsis_function_type)(CRX_SELF, crx_type_t *,
                                               crx_type_t *[], int);
     crx_type_t *(*get_pointer_type)(CRX_SELF, crx_type_t *);
@@ -40,6 +41,8 @@
     void (*define_type)(CRX_SELF, const char *, crx_type_t *);
     void (*define_var)(CRX_SELF, const char *, crx_type_t *,
                        void *);
+    void (*define_func)(CRX_SELF, const char *, crx_type_t *,
+                        crx_type_t *[], int, crx_trampoline0_fn, void *);
     void (*error)(CRX_SELF, const char *);
 } crx_builder_t;
 #undef CRX_SELF
diff --git a/creflect/model.py b/creflect/model.py
--- a/creflect/model.py
+++ b/creflect/model.py
@@ -160,20 +160,14 @@
         # limitations so far:
         assert not self.ellipsis, "XXX"
         assert inspect is not None, "XXX"
-        if inspect.started:
-            assert inspect.levels == ['*'], "XXX"
-
-        if inspect.started:
-            localvar = 'f'
-        else:
-            assert inspect.varname is not None
-            localvar = inspect.varname
+        assert inspect.started, "XXX"
+        assert inspect.levels == ['*'], "XXX"
 
         t1 = self.result.inspect_type(block, None)
         call = ['    ']
         if not isinstance(self.result, VoidType):
             call.append('*(%s)result = ' % (self.result.get_c_name('*'),))
-        call.append('%s(' % localvar)
+        call.append('f(')
         t_args = [arg.inspect_type(block, None) for arg in self.args]
         a2 = block.add_array_crx_types(len(t_args))
         for i, t in enumerate(t_args):
@@ -194,7 +188,7 @@
             crx_func_name,))
         if inspect.started:
             assert inspect.levels == ['*']
-            wrline('    %s %s = func;' % (inspect.typename, localvar))
+            wrline('    %s f = func;' % inspect.typename)
         wrline(''.join(call))
         wrline('}')
         wrline('')
@@ -216,6 +210,34 @@
         wrline('')
         return shadowname
 
+    def write_func_decl(self, block, fnname):
+        # XXX code duplication!
+        t1 = self.result.inspect_type(block, None)
+        call = ['    ']
+        if not isinstance(self.result, VoidType):
+            call.append('*(%s)result = ' % (self.result.get_c_name('*'),))
+        call.append('%s(' % fnname)
+        t_args = [arg.inspect_type(block, None) for arg in self.args]
+        a2 = block.add_array_crx_types(len(t_args))
+        for i, t in enumerate(t_args):
+            block.writeline('%s[%d] = %s;' % (a2, i, t))
+            if i > 0:
+                call.append(', ')
+            call.append('*(%s)args[%d]' % (self.args[i].get_c_name('*'), i))
+        call.append(');')
+        #
+        toplevel = block.crx_top_level
+        crx_func_name = '%s__c_%s' % (toplevel.crx_func_name, fnname)
+        wrline = toplevel.writeline
+        wrline('static void %s(void *args[], void *result) {' % (
+            crx_func_name,))
+        wrline(''.join(call))
+        wrline('}')
+        wrline('')
+        shadow = self.shadow_global_var(toplevel, fnname)
+        block.writeline('cb->define_func(cb, "%s", %s, %s, %d, &%s, &%s);' % (
+            fnname, t1, a2, len(t_args), crx_func_name, shadow))
+
 
 class PointerType(BaseType):
     _attrs_ = ('totype',)
@@ -512,3 +534,14 @@
         block.writeline('cb->define_var(cb, "%s", %s, &%s);' % (
             self.name, t1, shadow))
         funcblock.write_subblock(block)
+
+
+class FuncDecl(object):
+    def __init__(self, name, type):
+        self.name = name
+        self.type = type
+
+    def write_declaration(self, funcblock):
+        block = CodeBlock(funcblock)
+        self.type.write_func_decl(block, self.name)
+        funcblock.write_subblock(block)
diff --git a/test/cgcompile.c b/test/cgcompile.c
--- a/test/cgcompile.c
+++ b/test/cgcompile.c
@@ -80,7 +80,7 @@
 
 static crx_type_t *tst_get_function_type(crx_builder_t *cb, crx_type_t *ret,
                                          crx_type_t *args[], int nargs,
-                                         crx_trampoline_fn trampl)
+                                         crx_trampoline1_fn trampl)
 {
     int i;
     const char *prev = "FUNC( ";
@@ -162,6 +162,17 @@
     printf("VAR %s: %s\n", name, t->text);
 }
 
+static void tst_define_func(crx_builder_t *cb, const char *name,
+                            crx_type_t *ret, crx_type_t *args[], int nargs,
+                            crx_trampoline0_fn trampl, void *directcall)
+{
+    int i;
+    printf("FUNC %s: ", name);
+    for (i = 0; i < nargs; i++)
+        printf("%s -> ", args[i]->text);
+    printf("%s\n", ret->text);
+}
+
 static void tst_error(crx_builder_t *cb, const char *msg)
 {
     printf("ERROR: %s\n", msg);
@@ -187,6 +198,7 @@
     tst_complete_enum,
     tst_define_type,
     tst_define_var,
+    tst_define_func,
     tst_error,
 };
 
diff --git a/test/codegen/func-001.c b/test/codegen/func-001.c
--- a/test/codegen/func-001.c
+++ b/test/codegen/func-001.c
@@ -6,7 +6,7 @@
 
 # ____________________________________________________________
 
-static void testfunc_001__c_f(void *func, void *args[], void *result) {
+static void testfunc_001__c_f(void *args[], void *result) {
     *(int *)result = f();
 }
 
@@ -16,11 +16,10 @@
 
 void testfunc_001(crx_builder_t *cb)
 {
-    crx_type_t *t1, *t2;
+    crx_type_t *t1;
     {
         t1 = cb->get_signed_type(cb, sizeof(int), "int");
-        t2 = cb->get_function_type(cb, t1, 0, 0, &testfunc_001__c_f);
-        cb->define_var(cb, "f", t2, &testfunc_001__d_f);
-#expect VAR f: FUNC( int )
+        cb->define_func(cb, "f", t1, 0, 0, &testfunc_001__c_f, &testfunc_001__d_f);
+#expect FUNC f: int
     }
 }
diff --git a/test/codegen/func-001b.c b/test/codegen/func-001b.c
--- a/test/codegen/func-001b.c
+++ b/test/codegen/func-001b.c
@@ -10,7 +10,7 @@
 
 # ____________________________________________________________
 
-static void testfunc_001b__c_f(void *func, void *args[], void *result) {
+static void testfunc_001b__c_f(void *args[], void *result) {
     *(unsigned int *)result = f(*(long *)args[0]);
 }
 
@@ -18,7 +18,7 @@
     return f(a0);
 }
 
-static void testfunc_001b__c_g(void *func, void *args[], void *result) {
+static void testfunc_001b__c_g(void *args[], void *result) {
     *(long *)result = g(*(int *)args[0], *(int *)args[1]);
 }
 
@@ -26,7 +26,7 @@
     return g(a0, a1);
 }
 
-static void testfunc_001b__c_h(void *func, void *args[], void *result) {
+static void testfunc_001b__c_h(void *args[], void *result) {
     *(long *)result = h(*(int *)args[0], *(int *)args[1]);
 }
 
@@ -36,28 +36,25 @@
 
 void testfunc_001b(crx_builder_t *cb)
 {
-    crx_type_t *t1, *t2, *a3[1], *t4, *t5, *a6[2], *t7, *a8[2], *t9;
+    crx_type_t *t1, *t2, *a3[1], *t4, *a5[2], *a6[2];
     {
         t1 = cb->get_unsigned_type(cb, sizeof(unsigned int), "unsigned int");
         t2 = cb->get_signed_type(cb, sizeof(long), "long");
         a3[0] = t2;
-        t4 = cb->get_function_type(cb, t1, a3, 1, &testfunc_001b__c_f);
-        cb->define_var(cb, "f", t4, &testfunc_001b__d_f);
-#expect VAR f: FUNC( long -> unsigned int )
+        cb->define_func(cb, "f", t1, a3, 1, &testfunc_001b__c_f, &testfunc_001b__d_f);
+#expect FUNC f: long -> unsigned int
     }
     {
-        t5 = cb->get_signed_type(cb, sizeof(int), "int");
-        a6[0] = t5;
-        a6[1] = t5;
-        t7 = cb->get_function_type(cb, t2, a6, 2, &testfunc_001b__c_g);
-        cb->define_var(cb, "g", t7, &testfunc_001b__d_g);
-#expect VAR g: FUNC( int -> int -> long )
+        t4 = cb->get_signed_type(cb, sizeof(int), "int");
+        a5[0] = t4;
+        a5[1] = t4;
+        cb->define_func(cb, "g", t2, a5, 2, &testfunc_001b__c_g, &testfunc_001b__d_g);
+#expect FUNC g: int -> int -> long
     }
     {
-        a8[0] = t5;
-        a8[1] = t5;
-        t9 = cb->get_function_type(cb, t2, a8, 2, &testfunc_001b__c_h);
-        cb->define_var(cb, "h", t9, &testfunc_001b__d_h);
-#expect VAR h: FUNC( int -> int -> long )
+        a6[0] = t4;
+        a6[1] = t4;
+        cb->define_func(cb, "h", t2, a6, 2, &testfunc_001b__c_h, &testfunc_001b__d_h);
+#expect FUNC h: int -> int -> long
     }
 }

From noreply at buildbot.pypy.org  Tue Nov 18 03:06:26 2014
From: noreply at buildbot.pypy.org (rlamy)
Date: Tue, 18 Nov 2014 03:06:26 +0100 (CET)
Subject: [pypy-commit] pypy expressions-2: Hide c_last_exception as an
 implementation detail of Block
Message-ID: <20141118020626.59B761C34C4@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy 
Branch: expressions-2
Changeset: r74567:8179c5533982
Date: 2014-11-18 02:06 +0000
http://bitbucket.org/pypy/pypy/changeset/8179c5533982/

Log:	Hide c_last_exception as an implementation detail of Block

diff --git a/rpython/annotator/annrpython.py b/rpython/annotator/annrpython.py
--- a/rpython/annotator/annrpython.py
+++ b/rpython/annotator/annrpython.py
@@ -6,8 +6,8 @@
 from rpython.tool.pairtype import pair
 from rpython.tool.error import (format_blocked_annotation_error,
                              gather_error, source_lines)
-from rpython.flowspace.model import (Variable, Constant, FunctionGraph,
-                                      c_last_exception, checkgraph)
+from rpython.flowspace.model import (
+    Variable, Constant, FunctionGraph, checkgraph)
 from rpython.translator import simplify, transform
 from rpython.annotator import model as annmodel, signature
 from rpython.annotator.argument import simple_args
@@ -407,8 +407,7 @@
                     self.bookkeeper.leave()
 
         except BlockedInference as e:
-            if (e.op is block.operations[-1] and
-                block.exitswitch == c_last_exception):
+            if (e.op is block.operations[-1] and block.canraise):
                 # this is the case where the last operation of the block will
                 # always raise an exception which is immediately caught by
                 # an exception handler.  We then only follow the exceptional
@@ -450,8 +449,8 @@
 
         # filter out those exceptions which cannot
         # occour for this specific, typed operation.
-        if block.exitswitch == c_last_exception:
-            op = block.operations[-1]
+        if block.canraise:
+            op = block.raising_op
             can_only_throw = op.get_can_only_throw(self)
             if can_only_throw is not None:
                 candidates = can_only_throw
diff --git a/rpython/flowspace/model.py b/rpython/flowspace/model.py
--- a/rpython/flowspace/model.py
+++ b/rpython/flowspace/model.py
@@ -153,6 +153,13 @@
 
     view = show
 
+last_exception = Atom('last_exception')
+c_last_exception = Constant(last_exception)
+# if Block().exitswitch == Constant(last_exception), it means that we are
+# interested in catching the exception that the *last operation* of the
+# block could raise.  The exitcases of the links are None for no exception
+# or XxxError classes to catch the matching exceptions.
+
 
 class Block(object):
     __slots__ = """inputargs operations exitswitch
@@ -187,6 +194,15 @@
             txt = "%s(%s)" % (txt, self.exitswitch)
         return txt
 
+    @property
+    def canraise(self):
+        return self.exitswitch is c_last_exception
+
+    @property
+    def raising_op(self):
+        if self.canraise:
+            return self.operations[-1]
+
     def getvariables(self):
         "Return all variables mentioned in this Block."
         result = self.inputargs[:]
@@ -453,13 +469,6 @@
     def __repr__(self):
         return self.__name__
 
-last_exception = Atom('last_exception')
-c_last_exception = Constant(last_exception)
-# if Block().exitswitch == Constant(last_exception), it means that we are
-# interested in catching the exception that the *last operation* of the
-# block could raise.  The exitcases of the links are None for no exception
-# or XxxError classes to catch the matching exceptions.
-
 def uniqueitems(lst):
     "Returns a list with duplicate elements removed."
     result = []
@@ -613,11 +622,11 @@
                 assert len(block.exits) <= 1
                 if block.exits:
                     assert block.exits[0].exitcase is None
-            elif block.exitswitch == Constant(last_exception):
+            elif block.canraise:
                 assert len(block.operations) >= 1
                 # check if an exception catch is done on a reasonable
                 # operation
-                assert block.operations[-1].opname not in ("keepalive",
+                assert block.raising_op.opname not in ("keepalive",
                                                            "cast_pointer",
                                                            "same_as")
                 assert len(block.exits) >= 2
diff --git a/rpython/flowspace/test/test_objspace.py b/rpython/flowspace/test/test_objspace.py
--- a/rpython/flowspace/test/test_objspace.py
+++ b/rpython/flowspace/test/test_objspace.py
@@ -4,7 +4,7 @@
 from contextlib import contextmanager
 
 from rpython.flowspace.model import (
-    Constant, mkentrymap, c_last_exception, const)
+    Constant, mkentrymap, const)
 from rpython.translator.simplify import simplify_graph
 from rpython.flowspace.objspace import build_flow
 from rpython.flowspace.flowcontext import FlowingError, FlowContext
@@ -826,7 +826,7 @@
                 return None
         graph = self.codetest(myfunc)
         simplify_graph(graph)
-        assert graph.startblock.exitswitch == c_last_exception
+        assert graph.startblock.canraise
         assert graph.startblock.exits[0].target is graph.returnblock
         assert graph.startblock.exits[1].target is graph.returnblock
 
diff --git a/rpython/jit/codewriter/flatten.py b/rpython/jit/codewriter/flatten.py
--- a/rpython/jit/codewriter/flatten.py
+++ b/rpython/jit/codewriter/flatten.py
@@ -1,4 +1,4 @@
-from rpython.flowspace.model import Variable, Constant, c_last_exception
+from rpython.flowspace.model import Variable, Constant
 from rpython.jit.metainterp.history import AbstractDescr, getkind
 from rpython.rtyper.lltypesystem import lltype
 
@@ -167,7 +167,7 @@
             # up in the manually hacked graphs for generators...
             self.make_link(link)
         #
-        elif block.exitswitch is c_last_exception:
+        elif block.canraise:
             # An exception block. See test_exc_exitswitch in test_flatten.py
             # for an example of what kind of code this makes.
             index = -1
@@ -184,7 +184,7 @@
                     # actually a '-live-'
                     self.make_link(block.exits[0])
                     return
-            # 
+            #
             self.emitline('catch_exception', TLabel(block.exits[0]))
             self.make_link(block.exits[0])
             self.emitline(Label(block.exits[0]))
diff --git a/rpython/jit/codewriter/jtransform.py b/rpython/jit/codewriter/jtransform.py
--- a/rpython/jit/codewriter/jtransform.py
+++ b/rpython/jit/codewriter/jtransform.py
@@ -8,7 +8,7 @@
 from rpython.jit.metainterp.history import getkind
 from rpython.jit.metainterp.typesystem import deref, arrayItem
 from rpython.jit.metainterp.blackhole import BlackholeInterpreter
-from rpython.flowspace.model import SpaceOperation, Variable, Constant, c_last_exception
+from rpython.flowspace.model import SpaceOperation, Variable, Constant
 from rpython.rlib import objectmodel
 from rpython.rlib.jit import _we_are_jitted
 from rpython.rlib.rgc import lltype_is_gc
@@ -110,7 +110,7 @@
                 else:
                     raise TypeError(repr(op1))
         #
-        if block.exitswitch == c_last_exception:
+        if block.canraise:
             if len(newoperations) == count_before_last_operation:
                 self._killed_exception_raising_operation(block)
         block.operations = newoperations
@@ -175,7 +175,7 @@
 
     def follow_constant_exit(self, block):
         v = block.exitswitch
-        if isinstance(v, Constant) and v != c_last_exception:
+        if isinstance(v, Constant) and not block.canraise:
             llvalue = v.value
             for link in block.exits:
                 if link.llexitcase == llvalue:
@@ -192,7 +192,7 @@
         if len(block.exits) != 2:
             return False
         v = block.exitswitch
-        if (v == c_last_exception or isinstance(v, tuple)
+        if (block.canraise or isinstance(v, tuple)
             or v.concretetype != lltype.Bool):
             return False
         for op in block.operations[::-1]:
diff --git a/rpython/memory/gctransform/test/test_transform.py b/rpython/memory/gctransform/test/test_transform.py
--- a/rpython/memory/gctransform/test/test_transform.py
+++ b/rpython/memory/gctransform/test/test_transform.py
@@ -1,5 +1,5 @@
 from rpython.memory.gctransform.transform import BaseGCTransformer
-from rpython.flowspace.model import c_last_exception, Variable
+from rpython.flowspace.model import Variable
 from rpython.translator.backendopt.support import var_needsgc
 from rpython.translator.translator import TranslationContext, graphof
 from rpython.translator.exceptiontransform import ExceptionTransformer
@@ -119,8 +119,8 @@
     if pop_alives == len(block.operations):
         # it's a block we inserted
         return
+    assert not block.canraise
     for link in block.exits:
-        assert block.exitswitch is not c_last_exception
         refs_out = 0
         for v2 in link.target.inputargs:
             if var_needsgc(v2) and not is_borrowed(v2):
diff --git a/rpython/memory/gctransform/transform.py b/rpython/memory/gctransform/transform.py
--- a/rpython/memory/gctransform/transform.py
+++ b/rpython/memory/gctransform/transform.py
@@ -1,6 +1,6 @@
 from rpython.rtyper.lltypesystem import lltype, llmemory
-from rpython.flowspace.model import SpaceOperation, Variable, Constant, \
-     c_last_exception, checkgraph
+from rpython.flowspace.model import (
+    SpaceOperation, Variable, Constant, checkgraph)
 from rpython.translator.unsimplify import insert_empty_block
 from rpython.translator.unsimplify import insert_empty_startblock
 from rpython.translator.unsimplify import starts_with_empty_block
@@ -180,7 +180,7 @@
             hop.dispatch()
 
         if len(block.exits) != 0: # i.e not the return block
-            assert block.exitswitch is not c_last_exception
+            assert not block.canraise
 
             deadinallexits = set(self.livevars)
             for link in block.exits:
diff --git a/rpython/rtyper/llinterp.py b/rpython/rtyper/llinterp.py
--- a/rpython/rtyper/llinterp.py
+++ b/rpython/rtyper/llinterp.py
@@ -5,8 +5,7 @@
 
 import py
 
-from rpython.flowspace.model import (FunctionGraph, Constant, Variable,
-    c_last_exception)
+from rpython.flowspace.model import (FunctionGraph, Constant, Variable)
 from rpython.rlib import rstackovf
 from rpython.rlib.objectmodel import (ComputedIntSymbolic, CDefinedIntSymbolic,
     Symbolic)
@@ -271,7 +270,6 @@
             is None, values is the concrete return value.
         """
         self.curr_block = block
-        catch_exception = block.exitswitch == c_last_exception
         e = None
 
         try:
@@ -279,7 +277,7 @@
                 self.curr_operation_index = i
                 self.eval_operation(op)
         except LLException, e:
-            if not (catch_exception and op is block.operations[-1]):
+            if op is not block.raising_op:
                 raise
         except RuntimeError, e:
             rstackovf.check_stack_overflow()
@@ -291,7 +289,7 @@
             evalue = exdata.get_standard_ll_exc_instance(rtyper, classdef)
             etype = exdata.fn_type_of_exc_inst(evalue)
             e = LLException(etype, evalue)
-            if not (catch_exception and op is block.operations[-1]):
+            if op is not block.raising_op:
                 raise e
 
         # determine nextblock and/or return value
@@ -333,7 +331,7 @@
             # single-exit block
             assert len(block.exits) == 1
             link = block.exits[0]
-        elif catch_exception:
+        elif block.canraise:
             link = block.exits[0]
             if e:
                 exdata = self.llinterpreter.typer.exceptiondata
diff --git a/rpython/rtyper/rtyper.py b/rpython/rtyper/rtyper.py
--- a/rpython/rtyper/rtyper.py
+++ b/rpython/rtyper/rtyper.py
@@ -17,7 +17,7 @@
 
 from rpython.annotator import model as annmodel, unaryop, binaryop
 from rpython.rtyper.llannotation import SomePtr, lltype_to_annotation
-from rpython.flowspace.model import Variable, Constant, SpaceOperation, c_last_exception
+from rpython.flowspace.model import Variable, Constant, SpaceOperation
 from rpython.rtyper.annlowlevel import annotate_lowlevel_helper, LowLevelAnnotatorPolicy
 from rpython.rtyper.error import TyperError
 from rpython.rtyper.exceptiondata import ExceptionData
@@ -352,7 +352,7 @@
         if (pos is not None and pos != len(newops) - 1):
             # this is for the case where the llop that raises the exceptions
             # is not the last one in the list.
-            assert block.exitswitch == c_last_exception
+            assert block.canraise
             noexclink = block.exits[0]
             assert noexclink.exitcase is None
             if pos == "removed":
@@ -389,7 +389,7 @@
             if isinstance(block.exitswitch, Variable):
                 r_case = self.bindingrepr(block.exitswitch)
             else:
-                assert block.exitswitch == c_last_exception
+                assert block.canraise
                 r_case = rclass.get_type_repr(self)
             link.llexitcase = r_case.convert_const(link.exitcase)
         else:
@@ -460,7 +460,7 @@
             for op in block.operations[:-1]:
                 yield HighLevelOp(self, op, [], llops)
             # look for exception links for the last operation
-            if block.exitswitch == c_last_exception:
+            if block.canraise:
                 exclinks = block.exits[1:]
             else:
                 exclinks = []
diff --git a/rpython/translator/backendopt/constfold.py b/rpython/translator/backendopt/constfold.py
--- a/rpython/translator/backendopt/constfold.py
+++ b/rpython/translator/backendopt/constfold.py
@@ -1,5 +1,5 @@
 from rpython.flowspace.model import (Constant, Variable, SpaceOperation,
-    c_last_exception, mkentrymap)
+    mkentrymap)
 from rpython.rtyper.lltypesystem import lltype
 from rpython.rtyper.lltypesystem.lloperation import llop
 from rpython.translator.unsimplify import insert_empty_block, split_block
@@ -66,7 +66,7 @@
 def constant_fold_block(block):
     constants = {}
     block.operations = fold_op_list(block.operations, constants,
-                           exc_catch=block.exitswitch == c_last_exception)
+                                    exc_catch=block.canraise)
     if constants:
         if block.exitswitch in constants:
             switch = constants[block.exitswitch].value
@@ -155,7 +155,7 @@
     folded_count = fold_op_list(block.operations, constants, exit_early=True)
 
     n = len(block.operations)
-    if block.exitswitch == c_last_exception:
+    if block.canraise:
         n -= 1
     # is the next, non-folded operation an indirect_call?
     if folded_count < n:
diff --git a/rpython/translator/backendopt/inline.py b/rpython/translator/backendopt/inline.py
--- a/rpython/translator/backendopt/inline.py
+++ b/rpython/translator/backendopt/inline.py
@@ -1,7 +1,7 @@
 import sys
 
 from rpython.flowspace.model import (Variable, Constant, Block, Link,
-    SpaceOperation, c_last_exception, FunctionGraph, mkentrymap)
+    SpaceOperation, FunctionGraph, mkentrymap)
 from rpython.rtyper.lltypesystem.lltype import Bool, Signed, typeOf, Void, Ptr, normalizeptr
 from rpython.tool.algo import sparsemat
 from rpython.translator.backendopt import removenoops
@@ -112,7 +112,7 @@
     for block in graph.iterblocks():
         if block is graph.exceptblock:
             return True      # the except block is reachable
-        if block.exitswitch == c_last_exception:
+        if block.canraise:
             consider_ops_to = -1
         else:
             consider_ops_to = len(block.operations)
@@ -132,7 +132,7 @@
         else:
             return True # conservatively
     for block in from_graph.iterblocks():
-        if block.exitswitch == c_last_exception:
+        if block.canraise:
             consider_ops_to = -1
         else:
             consider_ops_to = len(block.operations)
@@ -196,8 +196,7 @@
         self.op = block.operations[index_operation]
         self.graph_to_inline = self.get_graph_from_op(self.op)
         self.exception_guarded = False
-        if (block.exitswitch == c_last_exception and
-            index_operation == len(block.operations) - 1):
+        if (self.op is block.raising_op):
             self.exception_guarded = True
             if self.inline_guarded_calls:
                 if (not self.inline_guarded_calls_no_matter_what and
diff --git a/rpython/translator/backendopt/mallocv.py b/rpython/translator/backendopt/mallocv.py
--- a/rpython/translator/backendopt/mallocv.py
+++ b/rpython/translator/backendopt/mallocv.py
@@ -1,7 +1,6 @@
 from rpython.flowspace.model import Variable, Constant, Block, Link
 from rpython.flowspace.model import SpaceOperation, copygraph
 from rpython.flowspace.model import checkgraph
-from rpython.flowspace.model import c_last_exception
 from rpython.translator.backendopt.support import log
 from rpython.translator.simplify import join_blocks
 from rpython.translator.unsimplify import varoftype
@@ -534,7 +533,7 @@
             return None
 
     def has_exception_catching(self, catchingframe):
-        if catchingframe.sourceblock.exitswitch != c_last_exception:
+        if not catchingframe.sourceblock.canraise:
             return False
         else:
             operations = catchingframe.sourceblock.operations
@@ -711,7 +710,7 @@
         self.specblock.exitswitch = self.rename_nonvirtual(block.exitswitch,
                                                            'exitswitch')
         links = block.exits
-        catch_exc = self.specblock.exitswitch == c_last_exception
+        catch_exc = self.specblock.canraise
 
         if not catch_exc and isinstance(self.specblock.exitswitch, Constant):
             # constant-fold the switch
diff --git a/rpython/translator/backendopt/removeassert.py b/rpython/translator/backendopt/removeassert.py
--- a/rpython/translator/backendopt/removeassert.py
+++ b/rpython/translator/backendopt/removeassert.py
@@ -1,4 +1,4 @@
-from rpython.flowspace.model import Constant, checkgraph, c_last_exception
+from rpython.flowspace.model import Constant, checkgraph
 from rpython.rtyper.lltypesystem import lltype
 from rpython.rtyper.rtyper import LowLevelOpList, inputconst
 from rpython.translator.backendopt.support import log
@@ -59,7 +59,7 @@
     if len(exits) <= 1:
         return False
     remove_condition = len(exits) == 2
-    if block.exitswitch == c_last_exception:
+    if block.canraise:
         if link is exits[0]:
             return False       # cannot remove the non-exceptional path
     else:
diff --git a/rpython/translator/c/funcgen.py b/rpython/translator/c/funcgen.py
--- a/rpython/translator/c/funcgen.py
+++ b/rpython/translator/c/funcgen.py
@@ -3,8 +3,7 @@
 from rpython.translator.c.support import cdecl
 from rpython.translator.c.support import llvalue_from_constant, gen_assignments
 from rpython.translator.c.support import c_string_constant, barebonearray
-from rpython.flowspace.model import Variable, Constant
-from rpython.flowspace.model import c_last_exception, copygraph
+from rpython.flowspace.model import Variable, Constant, copygraph
 from rpython.rtyper.lltypesystem.lltype import (Ptr, Void, Bool, Signed, Unsigned,
     SignedLongLong, Float, UnsignedLongLong, Char, UniChar, ContainerType,
     Array, FixedSizeArray, ForwardReference, FuncType)
@@ -232,7 +231,7 @@
                 for op in self.gen_link(block.exits[0]):
                     yield op
             else:
-                assert block.exitswitch != c_last_exception
+                assert not block.canraise
                 # block ending in a switch on a value
                 TYPE = self.lltypemap(block.exitswitch)
                 if TYPE == Bool:
diff --git a/rpython/translator/exceptiontransform.py b/rpython/translator/exceptiontransform.py
--- a/rpython/translator/exceptiontransform.py
+++ b/rpython/translator/exceptiontransform.py
@@ -3,7 +3,7 @@
 from rpython.translator.unsimplify import insert_empty_block, split_block
 from rpython.translator.backendopt import canraise, inline
 from rpython.flowspace.model import Block, Constant, Variable, Link, \
-    c_last_exception, SpaceOperation, FunctionGraph, mkentrymap
+    SpaceOperation, FunctionGraph, mkentrymap
 from rpython.rtyper.lltypesystem import lltype, llmemory, rffi
 from rpython.rtyper.lltypesystem import lloperation
 from rpython.rtyper.rclass import ll_inst_type
@@ -243,7 +243,7 @@
         elif block is graph.returnblock:
             return need_exc_matching, n_gen_exc_checks
         last_operation = len(block.operations) - 1
-        if block.exitswitch == c_last_exception:
+        if block.canraise:
             need_exc_matching = True
             last_operation -= 1
         elif (len(block.exits) == 1 and
@@ -267,7 +267,7 @@
             self.gen_exc_check(block, graph.returnblock, afterblock)
             n_gen_exc_checks += 1
         if need_exc_matching:
-            assert lastblock.exitswitch == c_last_exception
+            assert lastblock.canraise
             if not self.raise_analyzer.can_raise(lastblock.operations[-1]):
                 lastblock.exitswitch = None
                 lastblock.recloseblock(lastblock.exits[0])
diff --git a/rpython/translator/simplify.py b/rpython/translator/simplify.py
--- a/rpython/translator/simplify.py
+++ b/rpython/translator/simplify.py
@@ -8,8 +8,8 @@
 from collections import defaultdict
 
 from rpython.tool.algo.unionfind import UnionFind
-from rpython.flowspace.model import (Variable, Constant,
-                                     c_last_exception, checkgraph, mkentrymap)
+from rpython.flowspace.model import (
+        Variable, Constant, checkgraph, mkentrymap)
 from rpython.flowspace.operation import OverflowingOperation, op
 from rpython.flowspace.expression import V_Type
 from rpython.rlib import rarithmetic
@@ -44,8 +44,6 @@
 
 
 def replace_exitswitch_by_constant(block, const):
-    assert isinstance(const, Constant)
-    assert const != c_last_exception
     newexits = [link for link in block.exits
                      if link.exitcase == const.value]
     if len(newexits) == 0:
@@ -127,13 +125,12 @@
     chain of is_/issubtype tests. We collapse them all into
     the block's single list of exits.
     """
-    clastexc = c_last_exception
     renaming = {}
     def rename(v):
         return renaming.get(v, v)
 
     for block in graph.iterblocks():
-        if not (block.exitswitch == clastexc
+        if not (block.canraise
                 and block.exits[-1].exitcase is Exception):
             continue
         covered = [link.exitcase for link in block.exits[1:-1]]
@@ -193,8 +190,8 @@
 def transform_xxxitem(graph):
     # xxx setitem too
     for block in graph.iterblocks():
-        if block.operations and block.exitswitch == c_last_exception:
-            last_op = block.operations[-1]
+        if block.canraise:
+            last_op = block.raising_op
             if last_op.opname == 'getitem':
                 postfx = []
                 for exit in block.exits:
@@ -211,9 +208,6 @@
 
 def remove_dead_exceptions(graph):
     """Exceptions can be removed if they are unreachable"""
-
-    clastexc = c_last_exception
-
     def issubclassofmember(cls, seq):
         for member in seq:
             if member and issubclass(cls, member):
@@ -221,7 +215,7 @@
         return False
 
     for block in list(graph.iterblocks()):
-        if block.exitswitch != clastexc:
+        if not block.canraise:
             continue
         exits = []
         seen = []
@@ -257,7 +251,7 @@
             continue
         source = link.prevblock
         switch = source.exitswitch
-        if (isinstance(switch, Constant) and switch != c_last_exception):
+        if (isinstance(switch, Constant) and not source.canraise):
             exits = replace_exitswitch_by_constant(source, switch)
             stack.extend(exits)
         else:
@@ -334,7 +328,8 @@
             newexitswitch = rename(link.target.exitswitch)
             link.prevblock.exitswitch = newexitswitch
             link.prevblock.recloseblock(*exits)
-            if isinstance(newexitswitch, Constant) and newexitswitch != c_last_exception:
+            if (isinstance(newexitswitch, Constant) and
+                    not link.prevblock.canraise):
                 exits = replace_exitswitch_by_constant(link.prevblock,
                                                        newexitswitch)
             stack.extend(exits)
@@ -358,7 +353,7 @@
                 # can we remove this exit without breaking the graph?
                 if len(block.exits) < 2:
                     break
-                if block.exitswitch == c_last_exception:
+                if block.canraise:
                     if exit.exitcase is None:
                         break
                     if len(block.exits) == 2:
@@ -460,12 +455,7 @@
     start_blocks = find_start_blocks(graphs)
 
     def canremove(op, block):
-        if op.opname not in CanRemove:
-            return False
-        if block.exitswitch != c_last_exception:
-            return True
-        # cannot remove the exc-raising operation
-        return op is not block.operations[-1]
+        return op.opname in CanRemove and op is not block.raising_op
 
     # compute dependencies and an initial read_vars
     for block in blocks:
@@ -535,9 +525,8 @@
                     if translator is not None:
                         graph = get_graph(op.args[0], translator)
                         if (graph is not None and
-                            has_no_side_effects(translator, graph) and
-                            (block.exitswitch != c_last_exception or
-                             i != len(block.operations)- 1)):
+                                has_no_side_effects(translator, graph) and
+                                op is not block.raising_op):
                             del block.operations[i]
         # look for output variables never used
         # warning: this must be completely done *before* we attempt to
@@ -781,9 +770,8 @@
     # collect relevant operations based on the family of their result
     for block in graph.iterblocks():
         if (len(block.operations) == 1 and
-            block.operations[0].opname == 'next' and
-            block.exitswitch == c_last_exception and
-            len(block.exits) >= 2):
+                block.operations[0].opname == 'next' and
+                block.canraise and len(block.exits) >= 2):
             cases = [link.exitcase for link in block.exits]
             if None in cases and StopIteration in cases:
                 # it's a straightforward loop start block
diff --git a/rpython/translator/transform.py b/rpython/translator/transform.py
--- a/rpython/translator/transform.py
+++ b/rpython/translator/transform.py
@@ -5,9 +5,8 @@
 completed.
 """
 
-from rpython.flowspace.model import SpaceOperation
-from rpython.flowspace.model import Variable, Constant, Link
-from rpython.flowspace.model import c_last_exception, checkgraph
+from rpython.flowspace.model import (
+    SpaceOperation, Variable, Constant, Link, checkgraph)
 from rpython.flowspace.expression import V_Type
 from rpython.annotator import model as annmodel
 from rpython.rtyper.lltypesystem import lltype
@@ -168,7 +167,7 @@
                     else:
                         new_args.append(arg)
                 exit.args = new_args
-                if block.exitswitch == c_last_exception:
+                if block.canraise:
                     block.operations[-1:-1] = new_ops
                 else:
                     block.operations.extend(new_ops)
@@ -194,7 +193,7 @@
                 if not block.exits:
                     # oups! cannot reach the end of this block
                     cutoff_alwaysraising_block(self, block)
-                elif block.exitswitch == c_last_exception:
+                elif block.canraise:
                     # exceptional exit
                     if block.exits[0].exitcase is not None:
                         # killed the non-exceptional path!
@@ -311,4 +310,3 @@
     transform_dead_op_vars(ann, block_subset)
     if ann.translator:
         checkgraphs(ann, block_subset)
-

From noreply at buildbot.pypy.org  Tue Nov 18 03:22:49 2014
From: noreply at buildbot.pypy.org (rlamy)
Date: Tue, 18 Nov 2014 03:22:49 +0100 (CET)
Subject: [pypy-commit] pypy expressions-2: fix
Message-ID: <20141118022249.E47DD1C0F1B@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy 
Branch: expressions-2
Changeset: r74568:973af528a300
Date: 2014-11-18 02:22 +0000
http://bitbucket.org/pypy/pypy/changeset/973af528a300/

Log:	fix

diff --git a/rpython/flowspace/model.py b/rpython/flowspace/model.py
--- a/rpython/flowspace/model.py
+++ b/rpython/flowspace/model.py
@@ -153,13 +153,6 @@
 
     view = show
 
-last_exception = Atom('last_exception')
-c_last_exception = Constant(last_exception)
-# if Block().exitswitch == Constant(last_exception), it means that we are
-# interested in catching the exception that the *last operation* of the
-# block could raise.  The exitcases of the links are None for no exception
-# or XxxError classes to catch the matching exceptions.
-
 
 class Block(object):
     __slots__ = """inputargs operations exitswitch
@@ -469,6 +462,13 @@
     def __repr__(self):
         return self.__name__
 
+last_exception = Atom('last_exception')
+c_last_exception = Constant(last_exception)
+# if Block().exitswitch == Constant(last_exception), it means that we are
+# interested in catching the exception that the *last operation* of the
+# block could raise.  The exitcases of the links are None for no exception
+# or XxxError classes to catch the matching exceptions.
+
 def uniqueitems(lst):
     "Returns a list with duplicate elements removed."
     result = []

From noreply at buildbot.pypy.org  Tue Nov 18 07:43:22 2014
From: noreply at buildbot.pypy.org (fijal)
Date: Tue, 18 Nov 2014 07:43:22 +0100 (CET)
Subject: [pypy-commit] pypy optresult: push push push,
	until we get back to optpure
Message-ID: <20141118064322.C42FC1C0F1B@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: optresult
Changeset: r74569:3c538c4e3d52
Date: 2014-11-17 17:10 +0200
http://bitbucket.org/pypy/pypy/changeset/3c538c4e3d52/

Log:	push push push, until we get back to optpure

diff --git a/rpython/jit/metainterp/optimizeopt/intbounds.py b/rpython/jit/metainterp/optimizeopt/intbounds.py
--- a/rpython/jit/metainterp/optimizeopt/intbounds.py
+++ b/rpython/jit/metainterp/optimizeopt/intbounds.py
@@ -237,14 +237,13 @@
         v1 = self.getvalue(op.getarg(0))
         v2 = self.getvalue(op.getarg(1))
         resbound = v1.intbound.add_bound(v2.intbound)
-        r = self.getvalue(op)
         if resbound.bounded():
             # Transform into INT_ADD.  The following guard will be killed
             # by optimize_GUARD_NO_OVERFLOW; if we see instead an
             # optimize_GUARD_OVERFLOW, then InvalidLoop.
-            op = op.copy_and_change(rop.INT_ADD)
+            op = self.optimizer.replace_op_with(op, rop.INT_ADD)
         self.emit_operation(op) # emit the op
-        r.box = op
+        r = self.getvalue(op)
         r.intbound.intersect(resbound)
 
     def optimize_INT_SUB_OVF(self, op):
@@ -255,8 +254,7 @@
             return
         resbound = v1.intbound.sub_bound(v2.intbound)
         if resbound.bounded():
-            xxxx
-            op = op.copy_and_change(rop.INT_SUB)
+            op = self.optimizer.replace_op_with(op, rop.INT_SUB)
         self.emit_operation(op) # emit the op
         r = self.getvalue(op)
         r.intbound.intersect(resbound)
diff --git a/rpython/jit/metainterp/optimizeopt/optimizer.py b/rpython/jit/metainterp/optimizeopt/optimizer.py
--- a/rpython/jit/metainterp/optimizeopt/optimizer.py
+++ b/rpython/jit/metainterp/optimizeopt/optimizer.py
@@ -1,6 +1,6 @@
 from rpython.jit.metainterp import jitprof, resume, compile
 from rpython.jit.metainterp.executor import execute_nonspec_const
-from rpython.jit.metainterp.history import BoxInt, BoxFloat, Const, ConstInt, REF
+from rpython.jit.metainterp.history import Const, ConstInt, REF
 from rpython.jit.metainterp.optimizeopt.intutils import IntBound, IntUnbounded, \
                                                      ImmutableIntUnbounded, \
                                                      IntLowerBound, MININT, MAXINT
@@ -55,7 +55,7 @@
         if intbound:
             self.intbound = intbound
         else:
-            if isinstance(box, BoxInt):
+            if box is not None and box.type == 'i':
                 self.intbound = IntBound(MININT, MAXINT)
             else:
                 self.intbound = IntUnbounded()
@@ -64,6 +64,15 @@
             self.make_constant(box)
         # invariant: box is a Const if and only if level == LEVEL_CONSTANT
 
+    def copy_attributes_from(self, other):
+        assert other.__class__ is OptValue
+        self.level = other.level
+        self.known_class = other.known_class
+        self.intbound = other.intbound
+        self.lenbound = other.lenbound
+        self.box = other.box
+        self.last_guard = other.last_guard
+
     def make_len_gt(self, mode, descr, val):
         if self.lenbound:
             assert self.lenbound.mode == mode
@@ -503,21 +512,37 @@
     def clear_newoperations(self):
         self._newoperations = []
 
-    def make_equal_to(self, box, value, replace=False):
-        assert isinstance(value, OptValue)
-        assert replace or box not in self.values
-        self.values[box] = value
+    def make_equal_to(self, box, newvalue):
+        if box in self.values:
+            v = self.getvalue(box)
+            v.copy_attributes_from(newvalue)
+        else:
+            self.values[box] = newvalue
+
+    def replace_op_with(self, oldop, newopnum, args=None):
+        newop = oldop._copy_and_change(newopnum, args=args)
+        v = self.getvalue(oldop)
+        v.box = newop
+        self.values[newop] = v
+        return newop
 
     def make_constant(self, box, constbox):
-        self.make_equal_to(box, ConstantValue(constbox))
+        try:
+            value = self.values[box]
+            value.level = LEVEL_CONSTANT
+            value.make_constant(constbox)
+        except KeyError:
+            self.values[box] = ConstantValue(constbox)
 
     def make_constant_int(self, box, intvalue):
         self.make_constant(box, ConstInt(intvalue))
 
     def new_ptr_box(self):
+        xxx
         return self.cpu.ts.BoxRef()
 
     def new_box(self, fieldofs):
+        xxx
         if fieldofs.is_pointer_field():
             return self.new_ptr_box()
         elif fieldofs.is_float_field():
@@ -534,6 +559,7 @@
             return CVAL_ZERO
 
     def new_box_item(self, arraydescr):
+        xxx
         if arraydescr.is_array_of_pointers():
             return self.new_ptr_box()
         elif arraydescr.is_array_of_floats():
diff --git a/rpython/jit/metainterp/optimizeopt/pure.py b/rpython/jit/metainterp/optimizeopt/pure.py
--- a/rpython/jit/metainterp/optimizeopt/pure.py
+++ b/rpython/jit/metainterp/optimizeopt/pure.py
@@ -43,7 +43,7 @@
             oldop = self.pure_operations.get(args, None)
             if oldop is not None and oldop.getdescr() is op.getdescr():
                 assert oldop.getopnum() == op.getopnum()
-                self.optimizer.make_equal_to(op, self.getvalue(oldop), True)
+                self.optimizer.make_equal_to(op, self.getvalue(oldop))
                 return
             else:
                 self.pure_operations[args] = op
diff --git a/rpython/jit/metainterp/optimizeopt/rewrite.py b/rpython/jit/metainterp/optimizeopt/rewrite.py
--- a/rpython/jit/metainterp/optimizeopt/rewrite.py
+++ b/rpython/jit/metainterp/optimizeopt/rewrite.py
@@ -215,13 +215,13 @@
             v2 = self.getvalue(rhs)
 
             if v1.is_constant():
-                if v1.box.getfloat() == 1.0:
-                    self.make_equal_to(op.result, v2)
+                if v1.box.getfloatstorage() == 1.0:
+                    self.make_equal_to(op, v2)
                     return
-                elif v1.box.getfloat() == -1.0:
-                    self.emit_operation(ResOperation(
-                        rop.FLOAT_NEG, [rhs], op.result
-                    ))
+                elif v1.box.getfloatstorage() == -1.0:
+                    newop = self.optimizer.replace_op_with(op, rop.FLOAT_NEG,
+                                                           args=[rhs])
+                    self.emit_operation(newop)
                     return
         self.emit_operation(op)
         self.pure(rop.FLOAT_MUL, [arg2, arg1], op)
@@ -248,7 +248,7 @@
     def optimize_FLOAT_NEG(self, op):
         v1 = op.getarg(0)
         self.emit_operation(op)
-        self.pure(rop.FLOAT_NEG, [op.result], v1)
+        self.pure(rop.FLOAT_NEG, [op], v1)
 
     def optimize_guard(self, op, constbox, emit_operation=True):
         value = self.getvalue(op.getarg(0))
@@ -313,8 +313,10 @@
                 if not previous_classbox.same_constant(expected_classbox):
                     r = self.optimizer.metainterp_sd.logger_ops.repr_of_resop(op)
                     raise InvalidLoop('A GUARD_VALUE (%s) was proven to always fail' % r)
-            op = old_guard_op.copy_and_change(rop.GUARD_VALUE,
-                                      args = [old_guard_op.getarg(0), op.getarg(1)])
+            arglist = [old_guard_op.getarg(0), op.getarg(1)]
+            op = self.optimizer.replace_op_with(old_guard_op,
+                                                rop.GUARD_VALUE,
+                                                args=arglist)
             self.getvalue(old_guard_op).box = op
             self.optimizer.replaces_guard[op] = old_guard_op
             # hack hack hack.  Change the guard_opnum on
@@ -364,8 +366,10 @@
             if old_guard_op.getopnum() == rop.GUARD_NONNULL:
                 # it was a guard_nonnull, which we replace with a
                 # guard_nonnull_class.
-                op = old_guard_op.copy_and_change (rop.GUARD_NONNULL_CLASS,
-                                         args = [old_guard_op.getarg(0), op.getarg(1)])
+                args = [old_guard_op.getarg(0), op.getarg(1)]
+                op = self.optimizer.replace_op_with(old_guard_op,
+                                                    rop.GUARD_NONNULL_CLASS,
+                                                    args)
                 self.optimizer.replaces_guard[op] = old_guard_op
                 # hack hack hack.  Change the guard_opnum on
                 # new_guard_op.getdescr() so that when resuming,
@@ -399,7 +403,8 @@
         # change the op to be a normal call, from the backend's point of view
         # there is no reason to have a separate operation for this
         self.loop_invariant_producer[key] = op
-        newop = op.copy_and_change(self.optimizer.call_for_descr(op.getdescr()))
+        opnum = self.optimizer.call_for_descr(op.getdescr())
+        newop = self.optimizer.replace_op_with(op, opnum)
         self.emit_operation(newop)
         resvalue = self.getvalue(op)
         resvalue.box = newop
diff --git a/rpython/jit/metainterp/resoperation.py b/rpython/jit/metainterp/resoperation.py
--- a/rpython/jit/metainterp/resoperation.py
+++ b/rpython/jit/metainterp/resoperation.py
@@ -86,7 +86,7 @@
     # common methods
     # --------------
 
-    def copy_and_change(self, opnum, args=None, result=None, descr=None):
+    def _copy_and_change(self, opnum, args=None, result=None, descr=None):
         "shallow copy: the returned operation is meant to be used in place of self"
         if args is None:
             args = self.getarglist()
@@ -270,8 +270,8 @@
     def setfailargs(self, fail_args):
         self._fail_args = fail_args
 
-    def copy_and_change(self, opnum, args=None, result=None, descr=None):
-        newop = AbstractResOp.copy_and_change(self, opnum, args, result, descr)
+    def _copy_and_change(self, opnum, args=None, result=None, descr=None):
+        newop = AbstractResOp._copy_and_change(self, opnum, args, result, descr)
         newop.setfailargs(self.getfailargs())
         return newop
 

From noreply at buildbot.pypy.org  Tue Nov 18 07:43:24 2014
From: noreply at buildbot.pypy.org (fijal)
Date: Tue, 18 Nov 2014 07:43:24 +0100 (CET)
Subject: [pypy-commit] pypy optresult: hack enough to get the basic test go
	to the backend
Message-ID: <20141118064324.45A5F1C0F1B@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: optresult
Changeset: r74570:8960f6fef760
Date: 2014-11-18 08:43 +0200
http://bitbucket.org/pypy/pypy/changeset/8960f6fef760/

Log:	hack enough to get the basic test go to the backend

diff --git a/rpython/jit/metainterp/compile.py b/rpython/jit/metainterp/compile.py
--- a/rpython/jit/metainterp/compile.py
+++ b/rpython/jit/metainterp/compile.py
@@ -128,9 +128,10 @@
     part = create_empty_loop(metainterp)
     part.inputargs = inputargs[:]
     h_ops = history.operations
-    part.operations = [ResOperation(rop.LABEL, inputargs, None, descr=TargetToken(jitcell_token))] + \
+    # XXX why do we clone here?
+    part.operations = [ResOperation(rop.LABEL, inputargs, descr=TargetToken(jitcell_token))] + \
                       [h_ops[i].clone() for i in range(start, len(h_ops))] + \
-                      [ResOperation(rop.LABEL, jumpargs, None, descr=jitcell_token)]
+                      [ResOperation(rop.LABEL, jumpargs, descr=jitcell_token)]
 
     try:
         optimize_trace(metainterp_sd, part, enable_opts)
diff --git a/rpython/jit/metainterp/executor.py b/rpython/jit/metainterp/executor.py
--- a/rpython/jit/metainterp/executor.py
+++ b/rpython/jit/metainterp/executor.py
@@ -437,7 +437,6 @@
 
 
 def execute(cpu, metainterp, opnum, descr, *argboxes):
-    xxx
     # only for opnums with a fixed arity
     num_args = len(argboxes)
     withdescr = has_descr(opnum)
diff --git a/rpython/jit/metainterp/history.py b/rpython/jit/metainterp/history.py
--- a/rpython/jit/metainterp/history.py
+++ b/rpython/jit/metainterp/history.py
@@ -1,7 +1,7 @@
 from rpython.rtyper.extregistry import ExtRegistryEntry
 from rpython.rtyper.lltypesystem import lltype, llmemory, rffi
 from rpython.rlib.objectmodel import we_are_translated, Symbolic
-from rpython.rlib.objectmodel import compute_unique_id
+from rpython.rlib.objectmodel import compute_unique_id, specialize
 from rpython.rlib.rarithmetic import r_int64, is_valid_int
 
 from rpython.conftest import option
@@ -172,6 +172,16 @@
         self.identifier = identifier      # for testing
 
 
+ at specialize.argtype(0)
+def newconst(value):
+    if isinstance(value, int):
+        return ConstInt(value)
+    elif isinstance(value, float):
+        return ConstFloat(value)
+    else:
+        assert lltype.typeOf(value) == llmemory.GCREF
+        return ConstPtr(value)
+        
 class Const(AbstractValue):
     __slots__ = ()
 
@@ -689,6 +699,7 @@
     @staticmethod
     def check_consistency_of_branch(operations, seen):
         "NOT_RPYTHON"
+        return # XXX for now
         for op in operations:
             for i in range(op.numargs()):
                 box = op.getarg(i)
@@ -763,8 +774,10 @@
         self.inputargs = None
         self.operations = []
 
-    def record(self, opnum, argboxes, resbox, descr=None):
-        op = ResOperation(opnum, argboxes, resbox, descr)
+    @specialize.argtype(3)
+    def record(self, opnum, argboxes, value, descr=None):
+        op = ResOperation(opnum, argboxes, descr)
+        op.setvalue(value)
         self.operations.append(op)
         return op
 
diff --git a/rpython/jit/metainterp/logger.py b/rpython/jit/metainterp/logger.py
--- a/rpython/jit/metainterp/logger.py
+++ b/rpython/jit/metainterp/logger.py
@@ -155,10 +155,10 @@
             s_offset = "+%d: " % offset
         args = ", ".join([self.repr_of_arg(op.getarg(i)) for i in range(op.numargs())])
 
-        if op.result is not None:
-            res = self.repr_of_arg(op.result) + " = "
-        else:
-            res = ""
+        #if op.result is not None:
+        #    res = self.repr_of_arg(op.result) + " = "
+        #else:
+        res = ""
         is_guard = op.is_guard()
         if op.getdescr() is not None:
             descr = op.getdescr()
diff --git a/rpython/jit/metainterp/optimizeopt/pure.py b/rpython/jit/metainterp/optimizeopt/pure.py
--- a/rpython/jit/metainterp/optimizeopt/pure.py
+++ b/rpython/jit/metainterp/optimizeopt/pure.py
@@ -42,7 +42,6 @@
                                                 op.getarglist(), op.getdescr())
             oldop = self.pure_operations.get(args, None)
             if oldop is not None and oldop.getdescr() is op.getdescr():
-                assert oldop.getopnum() == op.getopnum()
                 self.optimizer.make_equal_to(op, self.getvalue(oldop))
                 return
             else:
diff --git a/rpython/jit/metainterp/optimizeopt/rewrite.py b/rpython/jit/metainterp/optimizeopt/rewrite.py
--- a/rpython/jit/metainterp/optimizeopt/rewrite.py
+++ b/rpython/jit/metainterp/optimizeopt/rewrite.py
@@ -233,7 +233,7 @@
 
         # replace "x / const" by "x * (1/const)" if possible
         if v2.is_constant():
-            divisor = v2.box.getfloat()
+            divisor = v2.box.getfloatstorage()
             fraction = math.frexp(divisor)[0]
             # This optimization is valid for powers of two
             # but not for zeroes, some denormals and NaN:
@@ -242,7 +242,8 @@
                 rfraction = math.frexp(reciprocal)[0]
                 if rfraction == 0.5 or rfraction == -0.5:
                     c = ConstFloat(longlong.getfloatstorage(reciprocal))
-                    op = op.copy_and_change(rop.FLOAT_MUL, args=[arg1, c])
+                    op = self.optimizer.replace_op_with(op, rop.FLOAT_MUL,
+                                                        args=[arg1, c])
         self.emit_operation(op)
 
     def optimize_FLOAT_NEG(self, op):
diff --git a/rpython/jit/metainterp/optimizeopt/simplify.py b/rpython/jit/metainterp/optimizeopt/simplify.py
--- a/rpython/jit/metainterp/optimizeopt/simplify.py
+++ b/rpython/jit/metainterp/optimizeopt/simplify.py
@@ -48,7 +48,7 @@
         if not self.unroll:
             descr = op.getdescr()
             if isinstance(descr, JitCellToken):
-                return self.optimize_JUMP(op.copy_and_change(rop.JUMP))
+                return self.optimize_JUMP(op._copy_and_change(rop.JUMP))
             self.last_label_descr = op.getdescr()
         self.emit_operation(op)
 
diff --git a/rpython/jit/metainterp/pyjitpl.py b/rpython/jit/metainterp/pyjitpl.py
--- a/rpython/jit/metainterp/pyjitpl.py
+++ b/rpython/jit/metainterp/pyjitpl.py
@@ -1883,11 +1883,11 @@
         # execute the operation
         profiler = self.staticdata.profiler
         profiler.count_ops(opnum)
-        resbox = executor.execute(self.cpu, self, opnum, descr, *argboxes)
+        resvalue = executor.execute(self.cpu, self, opnum, descr, *argboxes)
         if rop._ALWAYS_PURE_FIRST <= opnum <= rop._ALWAYS_PURE_LAST:
-            return self._record_helper_pure(opnum, resbox, descr, *argboxes)
+            return self._record_helper_pure(opnum, resvalue, descr, *argboxes)
         else:
-            return self._record_helper_nonpure_varargs(opnum, resbox, descr,
+            return self._record_helper_nonpure_varargs(opnum, resvalue, descr,
                                                        list(argboxes))
 
     @specialize.arg(1)
@@ -1896,6 +1896,7 @@
         # execute the operation
         profiler = self.staticdata.profiler
         profiler.count_ops(opnum)
+        xxx
         resbox = executor.execute_varargs(self.cpu, self,
                                           opnum, argboxes, descr)
         # check if the operation can be constant-folded away
@@ -1906,16 +1907,18 @@
             resbox = self._record_helper_nonpure_varargs(opnum, resbox, descr, argboxes)
         return resbox
 
-    def _record_helper_pure(self, opnum, resbox, descr, *argboxes):
+    @specialize.argtype(2)
+    def _record_helper_pure(self, opnum, resvalue, descr, *argboxes):
         canfold = self._all_constants(*argboxes)
         if canfold:
-            resbox = resbox.constbox()       # ensure it is a Const
-            return resbox
+            return history.newconst(resvalue)
         else:
-            resbox = resbox.nonconstbox()    # ensure it is a Box
-            return self._record_helper_nonpure_varargs(opnum, resbox, descr, list(argboxes))
+            return self._record_helper_nonpure_varargs(opnum, resvalue, descr,
+                                                       list(argboxes))
 
+    @specialize.argtype(2)
     def _record_helper_pure_varargs(self, opnum, resbox, descr, argboxes):
+        xxx
         canfold = self._all_constants_varargs(argboxes)
         if canfold:
             resbox = resbox.constbox()       # ensure it is a Const
@@ -1924,19 +1927,19 @@
             resbox = resbox.nonconstbox()    # ensure it is a Box
             return self._record_helper_nonpure_varargs(opnum, resbox, descr, argboxes)
 
-    def _record_helper_nonpure_varargs(self, opnum, resbox, descr, argboxes):
-        assert resbox is None or isinstance(resbox, Box)
+    @specialize.argtype(2)
+    def _record_helper_nonpure_varargs(self, opnum, resvalue, descr, argboxes):
         if (rop._OVF_FIRST <= opnum <= rop._OVF_LAST and
             self.last_exc_value_box is None and
             self._all_constants_varargs(argboxes)):
-            return resbox.constbox()
+            return history.newconst(resvalue)
         # record the operation
         profiler = self.staticdata.profiler
         profiler.count_ops(opnum, Counters.RECORDED_OPS)
         self.heapcache.invalidate_caches(opnum, descr, argboxes)
-        op = self.history.record(opnum, argboxes, resbox, descr)
+        op = self.history.record(opnum, argboxes, resvalue, descr)
         self.attach_debug_info(op)
-        return resbox
+        return op
 
     def execute_new_with_vtable(self, known_class):
         resbox = self.execute_and_record(rop.NEW_WITH_VTABLE, None,
diff --git a/rpython/jit/metainterp/resoperation.py b/rpython/jit/metainterp/resoperation.py
--- a/rpython/jit/metainterp/resoperation.py
+++ b/rpython/jit/metainterp/resoperation.py
@@ -115,7 +115,7 @@
         descr = self.getdescr()
         if descr is not None:
             descr = descr.clone_if_mutable()
-        op = ResOperation(self.getopnum(), args[:], self.result, descr)
+        op = ResOperation(self.getopnum(), args[:], descr)
         if not we_are_translated():
             op.name = self.name
             op.pc = self.pc
@@ -340,6 +340,9 @@
             name = self.type + str(len(memo))
             memo[self] = name
             return name
+
+    def getdescr(self):
+        return None
         
 class InputArgInt(IntOp, AbstractInputArg):
     def __init__(self, intval):

From noreply at buildbot.pypy.org  Tue Nov 18 11:34:20 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Tue, 18 Nov 2014 11:34:20 +0100 (CET)
Subject: [pypy-commit] cffi default: Revert a side-effect of f31f43f81992:
 allow again multiple declarations
Message-ID: <20141118103420.D67C21D289A@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r1574:22b3062ac3a9
Date: 2014-11-18 11:34 +0100
http://bitbucket.org/cffi/cffi/changeset/22b3062ac3a9/

Log:	Revert a side-effect of f31f43f81992: allow again multiple
	declarations of the same constant. Occurs in cases of ffi.include()
	overuse.

diff --git a/cffi/cparser.py b/cffi/cparser.py
--- a/cffi/cparser.py
+++ b/cffi/cparser.py
@@ -208,6 +208,8 @@
 
     def _add_constants(self, key, val):
         if key in self._int_constants:
+            if self._int_constants[key] == val:
+                return     # ignore identical double declarations
             raise api.FFIError(
                 "multiple declarations of constant: %s" % (key,))
         self._int_constants[key] = val
diff --git a/testing/backend_tests.py b/testing/backend_tests.py
--- a/testing/backend_tests.py
+++ b/testing/backend_tests.py
@@ -1,7 +1,7 @@
 import py
 import platform
 import sys, ctypes
-from cffi import FFI, CDefError
+from cffi import FFI, CDefError, FFIError
 from testing.support import *
 
 SIZE_OF_INT   = ctypes.sizeof(ctypes.c_int)
@@ -1564,6 +1564,12 @@
         p = ffi2.new("foo_p", [142])
         assert p.x == 142
 
+    def test_ignore_multiple_declarations_of_constant(self):
+        ffi = FFI(backend=self.Backend())
+        ffi.cdef("#define FOO 42")
+        ffi.cdef("#define FOO 42")
+        py.test.raises(FFIError, ffi.cdef, "#define FOO 43")
+
     def test_struct_packed(self):
         ffi = FFI(backend=self.Backend())
         ffi.cdef("struct nonpacked { char a; int b; };")

From noreply at buildbot.pypy.org  Tue Nov 18 15:47:22 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Tue, 18 Nov 2014 15:47:22 +0100 (CET)
Subject: [pypy-commit] creflect default: Starting on global vars
Message-ID: <20141118144722.063091C104D@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r67:c78b70b93c73
Date: 2014-11-18 15:47 +0100
http://bitbucket.org/cffi/creflect/changeset/c78b70b93c73/

Log:	Starting on global vars

diff --git a/creflect/cparser.py b/creflect/cparser.py
--- a/creflect/cparser.py
+++ b/creflect/cparser.py
@@ -97,7 +97,6 @@
             assert isinstance(tp, model.FunctionType)
             self.declarations.append(model.FuncDecl(decl.name, tp))
         else:
-            xxxxxxxxxxxxxx
             const = 'const' in decl.quals
             if isinstance(node, pycparser.c_ast.Struct):
                 if node.decls is not None:
@@ -117,7 +116,7 @@
                 if self._is_constant_globalvar(node):
                     self._declare('constant ' + decl.name, tp)
                 else:
-                    self._declare('variable ' + decl.name, tp)
+                    self.declarations.append(model.VarDecl(decl.name, tp))
 
     def _parse_typedef(self, decl):
         if not decl.name:
@@ -203,10 +202,7 @@
         params = list(getattr(typenode.args, 'params', []))
         ellipsis = (
             len(params) > 0 and
-            isinstance(params[-1].type, pycparser.c_ast.TypeDecl) and
-            isinstance(params[-1].type.type,
-                       pycparser.c_ast.IdentifierType) and
-            params[-1].type.type.names == ['__dotdotdot__'])
+            isinstance(params[-1], pycparser.c_ast.EllipsisParam))
         if ellipsis:
             params.pop()
             if not params:
@@ -229,6 +225,13 @@
         else:
             return type
 
+    def _is_constant_globalvar(self, typenode):
+        if isinstance(typenode, pycparser.c_ast.PtrDecl):
+            return 'const' in typenode.quals
+        if isinstance(typenode, pycparser.c_ast.TypeDecl):
+            return 'const' in typenode.quals
+        return False
+
     def _get_struct_union_enum_type(self, kind, type, const, approx_name=None):
         name = type.name or approx_name
         if not name or name.startswith('$$$'):
diff --git a/creflect/model.py b/creflect/model.py
--- a/creflect/model.py
+++ b/creflect/model.py
@@ -23,10 +23,6 @@
     def inspect_nonconst_type(self, block, inspect):
         raise NotImplementedError
 
-    def _flush_inspect(self, inspect):
-        if inspect is not None and inspect.started:
-            inspect.assign_to_p1('0')
-
     def shadow_global_var(self, top_level_block, name):
         return name
 
@@ -51,7 +47,7 @@
             self.c_name_with_marker = 'const ' + self.c_name_with_marker
 
     def inspect_nonconst_type(self, block, inspect):
-        self._flush_inspect(inspect)
+        inspect.flush()
         return block.write_crx_type_var('cb->get_void_type(cb)')
 
 void_type = VoidType(const=False)
@@ -99,7 +95,7 @@
         return self.ALL_PRIMITIVE_TYPES[self.name] == 'f'
 
     def inspect_nonconst_type(self, block, inspect):
-        if inspect is None:
+        if not isinstance(inspect, TypeInspector):
             if self.is_signed_type():
                 expr = 'cb->get_signed_type(cb, sizeof(%s), "%s")' % (
                     self.name, self.name)
@@ -152,48 +148,62 @@
         self.c_name_with_marker = (
             self.result.c_name_with_marker.replace('&', replace_with))
 
-    def inspect_type(self, block, inspect):
-        # this class overrides inspect_type() instead of
-        # inspect_nonconst_type(), to avoid the extra call to get_const_type()
-        self._flush_inspect(inspect)
+    def _get_arg_ret_types(self, block):
+        inspect = MissingInspector()
+        t1 = self.result.inspect_type(block, inspect)
+        t_args = [arg.inspect_type(block, inspect) for arg in self.args]
+        a2 = block.add_array_crx_types(len(t_args))
+        for i, t in enumerate(t_args):
+            block.writeline('%s[%d] = %s;' % (a2, i, t))
+        return t1, a2
 
-        # limitations so far:
-        assert not self.ellipsis, "XXX"
-        assert inspect is not None, "XXX"
-        assert inspect.started, "XXX"
-        assert inspect.levels == ['*'], "XXX"
-
-        t1 = self.result.inspect_type(block, None)
+    def _get_c_call_sequence(self, varname):
         call = ['    ']
         if not isinstance(self.result, VoidType):
             call.append('*(%s)result = ' % (self.result.get_c_name('*'),))
-        call.append('f(')
-        t_args = [arg.inspect_type(block, None) for arg in self.args]
-        a2 = block.add_array_crx_types(len(t_args))
-        for i, t in enumerate(t_args):
-            block.writeline('%s[%d] = %s;' % (a2, i, t))
+        call.append(varname)
+        call.append('(')
+        for i in range(len(self.args)):
             if i > 0:
                 call.append(', ')
             call.append('*(%s)args[%d]' % (self.args[i].get_c_name('*'), i))
         call.append(');')
-        #
+        return ''.join(call)
+
+    def inspect_type(self, block, inspect):
+        # this class overrides inspect_type() instead of
+        # inspect_nonconst_type(), to avoid the extra call to get_const_type()
+        if self.ellipsis:
+            return self.inspect_type_ellipsis(block, inspect)
+
+        inspect.flush()
+
+        # limitations so far:
+        assert isinstance(inspect, TypeInspector)
+        assert inspect.started, "XXX"
+        assert inspect.levels == ['*'], "XXX"
+
         toplevel = block.crx_top_level
-        if inspect.varname is not None:
-            extraname = 'c_' + inspect.varname
-        else:
-            extraname = block._get_next_name('f')
+        extraname = block._get_next_name('f')
         crx_func_name = '%s__%s' % (toplevel.crx_func_name, extraname)
         wrline = toplevel.writeline
         wrline('static void %s(void *func, void *args[], void *result) {' % (
             crx_func_name,))
-        if inspect.started:
-            assert inspect.levels == ['*']
-            wrline('    %s f = func;' % inspect.typename)
-        wrline(''.join(call))
+        wrline('    %s f = func;' % inspect.typename)
+        wrline(self._get_c_call_sequence('f'))
         wrline('}')
         wrline('')
+        t1, a2 = self._get_arg_ret_types(block)
         expr = 'cb->get_function_type(cb, %s, %s, %d, &%s)' % (
-            t1, a2, len(t_args), crx_func_name)
+            t1, a2, len(self.args), crx_func_name)
+        return block.write_crx_type_var(expr)
+
+    def inspect_type_ellipsis(self, block, inspect):
+        if not isinstance(inspect, TypeInspector):
+            XXX
+        else:
+            star_p1 = inspect.fetch_star_p1()
+            xxx
         return block.write_crx_type_var(expr)
 
     def shadow_global_var(self, top_level_block, fnname):
@@ -211,32 +221,20 @@
         return shadowname
 
     def write_func_decl(self, block, fnname):
-        # XXX code duplication!
-        t1 = self.result.inspect_type(block, None)
-        call = ['    ']
-        if not isinstance(self.result, VoidType):
-            call.append('*(%s)result = ' % (self.result.get_c_name('*'),))
-        call.append('%s(' % fnname)
-        t_args = [arg.inspect_type(block, None) for arg in self.args]
-        a2 = block.add_array_crx_types(len(t_args))
-        for i, t in enumerate(t_args):
-            block.writeline('%s[%d] = %s;' % (a2, i, t))
-            if i > 0:
-                call.append(', ')
-            call.append('*(%s)args[%d]' % (self.args[i].get_c_name('*'), i))
-        call.append(');')
-        #
+        # a special-case for global function declarations
+        assert not self.ellipsis
         toplevel = block.crx_top_level
         crx_func_name = '%s__c_%s' % (toplevel.crx_func_name, fnname)
         wrline = toplevel.writeline
         wrline('static void %s(void *args[], void *result) {' % (
             crx_func_name,))
-        wrline(''.join(call))
+        wrline(self._get_c_call_sequence(fnname))
         wrline('}')
         wrline('')
         shadow = self.shadow_global_var(toplevel, fnname)
+        t1, a2 = self._get_arg_ret_types(block)
         block.writeline('cb->define_func(cb, "%s", %s, %s, %d, &%s, &%s);' % (
-            fnname, t1, a2, len(t_args), crx_func_name, shadow))
+            fnname, t1, a2, len(self.args), crx_func_name, shadow))
 
 
 class PointerType(BaseType):
@@ -257,7 +255,7 @@
         self.c_name_with_marker = base.replace('&', extra)
 
     def inspect_nonconst_type(self, block, inspect):
-        if inspect is not None:
+        if isinstance(inspect, TypeInspector):
             star_p1 = inspect.fetch_star_p1()
             new_var = 'p%d' % (len(inspect.levels) + 2,)
             block.writedecl("char *%s;" % (new_var,))
@@ -278,6 +276,12 @@
                     block.writeline("}")
                 inspect.after_star_p1_assignment.append(after)
             inspect.levels.append('*')
+        elif isinstance(inspect, VarInspector):
+            block.writeline("void *p1 = %s;  /* check that '%s' is a"
+                            " pointer */" % (inspect.varname, inspect.varname))
+            block.writeline("if (0) { %s = p1; }  /* check that '%s' is a"
+                            " pointer variable, and not an array or a"
+                            " constant */" % (inspect.varname, inspect.varname))
         t1 = self.totype.inspect_type(block, inspect)
         return block.write_crx_type_var('cb->get_pointer_type(cb, %s)' % t1)
 
@@ -302,7 +306,7 @@
     def inspect_type(self, block, inspect):
         # this class overrides inspect_type() instead of
         # inspect_nonconst_type(), to avoid the extra call to get_const_type()
-        if inspect is not None:
+        if isinstance(inspect, TypeInspector):
             star_p1 = inspect.fetch_star_p1()
             errmsg = "type '%s' is not an array, but a pointer type" % (
                 inspect.get_comment_type(0, False),)
@@ -338,11 +342,25 @@
 # ____________________________________________________________
 
 
+class MissingInspector(object):
+
+    def flush(self):
+        pass
+
+
+class VarInspector(object):
+    def __init__(self, block, varname):
+        self.block = block
+        self.varname = varname
+
+    def flush(self):
+        pass
+
+
 class TypeInspector(object):
-    def __init__(self, block, typename, structfield=None, varname=None):
+    def __init__(self, block, typename, structfield=None):
         self.block = block
         self.typename = typename
-        self.varname = varname
         self.structfield = structfield
         self.started = False
         if structfield is None:
@@ -362,6 +380,10 @@
             self.at_end = []
             self.assign_target = None
 
+    def flush(self):
+        if self.started:
+            self.assign_to_p1('0')
+
     def get_comment_type(self, levels_delta, ignore_array, minlevel=0):
         end = len(self.levels)
         while ignore_array and end > 0 and self.levels[end - 1] == '[]':
@@ -527,21 +549,23 @@
 
     def write_declaration(self, funcblock):
         block = CodeBlock(funcblock)
-        inspect = TypeInspector(block, None, varname=self.name)
+        inspect = VarInspector(block, self.name)
         t1 = self.type.inspect_type(block, inspect)
-        inspect.stop()
         shadow = self.type.shadow_global_var(block.crx_top_level, self.name)
         block.writeline('cb->define_var(cb, "%s", %s, &%s);' % (
             self.name, t1, shadow))
         funcblock.write_subblock(block)
 
 
-class FuncDecl(object):
+class FuncDecl(VarDecl):
     def __init__(self, name, type):
         self.name = name
         self.type = type
 
     def write_declaration(self, funcblock):
-        block = CodeBlock(funcblock)
-        self.type.write_func_decl(block, self.name)
-        funcblock.write_subblock(block)
+        if self.type.ellipsis:
+            VarDecl.write_declaration(self, funcblock)
+        else:
+            block = CodeBlock(funcblock)
+            self.type.write_func_decl(block, self.name)
+            funcblock.write_subblock(block)
diff --git a/test/cgcompile.c b/test/cgcompile.c
--- a/test/cgcompile.c
+++ b/test/cgcompile.c
@@ -96,7 +96,8 @@
                                                   crx_type_t *ret,
                                                   crx_type_t *args[], int nargs)
 {
-    abort();
+    crx_type_t *dotdotdot = newtype2("... -> ", ret->text);
+    return tst_get_function_type(cb, dotdotdot, args, nargs, 0);
 }
 
 static crx_type_t *tst_get_pointer_type(crx_builder_t *cb, crx_type_t *t)
diff --git a/test/codegen/func-002.c b/test/codegen/func-002.c
--- a/test/codegen/func-002.c
+++ b/test/codegen/func-002.c
@@ -9,12 +9,13 @@
 
 void testfunc_002(crx_builder_t *cb)
 {
-    crx_type *t1, *a2[1], *t2;
+    crx_type_t *t1, *a2[1], *t3;
     {
         int (*p1)(int, ...) = f;  /* check that 'f' is a function with exactly the given signature */
         t1 = cb->get_signed_type(cb, sizeof(int), "int");
         a2[0] = t1;
-        t2 = cb->get_ellipsis_function_type(cb, t1, a2, 1);
-        cb->define_var(cb, "f", t2, p1);
+        t3 = cb->get_ellipsis_function_type(cb, t1, a2, 1);
+        cb->define_var(cb, "f", t3, p1);
+#expect VAR f: FUNC( int -> ... -> int )
     }
 }
diff --git a/test/codegen/glob-001.c b/test/codegen/glob-001.c
--- a/test/codegen/glob-001.c
+++ b/test/codegen/glob-001.c
@@ -2,15 +2,15 @@
 
 # ____________________________________________________________
 
-int testglob_001(char *r)
+void testglob_001(crx_builder_t *cb)
 {
-    if (!r)
-        return 28 + 17;
+    crx_type_t *t1, *t2;
     {
         void *p1 = someglob;  /* check that 'someglob' is a pointer */
-        r += sprintf(r, "/*%p*/", &someglob);
         if (0) { someglob = p1; }  /* check that 'someglob' is a pointer variable, and not an array or a constant */
-        r += sprintf(r, "void *someglob;\n");
+        t1 = cb->get_void_type(cb);
+        t2 = cb->get_pointer_type(cb, t1);
+        cb->define_var(cb, "someglob", t2, &someglob);
+#expect VAR someglob: PTR void
     }
-    return 0;
 }

From noreply at buildbot.pypy.org  Tue Nov 18 16:02:20 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Tue, 18 Nov 2014 16:02:20 +0100 (CET)
Subject: [pypy-commit] creflect default: Global arrays
Message-ID: <20141118150220.79DEA1C31CF@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r68:a9431fb05c50
Date: 2014-11-18 16:02 +0100
http://bitbucket.org/cffi/creflect/changeset/a9431fb05c50/

Log:	Global arrays

diff --git a/creflect/model.py b/creflect/model.py
--- a/creflect/model.py
+++ b/creflect/model.py
@@ -300,8 +300,12 @@
             brackets = '&[/*...*/]'
         else:
             brackets = '&[%d]' % length
+        if ' &' in self.item.c_name_with_marker:
+            replace_from = ' &'
+        else:
+            replace_from = '&'
         self.c_name_with_marker = (
-            self.item.c_name_with_marker.replace('&', brackets))
+            self.item.c_name_with_marker.replace(replace_from, brackets))
 
     def inspect_type(self, block, inspect):
         # this class overrides inspect_type() instead of
@@ -321,9 +325,21 @@
                 block.writeline("}")
             inspect.levels.append('[]')
             inspect.after_star_p1_assignment.append(after)
+        elif isinstance(inspect, VarInspector):
+            declitem = self.item.get_c_name("(*p1)")
+            block.writeline("%s[] = &%s;  /* check that '%s' is of type '%s'"
+                            " */" % (declitem, inspect.varname,
+                                     inspect.varname, self.get_c_name()))
+            block.writeline("(void)p1;")
+            star_p1 = inspect.varname
+        else:
+            star_p1 = None
         t1 = self.item.inspect_type(block, inspect)
-        expr = 'cb->get_array_type(cb, %s, sizeof(%s) / sizeof(*%s))' % (
-            t1, star_p1, star_p1)
+        if star_p1 is not None:
+            expr = 'cb->get_array_type(cb, %s, sizeof(%s) / sizeof(*%s))' % (
+                t1, star_p1, star_p1)
+        else:
+            expr = 'cb->get_incomplete_array_type(cb, %s)' % (t1,)
         return block.write_crx_type_var(expr)
 
 
diff --git a/test/codegen/glob-002.c b/test/codegen/glob-002.c
--- a/test/codegen/glob-002.c
+++ b/test/codegen/glob-002.c
@@ -4,12 +4,13 @@
 
 void testglob_002(crx_builder_t *cb)
 {
-    crx_type *t1, *t2;
+    crx_type_t *t1, *t2;
     {
         char (*p1)[] = &someglob;  /* check that 'someglob' is of type 'char[]' */
         (void)p1;
         t1 = cb->get_char_type(cb);
         t2 = cb->get_array_type(cb, t1, sizeof(someglob) / sizeof(*someglob));
         cb->define_var(cb, "someglob", t2, &someglob);
+#expect VAR someglob: ARRAY[100] char
     }
 }
diff --git a/test/codegen/glob-002b.c b/test/codegen/glob-002b.c
new file mode 100644
--- /dev/null
+++ b/test/codegen/glob-002b.c
@@ -0,0 +1,18 @@
+int someglob[100];
+
+// XXX can be improved to correct the size of the integer type
+
+# ____________________________________________________________
+
+void testglob_002b(crx_builder_t *cb)
+{
+    crx_type_t *t1, *t2;
+    {
+        int (*p1)[] = &someglob;  /* check that 'someglob' is of type 'int[]' */
+        (void)p1;
+        t1 = cb->get_signed_type(cb, sizeof(int), "int");
+        t2 = cb->get_array_type(cb, t1, sizeof(someglob) / sizeof(*someglob));
+        cb->define_var(cb, "someglob", t2, &someglob);
+#expect VAR someglob: ARRAY[100] int
+    }
+}

From noreply at buildbot.pypy.org  Tue Nov 18 16:16:51 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Tue, 18 Nov 2014 16:16:51 +0100 (CET)
Subject: [pypy-commit] creflect default: progress
Message-ID: <20141118151651.71B951D2487@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r69:91c5dbd9cca3
Date: 2014-11-18 16:17 +0100
http://bitbucket.org/cffi/creflect/changeset/91c5dbd9cca3/

Log:	progress

diff --git a/creflect/model.py b/creflect/model.py
--- a/creflect/model.py
+++ b/creflect/model.py
@@ -95,20 +95,7 @@
         return self.ALL_PRIMITIVE_TYPES[self.name] == 'f'
 
     def inspect_nonconst_type(self, block, inspect):
-        if not isinstance(inspect, TypeInspector):
-            if self.is_signed_type():
-                expr = 'cb->get_signed_type(cb, sizeof(%s), "%s")' % (
-                    self.name, self.name)
-            elif self.is_unsigned_type():
-                expr = 'cb->get_unsigned_type(cb, sizeof(%s), "%s")' % (
-                    self.name, self.name)
-            elif self.is_char_type():
-                expr = 'cb->get_char_type(cb)'
-            elif self.is_float_type():
-                xxx
-            else:
-                raise AssertionError
-        else:
+        if isinstance(inspect, TypeInspector):
             star_p1 = inspect.fetch_star_p1()
             comment1 = inspect.get_comment(0, False, "a valid type", minlevel=1)
             comment2 = inspect.get_comment(0, False, "an integer type")
@@ -128,6 +115,25 @@
                 xxx
             else:
                 raise AssertionError
+        else:
+            if isinstance(inspect, VarInspector):
+                decl = self.get_c_name("*p1")
+                block.writeline("%s = &%s;  /* check that '%s' is of type '%s'"
+                                " */" % (decl, inspect.varname,
+                                         inspect.varname, self.get_c_name()))
+                block.writeline("(void)p1;")
+            if self.is_signed_type():
+                expr = 'cb->get_signed_type(cb, sizeof(%s), "%s")' % (
+                    self.name, self.name)
+            elif self.is_unsigned_type():
+                expr = 'cb->get_unsigned_type(cb, sizeof(%s), "%s")' % (
+                    self.name, self.name)
+            elif self.is_char_type():
+                expr = 'cb->get_char_type(cb)'
+            elif self.is_float_type():
+                xxx
+            else:
+                raise AssertionError
         return block.write_crx_type_var(expr)
 
 
@@ -277,11 +283,21 @@
                 inspect.after_star_p1_assignment.append(after)
             inspect.levels.append('*')
         elif isinstance(inspect, VarInspector):
-            block.writeline("void *p1 = %s;  /* check that '%s' is a"
-                            " pointer */" % (inspect.varname, inspect.varname))
-            block.writeline("if (0) { %s = p1; }  /* check that '%s' is a"
-                            " pointer variable, and not an array or a"
-                            " constant */" % (inspect.varname, inspect.varname))
+            if isinstance(self.totype, VoidType):
+                block.writeline("void *p1 = %s;  /* check that '%s' is a"
+                                " pointer */" % (inspect.varname,
+                                                 inspect.varname))
+                block.writeline("if (0) { %s = p1; }  /* check that '%s' is a"
+                                " pointer variable, and not an array or a"
+                                " constant */" % (inspect.varname,
+                                                  inspect.varname))
+            else:
+                decl = self.totype.get_c_name("**p1")
+                block.writeline("%s = &%s;  /* check that '%s' is of type '%s'"
+                                " */" % (decl, inspect.varname,
+                                         inspect.varname, self.get_c_name()))
+                block.writeline("(void)p1;")
+            inspect = MissingInspector()
         t1 = self.totype.inspect_type(block, inspect)
         return block.write_crx_type_var('cb->get_pointer_type(cb, %s)' % t1)
 
@@ -332,6 +348,7 @@
                                      inspect.varname, self.get_c_name()))
             block.writeline("(void)p1;")
             star_p1 = inspect.varname
+            inspect = MissingInspector()
         else:
             star_p1 = None
         t1 = self.item.inspect_type(block, inspect)
diff --git a/test/codegen/glob-002c.c b/test/codegen/glob-002c.c
new file mode 100644
--- /dev/null
+++ b/test/codegen/glob-002c.c
@@ -0,0 +1,17 @@
+int someglob;
+
+// XXX can be improved to correct the size of the integer type
+
+# ____________________________________________________________
+
+void testglob_002c(crx_builder_t *cb)
+{
+    crx_type_t *t1;
+    {
+        int *p1 = &someglob;  /* check that 'someglob' is of type 'int' */
+        (void)p1;
+        t1 = cb->get_signed_type(cb, sizeof(int), "int");
+        cb->define_var(cb, "someglob", t1, &someglob);
+#expect VAR someglob: int
+    }
+}
diff --git a/test/codegen/glob-002d.c b/test/codegen/glob-002d.c
new file mode 100644
--- /dev/null
+++ b/test/codegen/glob-002d.c
@@ -0,0 +1,16 @@
+int *someglob;
+
+# ____________________________________________________________
+
+void testglob_002d(crx_builder_t *cb)
+{
+    crx_type_t *t1, *t2;
+    {
+        int **p1 = &someglob;  /* check that 'someglob' is of type 'int *' */
+        (void)p1;
+        t1 = cb->get_signed_type(cb, sizeof(int), "int");
+        t2 = cb->get_pointer_type(cb, t1);
+        cb->define_var(cb, "someglob", t2, &someglob);
+#expect VAR someglob: PTR int
+    }
+}

From noreply at buildbot.pypy.org  Tue Nov 18 17:14:30 2014
From: noreply at buildbot.pypy.org (Raemi)
Date: Tue, 18 Nov 2014 17:14:30 +0100 (CET)
Subject: [pypy-commit] pypy stmgc-c7: check for STM_WAIT_FREE_SEGMENT too
Message-ID: <20141118161430.F36871D28A4@cobra.cs.uni-duesseldorf.de>

Author: Remi Meier 
Branch: stmgc-c7
Changeset: r74571:d3bfa9d7cbef
Date: 2014-11-18 16:19 +0100
http://bitbucket.org/pypy/pypy/changeset/d3bfa9d7cbef/

Log:	check for STM_WAIT_FREE_SEGMENT too

diff --git a/pypy/stm/print_stm_log.py b/pypy/stm/print_stm_log.py
--- a/pypy/stm/print_stm_log.py
+++ b/pypy/stm/print_stm_log.py
@@ -187,7 +187,8 @@
                 t2 = threads.get(entry.otherthreadnum)
                 if t2 is not None and t2.in_transaction():
                     t2._conflict = ("remote", c, entry)
-        elif entry.event in (STM_WAIT_SYNC_PAUSE, STM_WAIT_CONTENTION):
+        elif entry.event in (STM_WAIT_SYNC_PAUSE, STM_WAIT_CONTENTION,
+                             STM_WAIT_FREE_SEGMENT):
             t = threads.get(entry.threadnum)
             if t is not None and t.in_transaction():
                 t.transaction_pause(entry)

From noreply at buildbot.pypy.org  Tue Nov 18 17:14:32 2014
From: noreply at buildbot.pypy.org (Raemi)
Date: Tue, 18 Nov 2014 17:14:32 +0100 (CET)
Subject: [pypy-commit] pypy stmgc-c7: add basic plot_stm_log script
Message-ID: <20141118161432.522161D28A4@cobra.cs.uni-duesseldorf.de>

Author: Remi Meier 
Branch: stmgc-c7
Changeset: r74572:08677229ea71
Date: 2014-11-18 17:14 +0100
http://bitbucket.org/pypy/pypy/changeset/08677229ea71/

Log:	add basic plot_stm_log script

diff --git a/pypy/stm/plot_stm_log.py b/pypy/stm/plot_stm_log.py
new file mode 100755
--- /dev/null
+++ b/pypy/stm/plot_stm_log.py
@@ -0,0 +1,188 @@
+#!/usr/bin/env python
+import matplotlib
+import sys
+matplotlib.use('gtkagg')
+
+args = None
+import matplotlib.pyplot as plt
+
+import print_stm_log as psl
+
+
+########## DRAWING STUFF ##########
+
+BOX_HEIGHT = 0.8
+HALF_HEIGHT = 0.1 + BOX_HEIGHT / 2
+QUARTER_HEIGHT = 0.1 + BOX_HEIGHT / 4
+
+
+def plot_boxes(boxes, y, ax):
+    coords = [(x, w) for x, w, c, i in boxes]
+    colors = [c for x, w, c, i in boxes]
+    bars = ax.broken_barh(coords, (y+0.1, BOX_HEIGHT),
+                          facecolors=colors, lw=1, edgecolor=(0, 0, 0),
+                          picker=True, antialiased=False, rasterized=True)
+
+    bars.boxes = boxes
+
+
+def plot_hlines(hlines, y, ax):
+    args = [[[x1, x2], [y+HALF_HEIGHT, y+HALF_HEIGHT], color] for
+            x1, x2, color in hlines]
+    # flatten:
+    args = [item for sublist in args for item in sublist]
+    ax.plot(*args, linewidth=3, antialiased=False, rasterized=True)
+
+
+####################################
+
+
+def add_box(boxes, x1, x2, color, info):
+    boxes.append((x1, x2-x1, color, info))
+
+
+def add_hline(hlines, x1, x2, color):
+    hlines.append((x1, x2, color))
+
+
+def add_transaction(boxes, hlines, inited, inevitabled,
+                    ended, aborted, atomics, info=""):
+    assert inited is not None
+
+    if inevitabled is not None:
+        add_box(boxes, inited, inevitabled, 'b', info)
+        add_box(boxes, inevitabled, ended, 'orange', info)
+    elif not aborted:
+        add_box(boxes, inited, ended, 'g', info)
+    else:
+        add_box(boxes, inited, ended, 'r', info)
+
+    for start, end in atomics:
+        add_hline(hlines, start, end, 'magenta')
+
+
+class Transaction(object):
+    def __init__(self, thread_num, start_time):
+        self.thread_num = thread_num
+        self.start_time = start_time
+        self.stop_time = 0
+        self.aborted = False
+        self.pauses = []
+
+
+def plot_log(logentries, ax):
+    curr_trs = {}
+    finished_trs = {}
+    for entry in logentries:
+        th_num = entry.threadnum
+
+        if entry.event == psl.STM_TRANSACTION_START:
+            if th_num in curr_trs:
+                print "WARNING: Start of transaction while there is one already running"
+            curr_trs[th_num] = Transaction(th_num, entry.timestamp)
+        elif entry.event == psl.STM_TRANSACTION_COMMIT:
+            tr = curr_trs.get(th_num)
+            if tr is not None:
+                tr.stop_time = entry.timestamp
+                xs = finished_trs.setdefault(th_num, [])
+                xs.append(curr_trs[th_num])
+                del curr_trs[th_num]
+        elif entry.event == psl.STM_TRANSACTION_ABORT:
+            tr = curr_trs.get(th_num)
+            if tr is not None:
+                tr.stop_time = entry.timestamp
+                tr.aborted = True
+                xs = finished_trs.setdefault(th_num, [])
+                xs.append(curr_trs[th_num])
+                del curr_trs[th_num]
+        elif entry.event in (psl.STM_WAIT_SYNC_PAUSE, psl.STM_WAIT_CONTENTION,
+                             psl.STM_WAIT_FREE_SEGMENT):
+            pass
+        elif entry.event == psl.STM_WAIT_DONE:
+            pass
+
+
+    plt.ion()
+    for th_num, trs in finished_trs.items():
+        plt.draw()
+        plt.show()
+        print "Thread", th_num
+
+        boxes = []
+        hlines = []
+        for tr in trs:
+            add_transaction(boxes, hlines,
+                            tr.start_time, None, tr.stop_time,
+                            tr.aborted, [])
+        plot_boxes(boxes, th_num, ax)
+        plot_hlines(hlines, th_num, ax)
+
+    plt.ioff()
+
+    return finished_trs
+
+
+def onpick(event):
+    if hasattr(event.artist, "info"):
+        print "== pick ==\n", event.artist.info
+    if hasattr(event.artist, "boxes"):
+        x = event.mouseevent.xdata
+        for x1, w, c, i in event.artist.boxes:
+            if x >= x1 and x <= x1+w:
+                print "== pick ==\n", i
+                break
+        else:
+            print "== pick ==\nCould not find info"
+
+
+
+
+def plot(logentries):
+    global fig
+
+
+    print "Draw..."
+    fig = plt.figure()
+    grid_spec = matplotlib.gridspec.GridSpec(1, 1)
+    axs = [fig.add_subplot(grid_spec[0])]
+
+    trs = plot_log(logentries, axs[0])
+
+    thread_count = len(trs)
+
+    axs[0].set_ylabel("Thread")
+    axs[0].set_ylim(0, thread_count)
+    axs[0].set_yticks([r+0.5 for r in range(thread_count)])
+    axs[0].set_yticklabels(range(1, thread_count+1))
+    #axs[0].set_xticks([])
+    print "Drawn."
+
+    axs[0].set_xlabel("Runtime [s]")
+
+    # adjust labels:
+    first_trs = [th[0].start_time for th in trs.values()]
+    last_trs = [th[-1].stop_time for th in trs.values()]
+    start_time, stop_time = min(first_trs), max(last_trs)
+    offset = (stop_time - start_time) * 0.1
+    axs[0].set_xlim((start_time - offset, stop_time + offset))
+
+    def label_format(x, pos):
+        return "%.2f" % (x - start_time)
+    major_formatter = matplotlib.ticker.FuncFormatter(label_format)
+    axs[0].xaxis.set_major_formatter(major_formatter)
+
+    # event connect
+    fig.canvas.mpl_connect('pick_event', onpick)
+
+    plt.draw()
+    plt.show()
+
+# ____________________________________________________________
+
+def main(argv):
+    assert len(argv) == 1, "expected a filename argument"
+    plot(psl.parse_log(argv[0]))
+    return 0
+
+if __name__ == '__main__':
+    sys.exit(main(sys.argv[1:]))

From noreply at buildbot.pypy.org  Tue Nov 18 17:21:43 2014
From: noreply at buildbot.pypy.org (Raemi)
Date: Tue, 18 Nov 2014 17:21:43 +0100 (CET)
Subject: [pypy-commit] pypy stmgc-c7: plot pauses
Message-ID: <20141118162143.AE90C1D28A4@cobra.cs.uni-duesseldorf.de>

Author: Remi Meier 
Branch: stmgc-c7
Changeset: r74573:bd12b97726c8
Date: 2014-11-18 17:21 +0100
http://bitbucket.org/pypy/pypy/changeset/bd12b97726c8/

Log:	plot pauses

diff --git a/pypy/stm/plot_stm_log.py b/pypy/stm/plot_stm_log.py
--- a/pypy/stm/plot_stm_log.py
+++ b/pypy/stm/plot_stm_log.py
@@ -31,7 +31,7 @@
             x1, x2, color in hlines]
     # flatten:
     args = [item for sublist in args for item in sublist]
-    ax.plot(*args, linewidth=3, antialiased=False, rasterized=True)
+    ax.plot(*args, linewidth=5, antialiased=False, rasterized=True)
 
 
 ####################################
@@ -46,7 +46,7 @@
 
 
 def add_transaction(boxes, hlines, inited, inevitabled,
-                    ended, aborted, atomics, info=""):
+                    ended, aborted, pauses, info=""):
     assert inited is not None
 
     if inevitabled is not None:
@@ -57,7 +57,7 @@
     else:
         add_box(boxes, inited, ended, 'r', info)
 
-    for start, end in atomics:
+    for start, end in pauses:
         add_hline(hlines, start, end, 'magenta')
 
 
@@ -97,27 +97,33 @@
                 del curr_trs[th_num]
         elif entry.event in (psl.STM_WAIT_SYNC_PAUSE, psl.STM_WAIT_CONTENTION,
                              psl.STM_WAIT_FREE_SEGMENT):
-            pass
+            tr = curr_trs.get(th_num)
+            if tr is not None:
+                tr.pauses.append((entry.timestamp, entry.timestamp))
         elif entry.event == psl.STM_WAIT_DONE:
-            pass
+            tr = curr_trs.get(th_num)
+            if tr is not None:
+                tr.pauses[-1] = (tr.pauses[-1][0], entry.timestamp)
 
 
-    plt.ion()
+    # plt.ion()
     for th_num, trs in finished_trs.items():
-        plt.draw()
-        plt.show()
+        # plt.draw()
+        # plt.show()
         print "Thread", th_num
+        print "> Transactions:", len(trs)
 
         boxes = []
         hlines = []
         for tr in trs:
             add_transaction(boxes, hlines,
                             tr.start_time, None, tr.stop_time,
-                            tr.aborted, [])
+                            tr.aborted, tr.pauses)
         plot_boxes(boxes, th_num, ax)
         plot_hlines(hlines, th_num, ax)
+        print "> Pauses:", len(hlines)
 
-    plt.ioff()
+    # plt.ioff()
 
     return finished_trs
 

From noreply at buildbot.pypy.org  Tue Nov 18 17:22:19 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Tue, 18 Nov 2014 17:22:19 +0100 (CET)
Subject: [pypy-commit] creflect default: First part of glob-003
Message-ID: <20141118162219.8D45A1D28A4@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r70:9af295178841
Date: 2014-11-18 17:16 +0100
http://bitbucket.org/cffi/creflect/changeset/9af295178841/

Log:	First part of glob-003

diff --git a/creflect/creflect.h b/creflect/creflect.h
--- a/creflect/creflect.h
+++ b/creflect/creflect.h
@@ -13,6 +13,15 @@
 typedef void (*crx_trampoline0_fn)(void *args[], void *res);
 typedef void (*crx_trampoline1_fn)(void *fn, void *args[], void *res);
 
+typedef union {
+    int as_int;
+    long as_long;
+    long long as_long_long;
+    unsigned int as_unsigned_int;
+    unsigned long as_unsigned_long;
+    unsigned long long as_unsigned_long_long;
+} crx_int_const_t;
+
 #define CRX_SELF struct _crx_builder_s *
 typedef struct _crx_builder_s {
     crx_type_t *(*get_void_type)(CRX_SELF);
@@ -43,6 +52,8 @@
                        void *);
     void (*define_func)(CRX_SELF, const char *, crx_type_t *,
                         crx_type_t *[], int, crx_trampoline0_fn, void *);
+    void (*define_int_const)(CRX_SELF, const char *, crx_type_t *,
+                             crx_int_const_t *);
     void (*error)(CRX_SELF, const char *);
 } crx_builder_t;
 #undef CRX_SELF
@@ -51,6 +62,16 @@
 #define CRX_INT_TYPE(cb, expr, guessname)                               \
     _creflect__int_type(cb, expr > 0, sizeof(expr), expr == 1, guessname)
 
+#define CRX_INT_CONST(cb, expr, vp)                                     \
+    _creflect__int_expr(cb, vp,                                         \
+                        "integer constant '" #expr "' is too large",    \
+                        !(((expr) * 0 + 4) << (sizeof(int)*8-2)),       \
+                        !(((expr) * 0L + 4L) << (sizeof(long)*8-2)),    \
+                        ((expr) * 0 - 1) > 0,                           \
+                        (expr) == (unsigned long long)(expr),           \
+                        (expr) == (long long)(expr),                    \
+                        (unsigned long long)(expr))
+
 __attribute__((unused))
 static crx_type_t *_creflect__int_type(crx_builder_t *cb, int expr_positive,
                                        size_t size_of_expr, int expr_equal_one,
@@ -63,3 +84,61 @@
     else
         return cb->get_unsigned_type(cb, size_of_expr, guessname);
 }
+
+__attribute__((unused))
+static crx_type_t *_creflect__int_expr(crx_builder_t *cb, crx_int_const_t *vp,
+                                       const char *toobig,
+                                       int fits_int, int fits_long,
+                                       int unsign, int fits_ull, int fits_ll,
+                                       unsigned long long value)
+{
+    size_t size;
+    const char *name;
+
+    if (unsign) {
+        if (!fits_ull) {
+            goto overflow;
+        }
+        else if (fits_int) {
+            vp->as_unsigned_int = (unsigned int)value;
+            size = sizeof(unsigned int);
+            name = "unsigned int";
+        }
+        else if (fits_long) {
+            vp->as_unsigned_long = (unsigned long)value;
+            size = sizeof(unsigned long);
+            name = "unsigned long";
+        }
+        else {
+            vp->as_unsigned_long_long = (unsigned long long)value;
+            size = sizeof(unsigned long long);
+            name = "unsigned long long";
+        }
+        return cb->get_unsigned_type(cb, size, name);
+    }
+    else {   /* signed */
+        if (!fits_ll) {
+            goto overflow;
+        }
+        else if (fits_int) {
+            vp->as_int = (int)value;
+            size = sizeof(int);
+            name = "int";
+        }
+        else if (fits_long) {
+            vp->as_long = (long)value;
+            size = sizeof(long);
+            name = "long";
+        }
+        else {
+            vp->as_long_long = (long long)value;
+            size = sizeof(long long);
+            name = "long long";
+        }
+        return cb->get_signed_type(cb, size, name);
+    }
+
+ overflow:
+    cb->error(cb, toobig);
+    return 0;
+}
diff --git a/test/cgcompile.c b/test/cgcompile.c
--- a/test/cgcompile.c
+++ b/test/cgcompile.c
@@ -174,6 +174,28 @@
     printf("%s\n", ret->text);
 }
 
+static void tst_define_int_const(crx_builder_t *cb, const char *name,
+                                 crx_type_t *t, crx_int_const_t *value)
+{
+    printf("INTCONST %s = %s ", name, t->text);
+    if (strcmp(t->text, "int") == 0)
+        printf("%d\n", value->as_int);
+    else if (strcmp(t->text, "long") == 0)
+        printf("%ld\n", value->as_long);
+    else if (strcmp(t->text, "long long") == 0)
+        printf("%lld\n", value->as_long_long);
+    else if (strcmp(t->text, "unsigned int") == 0)
+        printf("%u\n", value->as_unsigned_int);
+    else if (strcmp(t->text, "unsigned long") == 0)
+        printf("%lu\n", value->as_unsigned_long);
+    else if (strcmp(t->text, "unsigned long long") == 0)
+        printf("%llu\n", value->as_unsigned_long_long);
+    else {
+        printf("???\n");
+        abort();
+    }
+}
+
 static void tst_error(crx_builder_t *cb, const char *msg)
 {
     printf("ERROR: %s\n", msg);
@@ -200,6 +222,7 @@
     tst_define_type,
     tst_define_var,
     tst_define_func,
+    tst_define_int_const,
     tst_error,
 };
 
diff --git a/test/codegen/glob-003.c b/test/codegen/glob-003.c
--- a/test/codegen/glob-003.c
+++ b/test/codegen/glob-003.c
@@ -2,57 +2,14 @@
 
 # ____________________________________________________________
 
-int testglob_003(char *r)
+void testglob_003(crx_builder_t *cb)
 {
-    int e = 0;
-    char v[32];
-    if (!r)
-        return 64 + 1;
+    crx_type_t *t1;
     {
-        int z1, z2;
-        (void)(ab << 1);  /* check that 'ab' is an integer */
-        z1 = !((ab * 0 + 4) << (sizeof(int)*8-2));
-        z2 = !((ab * 0L + 4L) << (sizeof(long)*8-2));
-        if ((ab * 0 - 1) > 0) {    /* unsigned */
-            if (ab != (unsigned long long)ab) {
-                r += sprintf(r, "#error unsigned integer constant 'ab' is too large\n");
-                e = -1;
-                goto f1;
-            }
-            if (z1) {
-                r += sprintf(r, "unsigned int");
-                sprintf(v, "%u", (unsigned int)ab);
-            }
-            else if (z2) {
-                r += sprintf(r, "unsigned long");
-                sprintf(v, "%lu", (unsigned long)ab);
-            }
-            else {
-                r += sprintf(r, "unsigned long long");
-                sprintf(v, "%llu", (unsigned long long)ab);
-            }
-        }
-        else {    /* signed */
-            if (ab != (long long)ab) {
-                r += sprintf(r, "#error integer constant 'ab' is too large\n");
-                e = -1;
-                goto f1;
-            }
-            if (z1) {
-                r += sprintf(r, "int");
-                sprintf(v, "%d", (int)ab);
-            }
-            else if (z2) {
-                r += sprintf(r, "long");
-                sprintf(v, "%ld", (long)ab);
-            }
-            else {
-                r += sprintf(r, "long long");
-                sprintf(v, "%lld", (long long)ab);
-            }
-        }
+        crx_int_const_t v;
+        (void)((ab) << 1);  /* check that 'ab' is an integer */
+        t1 = CRX_INT_CONST(cb, ab, &v);
+        cb->define_int_const(cb, "ab", t1, &v);
+#expect INTCONST ab = int -42
     }
-    r += sprintf(r, " const ab = %s;\n", v);
- f1:
-    return e;
 }

From noreply at buildbot.pypy.org  Tue Nov 18 17:22:20 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Tue, 18 Nov 2014 17:22:20 +0100 (CET)
Subject: [pypy-commit] creflect default: constdecl
Message-ID: <20141118162220.AFC1E1D28A4@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r71:bdd95bc342bd
Date: 2014-11-18 17:22 +0100
http://bitbucket.org/cffi/creflect/changeset/bdd95bc342bd/

Log:	constdecl

diff --git a/creflect/cparser.py b/creflect/cparser.py
--- a/creflect/cparser.py
+++ b/creflect/cparser.py
@@ -114,7 +114,7 @@
             if decl.name:
                 tp = self._get_type(node, partial_length_ok=True)
                 if self._is_constant_globalvar(node):
-                    self._declare('constant ' + decl.name, tp)
+                    self.declarations.append(model.ConstDecl(decl.name, tp))
                 else:
                     self.declarations.append(model.VarDecl(decl.name, tp))
 
diff --git a/creflect/model.py b/creflect/model.py
--- a/creflect/model.py
+++ b/creflect/model.py
@@ -136,6 +136,15 @@
                 raise AssertionError
         return block.write_crx_type_var(expr)
 
+    def write_int_const_decl(self, block, varname):
+        block.writedecl("crx_int_const_t v;")
+        block.writeline("(void)((%s) << 1);  /* check that '%s' is an"
+                        " integer */" % (varname, varname))
+        expr = "CRX_INT_CONST(cb, %s, &v)" % varname
+        t1 = block.write_crx_type_var(expr)
+        block.writeline('cb->define_int_const(cb, "%s", %s, &v);' % (
+            varname, t1))
+
 
 class FunctionType(BaseType):
     _attrs_ = ('args', 'result', 'ellipsis')
@@ -591,10 +600,6 @@
 
 
 class FuncDecl(VarDecl):
-    def __init__(self, name, type):
-        self.name = name
-        self.type = type
-
     def write_declaration(self, funcblock):
         if self.type.ellipsis:
             VarDecl.write_declaration(self, funcblock)
@@ -602,3 +607,13 @@
             block = CodeBlock(funcblock)
             self.type.write_func_decl(block, self.name)
             funcblock.write_subblock(block)
+
+
+class ConstDecl(VarDecl):
+    def write_declaration(self, funcblock):
+        if isinstance(self.type, PrimitiveType):
+            block = CodeBlock(funcblock)
+            self.type.write_int_const_decl(block, self.name)
+            funcblock.write_subblock(block)
+        else:
+            VarDecl.write_declaration(self, funcblock)

From noreply at buildbot.pypy.org  Tue Nov 18 17:38:09 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Tue, 18 Nov 2014 17:38:09 +0100 (CET)
Subject: [pypy-commit] creflect default: Constant chars
Message-ID: <20141118163809.B3BA01D28C3@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r72:6131fc269b64
Date: 2014-11-18 17:38 +0100
http://bitbucket.org/cffi/creflect/changeset/6131fc269b64/

Log:	Constant chars

diff --git a/creflect/creflect.h b/creflect/creflect.h
--- a/creflect/creflect.h
+++ b/creflect/creflect.h
@@ -20,6 +20,7 @@
     unsigned int as_unsigned_int;
     unsigned long as_unsigned_long;
     unsigned long long as_unsigned_long_long;
+    char as_char;
 } crx_int_const_t;
 
 #define CRX_SELF struct _crx_builder_s *
@@ -62,15 +63,21 @@
 #define CRX_INT_TYPE(cb, expr, guessname)                               \
     _creflect__int_type(cb, expr > 0, sizeof(expr), expr == 1, guessname)
 
-#define CRX_INT_CONST(cb, expr, vp)                                     \
-    _creflect__int_expr(cb, vp,                                         \
-                        "integer constant '" #expr "' is too large",    \
-                        !(((expr) * 0 + 4) << (sizeof(int)*8-2)),       \
-                        !(((expr) * 0L + 4L) << (sizeof(long)*8-2)),    \
-                        ((expr) * 0 - 1) > 0,                           \
-                        (expr) == (unsigned long long)(expr),           \
-                        (expr) == (long long)(expr),                    \
-                        (unsigned long long)(expr))
+#define CRX_INT_CONST(cb, expr, vp)                     \
+    _creflect__int_const(cb, vp,                        \
+        "integer constant '" #expr "' is too large",    \
+        !(((expr) * 0 + 4) << (sizeof(int)*8-2)),       \
+        !(((expr) * 0L + 4L) << (sizeof(long)*8-2)),    \
+        ((expr) * 0 - 1) > 0,                           \
+        (expr) == (unsigned long long)(expr),           \
+        (expr) == (long long)(expr),                    \
+        (unsigned long long)(expr))
+
+#define CRX_CHAR_CONST(cb, expr, vp)                                    \
+    _creflect__char_const(cb, vp,                                       \
+        "char constant '" #expr "' is too large",                       \
+        (expr) == (signed char)(expr) || (expr) == (unsigned char)(expr), \
+        (char)(expr))
 
 __attribute__((unused))
 static crx_type_t *_creflect__int_type(crx_builder_t *cb, int expr_positive,
@@ -86,11 +93,11 @@
 }
 
 __attribute__((unused))
-static crx_type_t *_creflect__int_expr(crx_builder_t *cb, crx_int_const_t *vp,
-                                       const char *toobig,
-                                       int fits_int, int fits_long,
-                                       int unsign, int fits_ull, int fits_ll,
-                                       unsigned long long value)
+static crx_type_t *_creflect__int_const(crx_builder_t *cb, crx_int_const_t *vp,
+                                        const char *toobig,
+                                        int fits_int, int fits_long,
+                                        int unsign, int fits_ull, int fits_ll,
+                                        unsigned long long value)
 {
     size_t size;
     const char *name;
@@ -142,3 +149,14 @@
     cb->error(cb, toobig);
     return 0;
 }
+
+__attribute__((unused))
+static void _creflect__char_const(crx_builder_t *cb, crx_int_const_t *vp,
+                                  const char *toobig,
+                                  int fits_char, char value)
+{
+    if (!fits_char)
+        cb->error(cb, toobig);
+    else
+        vp->as_char = value;
+}
diff --git a/creflect/model.py b/creflect/model.py
--- a/creflect/model.py
+++ b/creflect/model.py
@@ -138,9 +138,18 @@
 
     def write_int_const_decl(self, block, varname):
         block.writedecl("crx_int_const_t v;")
-        block.writeline("(void)((%s) << 1);  /* check that '%s' is an"
-                        " integer */" % (varname, varname))
-        expr = "CRX_INT_CONST(cb, %s, &v)" % varname
+        comment = "an integer"
+        if self.is_char_type():
+            comment += " or char"
+        block.writeline("(void)((%s) << 1);  /* check that '%s' is %s */" % (
+            varname, varname, comment))
+        if self.is_integer_type():
+            expr = "CRX_INT_CONST(cb, %s, &v)" % varname
+        elif self.is_char_type():
+            block.writeline('CRX_CHAR_CONST(cb, %s, &v);' % varname)
+            expr = 'cb->get_char_type(cb)'
+        else:
+            xxxxxxx
         t1 = block.write_crx_type_var(expr)
         block.writeline('cb->define_int_const(cb, "%s", %s, &v);' % (
             varname, t1))
diff --git a/test/cgcompile.c b/test/cgcompile.c
--- a/test/cgcompile.c
+++ b/test/cgcompile.c
@@ -190,6 +190,8 @@
         printf("%lu\n", value->as_unsigned_long);
     else if (strcmp(t->text, "unsigned long long") == 0)
         printf("%llu\n", value->as_unsigned_long_long);
+    else if (strcmp(t->text, "char") == 0)
+        printf("'%c'\n", value->as_char);
     else {
         printf("???\n");
         abort();
diff --git a/test/codegen/glob-003b.c b/test/codegen/glob-003b.c
--- a/test/codegen/glob-003b.c
+++ b/test/codegen/glob-003b.c
@@ -2,57 +2,14 @@
 
 # ____________________________________________________________
 
-int testglob_003b(char *r)
+void testglob_003b(crx_builder_t *cb)
 {
-    int e = 0;
-    char v[32];
-    if (!r)
-        return 999 + 1;
+    crx_type_t *t1;
     {
-        int z1, z2;
-        (void)(ab << 1);  /* check that 'ab' is an integer */
-        z1 = !((ab * 0 + 4) << (sizeof(int)*8-2));
-        z2 = !((ab * 0L + 4L) << (sizeof(long)*8-2));
-        if ((ab * 0 - 1) > 0) {    /* unsigned */
-            if (ab != (unsigned long long)ab) {
-                r += sprintf(r, "#error unsigned integer constant 'ab' is too large\n");
-                e = -1;
-                goto f1;
-            }
-            if (z1) {
-                r += sprintf(r, "unsigned int");
-                sprintf(v, "%u", (unsigned int)ab);
-            }
-            else if (z2) {
-                r += sprintf(r, "unsigned long");
-                sprintf(v, "%lu", (unsigned long)ab);
-            }
-            else {
-                r += sprintf(r, "unsigned long long");
-                sprintf(v, "%llu", (unsigned long long)ab);
-            }
-        }
-        else {    /* signed */
-            if (ab != (long long)ab) {
-                r += sprintf(r, "#error integer constant 'ab' is too large\n");
-                e = -1;
-                goto f1;
-            }
-            if (z1) {
-                r += sprintf(r, "int");
-                sprintf(v, "%d", (int)ab);
-            }
-            else if (z2) {
-                r += sprintf(r, "long");
-                sprintf(v, "%ld", (long)ab);
-            }
-            else {
-                r += sprintf(r, "long long");
-                sprintf(v, "%lld", (long long)ab);
-            }
-        }
+        crx_int_const_t v;
+        (void)((ab) << 1);  /* check that 'ab' is an integer */
+        t1 = CRX_INT_CONST(cb, ab, &v);
+        cb->define_int_const(cb, "ab", t1, &v);
+#expect INTCONST ab = int 42
     }
-    r += sprintf(r, " const ab = %s;\n", v);
- f1:
-    return e;
 }
diff --git a/test/codegen/glob-003c.c b/test/codegen/glob-003c.c
--- a/test/codegen/glob-003c.c
+++ b/test/codegen/glob-003c.c
@@ -2,57 +2,14 @@
 
 # ____________________________________________________________
 
-int testglob_003c(char *r)
+void testglob_003c(crx_builder_t *cb)
 {
-    int e = 0;
-    char v[32];
-    if (!r)
-        return 999 + 1;
+    crx_type_t *t1;
     {
-        int z1, z2;
-        (void)(ab << 1);  /* check that 'ab' is an integer */
-        z1 = !((ab * 0 + 4) << (sizeof(int)*8-2));
-        z2 = !((ab * 0L + 4L) << (sizeof(long)*8-2));
-        if ((ab * 0 - 1) > 0) {    /* unsigned */
-            if (ab != (unsigned long long)ab) {
-                r += sprintf(r, "#error unsigned integer constant 'ab' is too large\n");
-                e = -1;
-                goto f1;
-            }
-            if (z1) {
-                r += sprintf(r, "unsigned int");
-                sprintf(v, "%u", (unsigned int)ab);
-            }
-            else if (z2) {
-                r += sprintf(r, "unsigned long");
-                sprintf(v, "%lu", (unsigned long)ab);
-            }
-            else {
-                r += sprintf(r, "unsigned long long");
-                sprintf(v, "%llu", (unsigned long long)ab);
-            }
-        }
-        else {    /* signed */
-            if (ab != (long long)ab) {
-                r += sprintf(r, "#error integer constant 'ab' is too large\n");
-                e = -1;
-                goto f1;
-            }
-            if (z1) {
-                r += sprintf(r, "int");
-                sprintf(v, "%d", (int)ab);
-            }
-            else if (z2) {
-                r += sprintf(r, "long");
-                sprintf(v, "%ld", (long)ab);
-            }
-            else {
-                r += sprintf(r, "long long");
-                sprintf(v, "%lld", (long long)ab);
-            }
-        }
+        crx_int_const_t v;
+        (void)((ab) << 1);  /* check that 'ab' is an integer */
+        t1 = CRX_INT_CONST(cb, ab, &v);
+        cb->define_int_const(cb, "ab", t1, &v);
+#expect INTCONST ab = unsigned int 42
     }
-    r += sprintf(r, " const ab = %s;\n", v);
- f1:
-    return e;
 }
diff --git a/test/codegen/glob-003d.c b/test/codegen/glob-003d.c
--- a/test/codegen/glob-003d.c
+++ b/test/codegen/glob-003d.c
@@ -2,61 +2,15 @@
 
 # ____________________________________________________________
 
-int testglob_003d(char *r)
+void testglob_003d(crx_builder_t *cb)
 {
-    int e = 0;
-    char v[32];
-    if (!r)
-        return 52;
+    crx_type_t *t1;
     {
-        int z1, z2;
-        (void)(ab << 1);  /* check that 'ab' is an integer */
-        z1 = !((ab * 0 + 4) << (sizeof(int)*8-2));
-        z2 = !((ab * 0L + 4L) << (sizeof(long)*8-2));
-        if (ab == (char)ab) {
-            r += sprintf(r, "char");
-            sprintf(v, "%d", (int)ab);
-        }
-        else if ((ab * 0 - 1) > 0) {    /* unsigned */
-            if (ab != (unsigned long long)ab) {
-                r += sprintf(r, "#error unsigned integer constant 'ab' is too large\n");
-                e = -1;
-                goto f1;
-            }
-            if (z1) {
-                r += sprintf(r, "unsigned int");
-                sprintf(v, "%u", (unsigned int)ab);
-            }
-            else if (z2) {
-                r += sprintf(r, "unsigned long");
-                sprintf(v, "%lu", (unsigned long)ab);
-            }
-            else {
-                r += sprintf(r, "unsigned long long");
-                sprintf(v, "%llu", (unsigned long long)ab);
-            }
-        }
-        else {    /* signed */
-            if (ab != (long long)ab) {
-                r += sprintf(r, "#error integer constant 'ab' is too large\n");
-                e = -1;
-                goto f1;
-            }
-            if (z1) {
-                r += sprintf(r, "int");
-                sprintf(v, "%d", (int)ab);
-            }
-            else if (z2) {
-                r += sprintf(r, "long");
-                sprintf(v, "%ld", (long)ab);
-            }
-            else {
-                r += sprintf(r, "long long");
-                sprintf(v, "%lld", (long long)ab);
-            }
-        }
+        crx_int_const_t v;
+        (void)((ab) << 1);  /* check that 'ab' is an integer or char */
+        CRX_CHAR_CONST(cb, ab, &v);
+        t1 = cb->get_char_type(cb);
+        cb->define_int_const(cb, "ab", t1, &v);
+#expect INTCONST ab = char '*'
     }
-    r += sprintf(r, " const ab = %s;\n", v);
- f1:
-    return e;
 }

From noreply at buildbot.pypy.org  Tue Nov 18 17:41:33 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Tue, 18 Nov 2014 17:41:33 +0100 (CET)
Subject: [pypy-commit] creflect default: Test that "const int" also work
	with #defines
Message-ID: <20141118164133.80AE41C104D@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r73:91a86160c338
Date: 2014-11-18 17:41 +0100
http://bitbucket.org/cffi/creflect/changeset/91a86160c338/

Log:	Test that "const int" also work with #defines

diff --git a/test/codegen/glob-003b.c b/test/codegen/glob-003b.c
--- a/test/codegen/glob-003b.c
+++ b/test/codegen/glob-003b.c
@@ -1,4 +1,8 @@
-int const ab = 42;
+const int ab;
+
+# ____________________________________________________________
+
+#define ab 42
 
 # ____________________________________________________________
 
diff --git a/test/test_cgcompile.py b/test/test_cgcompile.py
--- a/test/test_cgcompile.py
+++ b/test/test_cgcompile.py
@@ -25,7 +25,7 @@
     f.write('#define TESTFN(cb) test%s(cb)\n' % (basename.replace('-','_'),))
     f.write('\n')
     for line in inputlines:
-        if not line.startswith('#') or line.startswith('#include'):
+        if not (line.startswith('# ') or line.startswith('#expect')):
             f.write(line)
     f.write('\n')
     f.write('#include "%s/../cgcompile.c"\n' % path)

From noreply at buildbot.pypy.org  Tue Nov 18 17:50:55 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Tue, 18 Nov 2014 17:50:55 +0100 (CET)
Subject: [pypy-commit] creflect default: Preference number for
 otherwise-equivalent integer types
Message-ID: <20141118165055.C6DBB1C1147@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r74:38fe2925e976
Date: 2014-11-18 17:51 +0100
http://bitbucket.org/cffi/creflect/changeset/38fe2925e976/

Log:	Preference number for otherwise-equivalent integer types

diff --git a/creflect/creflect.h b/creflect/creflect.h
--- a/creflect/creflect.h
+++ b/creflect/creflect.h
@@ -63,8 +63,8 @@
 #define CRX_INT_TYPE(cb, expr, guessname)                               \
     _creflect__int_type(cb, expr > 0, sizeof(expr), expr == 1, guessname)
 
-#define CRX_INT_CONST(cb, expr, vp)                     \
-    _creflect__int_const(cb, vp,                        \
+#define CRX_INT_CONST(cb, expr, vp, pn)                 \
+    _creflect__int_const(cb, vp, pn,                    \
         "integer constant '" #expr "' is too large",    \
         !(((expr) * 0 + 4) << (sizeof(int)*8-2)),       \
         !(((expr) * 0L + 4L) << (sizeof(long)*8-2)),    \
@@ -94,6 +94,7 @@
 
 __attribute__((unused))
 static crx_type_t *_creflect__int_const(crx_builder_t *cb, crx_int_const_t *vp,
+                                        int preference_number,
                                         const char *toobig,
                                         int fits_int, int fits_long,
                                         int unsign, int fits_ull, int fits_ll,
@@ -102,6 +103,13 @@
     size_t size;
     const char *name;
 
+    if (preference_number >= 2)       /* "prefer long" */
+        if (sizeof(int) == sizeof(long))
+            fits_int = 0;
+    if (preference_number >= 3)       /* "prefer long long" */
+        if (sizeof(long) == sizeof(long long))
+            fits_long = 0;
+
     if (unsign) {
         if (!fits_ull) {
             goto overflow;
diff --git a/creflect/model.py b/creflect/model.py
--- a/creflect/model.py
+++ b/creflect/model.py
@@ -144,7 +144,14 @@
         block.writeline("(void)((%s) << 1);  /* check that '%s' is %s */" % (
             varname, varname, comment))
         if self.is_integer_type():
-            expr = "CRX_INT_CONST(cb, %s, &v)" % varname
+            if self.name.endswith('long'):
+                if self.name.endswith('long long'):
+                    preference = 3
+                else:
+                    preference = 2
+            else:
+                preference = 1
+            expr = "CRX_INT_CONST(cb, %s, &v, %d)" % (varname, preference)
         elif self.is_char_type():
             block.writeline('CRX_CHAR_CONST(cb, %s, &v);' % varname)
             expr = 'cb->get_char_type(cb)'
diff --git a/test/codegen/glob-003.c b/test/codegen/glob-003.c
--- a/test/codegen/glob-003.c
+++ b/test/codegen/glob-003.c
@@ -8,7 +8,7 @@
     {
         crx_int_const_t v;
         (void)((ab) << 1);  /* check that 'ab' is an integer */
-        t1 = CRX_INT_CONST(cb, ab, &v);
+        t1 = CRX_INT_CONST(cb, ab, &v, 1);
         cb->define_int_const(cb, "ab", t1, &v);
 #expect INTCONST ab = int -42
     }
diff --git a/test/codegen/glob-003b.c b/test/codegen/glob-003b.c
--- a/test/codegen/glob-003b.c
+++ b/test/codegen/glob-003b.c
@@ -12,7 +12,7 @@
     {
         crx_int_const_t v;
         (void)((ab) << 1);  /* check that 'ab' is an integer */
-        t1 = CRX_INT_CONST(cb, ab, &v);
+        t1 = CRX_INT_CONST(cb, ab, &v, 1);
         cb->define_int_const(cb, "ab", t1, &v);
 #expect INTCONST ab = int 42
     }
diff --git a/test/codegen/glob-003c.c b/test/codegen/glob-003c.c
--- a/test/codegen/glob-003c.c
+++ b/test/codegen/glob-003c.c
@@ -8,7 +8,7 @@
     {
         crx_int_const_t v;
         (void)((ab) << 1);  /* check that 'ab' is an integer */
-        t1 = CRX_INT_CONST(cb, ab, &v);
+        t1 = CRX_INT_CONST(cb, ab, &v, 1);
         cb->define_int_const(cb, "ab", t1, &v);
 #expect INTCONST ab = unsigned int 42
     }
diff --git a/test/codegen/glob-003e.c b/test/codegen/glob-003e.c
--- a/test/codegen/glob-003e.c
+++ b/test/codegen/glob-003e.c
@@ -2,58 +2,14 @@
 
 # ____________________________________________________________
 
-int testglob_003e(char *r)
+void testglob_003e(crx_builder_t *cb)
 {
-    int e = 0;
-    char v[32];
-    if (!r)
-        return 999 + 1;
+    crx_type_t *t1;
     {
-        int z1, z2;
-        (void)(ab << 1);  /* check that 'ab' is an integer */
-        z1 = !((ab * 0 + 4) << (sizeof(int)*8-2));
-        z2 = !((ab * 0L + 4L) << (sizeof(long)*8-2));
-        if (sizeof(int) == sizeof(long)) z1 = 0;
-        if ((ab * 0 - 1) > 0) {    /* unsigned */
-            if (ab != (unsigned long long)ab) {
-                r += sprintf(r, "#error unsigned integer constant 'ab' is too large\n");
-                e = -1;
-                goto f1;
-            }
-            if (z1) {
-                r += sprintf(r, "unsigned int");
-                sprintf(v, "%u", (unsigned int)ab);
-            }
-            else if (z2) {
-                r += sprintf(r, "unsigned long");
-                sprintf(v, "%lu", (unsigned long)ab);
-            }
-            else {
-                r += sprintf(r, "unsigned long long");
-                sprintf(v, "%llu", (unsigned long long)ab);
-            }
-        }
-        else {    /* signed */
-            if (ab != (long long)ab) {
-                r += sprintf(r, "#error integer constant 'ab' is too large\n");
-                e = -1;
-                goto f1;
-            }
-            if (z1) {
-                r += sprintf(r, "int");
-                sprintf(v, "%d", (int)ab);
-            }
-            else if (z2) {
-                r += sprintf(r, "long");
-                sprintf(v, "%ld", (long)ab);
-            }
-            else {
-                r += sprintf(r, "long long");
-                sprintf(v, "%lld", (long long)ab);
-            }
-        }
+        crx_int_const_t v;
+        (void)((ab) << 1);  /* check that 'ab' is an integer */
+        t1 = CRX_INT_CONST(cb, ab, &v, 2);
+        cb->define_int_const(cb, "ab", t1, &v);
+#expect INTCONST ab = long -42
     }
-    r += sprintf(r, " const ab = %s;\n", v);
- f1:
-    return e;
 }
diff --git a/test/codegen/glob-003f.c b/test/codegen/glob-003f.c
--- a/test/codegen/glob-003f.c
+++ b/test/codegen/glob-003f.c
@@ -2,59 +2,14 @@
 
 # ____________________________________________________________
 
-int testglob_003f(char *r)
+void testglob_003f(crx_builder_t *cb)
 {
-    int e = 0;
-    char v[32];
-    if (!r)
-        return 999 + 1;
+    crx_type_t *t1;
     {
-        int z1, z2;
-        (void)(ab << 1);  /* check that 'ab' is an integer */
-        z1 = !((ab * 0 + 4) << (sizeof(int)*8-2));
-        z2 = !((ab * 0L + 4L) << (sizeof(long)*8-2));
-        if (sizeof(int) == sizeof(long)) z1 = 0;
-        if (sizeof(long) == sizeof(long long)) z2 = 0;
-        if ((ab * 0 - 1) > 0) {    /* unsigned */
-            if (ab != (unsigned long long)ab) {
-                r += sprintf(r, "#error unsigned integer constant 'ab' is too large\n");
-                e = -1;
-                goto f1;
-            }
-            if (z1) {
-                r += sprintf(r, "unsigned int");
-                sprintf(v, "%u", (unsigned int)ab);
-            }
-            else if (z2) {
-                r += sprintf(r, "unsigned long");
-                sprintf(v, "%lu", (unsigned long)ab);
-            }
-            else {
-                r += sprintf(r, "unsigned long long");
-                sprintf(v, "%llu", (unsigned long long)ab);
-            }
-        }
-        else {    /* signed */
-            if (ab != (long long)ab) {
-                r += sprintf(r, "#error integer constant 'ab' is too large\n");
-                e = -1;
-                goto f1;
-            }
-            if (z1) {
-                r += sprintf(r, "int");
-                sprintf(v, "%d", (int)ab);
-            }
-            else if (z2) {
-                r += sprintf(r, "long");
-                sprintf(v, "%ld", (long)ab);
-            }
-            else {
-                r += sprintf(r, "long long");
-                sprintf(v, "%lld", (long long)ab);
-            }
-        }
+        crx_int_const_t v;
+        (void)((ab) << 1);  /* check that 'ab' is an integer */
+        t1 = CRX_INT_CONST(cb, ab, &v, 3);
+        cb->define_int_const(cb, "ab", t1, &v);
+#expect INTCONST ab = long long -42
     }
-    r += sprintf(r, " const ab = %s;\n", v);
- f1:
-    return e;
 }

From noreply at buildbot.pypy.org  Tue Nov 18 17:51:37 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Tue, 18 Nov 2014 17:51:37 +0100 (CET)
Subject: [pypy-commit] creflect default: adapt next test
Message-ID: <20141118165137.2254F1C1147@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r75:760e20f512dc
Date: 2014-11-18 17:51 +0100
http://bitbucket.org/cffi/creflect/changeset/760e20f512dc/

Log:	adapt next test

diff --git a/test/codegen/glob-003g.c b/test/codegen/glob-003g.c
--- a/test/codegen/glob-003g.c
+++ b/test/codegen/glob-003g.c
@@ -2,61 +2,14 @@
 
 # ____________________________________________________________
 
-int testglob_003g(char *r)
+void testglob_003g(crx_builder_t *cb)
 {
-    int e = 0;
-    char v[48];
-    if (!r)
-        return 63 + 1;
+    crx_type_t *t1;
     {
-        int z1, z2;
-        (void)(ab << 1);  /* check that 'ab' is an integer */
-        z1 = !((ab * 0 + 4) << (sizeof(int)*8-2));
-        z2 = !((ab * 0L + 4L) << (sizeof(long)*8-2));
-        if (ab == (short)ab) {
-            r += sprintf(r, "short");
-            sprintf(v, "%d", (int)ab);
-        }
-        else if ((ab * 0 - 1) > 0) {    /* unsigned */
-            if (ab != (unsigned long long)ab) {
-                r += sprintf(r, "#error unsigned integer constant 'ab' is too large\n");
-                e = -1;
-                goto f1;
-            }
-            if (z1) {
-                r += sprintf(r, "unsigned int");
-                sprintf(v, "%u", (unsigned int)ab);
-            }
-            else if (z2) {
-                r += sprintf(r, "unsigned long");
-                sprintf(v, "%lu", (unsigned long)ab);
-            }
-            else {
-                r += sprintf(r, "unsigned long long");
-                sprintf(v, "%llu", (unsigned long long)ab);
-            }
-        }
-        else {    /* signed */
-            if (ab != (long long)ab) {
-                r += sprintf(r, "#error integer constant 'ab' is too large\n");
-                e = -1;
-                goto f1;
-            }
-            if (z1) {
-                r += sprintf(r, "int");
-                sprintf(v, "%d", (int)ab);
-            }
-            else if (z2) {
-                r += sprintf(r, "long");
-                sprintf(v, "%ld", (long)ab);
-            }
-            else {
-                r += sprintf(r, "long long");
-                sprintf(v, "%lld", (long long)ab);
-            }
-        }
+        crx_int_const_t v;
+        (void)((ab) << 1);  /* check that 'ab' is an integer */
+        t1 = CRX_INT_CONST(cb, ab, &v, 1);
+        cb->define_int_const(cb, "ab", t1, &v);
+#expect INTCONST ab = int -42
     }
-    r += sprintf(r, " const ab = %s;\n", v);
- f1:
-    return e;
 }

From noreply at buildbot.pypy.org  Tue Nov 18 18:02:01 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Tue, 18 Nov 2014 18:02:01 +0100 (CET)
Subject: [pypy-commit] creflect default: Global function pointers
Message-ID: <20141118170201.DF7FF1C1147@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r76:5f6346bf551d
Date: 2014-11-18 18:02 +0100
http://bitbucket.org/cffi/creflect/changeset/5f6346bf551d/

Log:	Global function pointers

diff --git a/creflect/model.py b/creflect/model.py
--- a/creflect/model.py
+++ b/creflect/model.py
@@ -209,10 +209,11 @@
 
         inspect.flush()
 
-        # limitations so far:
-        assert isinstance(inspect, TypeInspector)
-        assert inspect.started, "XXX"
-        assert inspect.levels == ['*'], "XXX"
+        if (isinstance(inspect, TypeInspector) and inspect.started and
+                inspect.levels == ['*']):    # common case
+            decl = '%s f' % inspect.typename
+        else:
+            decl = self.get_c_name('(*f)')
 
         toplevel = block.crx_top_level
         extraname = block._get_next_name('f')
@@ -220,7 +221,7 @@
         wrline = toplevel.writeline
         wrline('static void %s(void *func, void *args[], void *result) {' % (
             crx_func_name,))
-        wrline('    %s f = func;' % inspect.typename)
+        wrline('    %s = func;' % decl)
         wrline(self._get_c_call_sequence('f'))
         wrline('}')
         wrline('')
@@ -317,7 +318,7 @@
                                 " constant */" % (inspect.varname,
                                                   inspect.varname))
             else:
-                decl = self.totype.get_c_name("**p1")
+                decl = self.totype.get_c_name("(**p1)")
                 block.writeline("%s = &%s;  /* check that '%s' is of type '%s'"
                                 " */" % (decl, inspect.varname,
                                          inspect.varname, self.get_c_name()))
diff --git a/test/codegen/func-003.c b/test/codegen/func-003.c
--- a/test/codegen/func-003.c
+++ b/test/codegen/func-003.c
@@ -2,7 +2,7 @@
 
 # ____________________________________________________________
 
-static void testfunc_003__f4(void *func, void *args[], void *result) {
+static void testfunc_003__f1(void *func, void *args[], void *result) {
     func_t f = func;
     *(int *)result = f(*(long *)args[0], *(long *)args[1], *(int *)args[2]);
 }
@@ -20,7 +20,7 @@
         a3[0] = t2;
         a3[1] = t2;
         a3[2] = t1;
-        t4 = cb->get_function_type(cb, t1, a3, 3, &testfunc_003__f4);
+        t4 = cb->get_function_type(cb, t1, a3, 3, &testfunc_003__f1);
         t5 = cb->get_pointer_type(cb, t4);
         cb->define_type(cb, "func_t", t5);
 #expect TYPEDEF func_t = PTR FUNC( long -> long -> int -> int )
diff --git a/test/codegen/glob-002d.c b/test/codegen/glob-002d.c
--- a/test/codegen/glob-002d.c
+++ b/test/codegen/glob-002d.c
@@ -6,7 +6,7 @@
 {
     crx_type_t *t1, *t2;
     {
-        int **p1 = &someglob;  /* check that 'someglob' is of type 'int *' */
+        int (**p1) = &someglob;  /* check that 'someglob' is of type 'int *' */
         (void)p1;
         t1 = cb->get_signed_type(cb, sizeof(int), "int");
         t2 = cb->get_pointer_type(cb, t1);
diff --git a/test/codegen/glob-004.c b/test/codegen/glob-004.c
--- a/test/codegen/glob-004.c
+++ b/test/codegen/glob-004.c
@@ -2,20 +2,24 @@
 
 # ____________________________________________________________
 
-static void __creflect_t1(void *func, void *args[], void *result) {
-    int(*f)(long, long) = func;
+static void testglob_004__f1(void *func, void *args[], void *result) {
+    int (*f)(long, long) = func;
     *(int *)result = f(*(long *)args[0], *(long *)args[1]);
 }
 
-int testglob_004(char *r)
+void testglob_004(crx_builder_t *cb)
 {
-    if (!r)
-        return 100;
+    crx_type_t *t1, *t2, *a3[2], *t4, *t5;
     {
-        int (**p1)(long, long) = &someglob;  /* check the exact type of 'someglob' */
+        int (**p1)(long, long) = &someglob;  /* check that 'someglob' is of type 'int (*)(long, long)' */
         (void)p1;
-        r += sprintf(r, "/*%p*/", &someglob);
-        r += sprintf(r, "int (*someglob)(long, long)/*%p*/;\n", &__creflect_t1);
+        t1 = cb->get_signed_type(cb, sizeof(int), "int");
+        t2 = cb->get_signed_type(cb, sizeof(long), "long");
+        a3[0] = t2;
+        a3[1] = t2;
+        t4 = cb->get_function_type(cb, t1, a3, 2, &testglob_004__f1);
+        t5 = cb->get_pointer_type(cb, t4);
+        cb->define_var(cb, "someglob", t5, &someglob);
+#expect VAR someglob: PTR FUNC( long -> long -> int )
     }
-    return 0;
 }

From noreply at buildbot.pypy.org  Tue Nov 18 18:08:57 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Tue, 18 Nov 2014 18:08:57 +0100 (CET)
Subject: [pypy-commit] creflect default: pass the remaining func tests
Message-ID: <20141118170857.3C6F61C1147@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r77:2120ff969e43
Date: 2014-11-18 18:09 +0100
http://bitbucket.org/cffi/creflect/changeset/2120ff969e43/

Log:	pass the remaining func tests

diff --git a/creflect/model.py b/creflect/model.py
--- a/creflect/model.py
+++ b/creflect/model.py
@@ -204,11 +204,11 @@
     def inspect_type(self, block, inspect):
         # this class overrides inspect_type() instead of
         # inspect_nonconst_type(), to avoid the extra call to get_const_type()
+        inspect.flush()
+
         if self.ellipsis:
             return self.inspect_type_ellipsis(block, inspect)
 
-        inspect.flush()
-
         if (isinstance(inspect, TypeInspector) and inspect.started and
                 inspect.levels == ['*']):    # common case
             decl = '%s f' % inspect.typename
@@ -231,14 +231,20 @@
         return block.write_crx_type_var(expr)
 
     def inspect_type_ellipsis(self, block, inspect):
-        if not isinstance(inspect, TypeInspector):
-            XXX
-        else:
-            star_p1 = inspect.fetch_star_p1()
-            xxx
+        if isinstance(inspect, VarInspector):
+            decl = self.get_c_name("(*p1)")
+            block.writeline("%s = &%s;  /* check that '%s' is a function with"
+                            " exactly the given signature */" % (
+                                decl, inspect.varname, inspect.varname))
+            block.writeline("(void)p1;")
+        t1, a2 = self._get_arg_ret_types(block)
+        expr = 'cb->get_ellipsis_function_type(cb, %s, %s, %d)' % (
+            t1, a2, len(self.args))
         return block.write_crx_type_var(expr)
 
     def shadow_global_var(self, top_level_block, fnname):
+        if self.ellipsis:
+            return fnname
         shadowname = '%s__d_%s' % (top_level_block.crx_func_name, fnname)
         wrline = top_level_block.writeline
         args = [arg.get_c_name('a%d' % i) for i, arg in enumerate(self.args)]
@@ -318,7 +324,7 @@
                                 " constant */" % (inspect.varname,
                                                   inspect.varname))
             else:
-                decl = self.totype.get_c_name("(**p1)")
+                decl = self.get_c_name("*p1")
                 block.writeline("%s = &%s;  /* check that '%s' is of type '%s'"
                                 " */" % (decl, inspect.varname,
                                          inspect.varname, self.get_c_name()))
diff --git a/test/codegen/func-002.c b/test/codegen/func-002.c
--- a/test/codegen/func-002.c
+++ b/test/codegen/func-002.c
@@ -11,11 +11,12 @@
 {
     crx_type_t *t1, *a2[1], *t3;
     {
-        int (*p1)(int, ...) = f;  /* check that 'f' is a function with exactly the given signature */
+        int (*p1)(int, ...) = &f;  /* check that 'f' is a function with exactly the given signature */
+        (void)p1;
         t1 = cb->get_signed_type(cb, sizeof(int), "int");
         a2[0] = t1;
         t3 = cb->get_ellipsis_function_type(cb, t1, a2, 1);
-        cb->define_var(cb, "f", t3, p1);
+        cb->define_var(cb, "f", t3, &f);
 #expect VAR f: FUNC( int -> ... -> int )
     }
 }
diff --git a/test/codegen/glob-002d.c b/test/codegen/glob-002d.c
--- a/test/codegen/glob-002d.c
+++ b/test/codegen/glob-002d.c
@@ -6,7 +6,7 @@
 {
     crx_type_t *t1, *t2;
     {
-        int (**p1) = &someglob;  /* check that 'someglob' is of type 'int *' */
+        int **p1 = &someglob;  /* check that 'someglob' is of type 'int *' */
         (void)p1;
         t1 = cb->get_signed_type(cb, sizeof(int), "int");
         t2 = cb->get_pointer_type(cb, t1);

From noreply at buildbot.pypy.org  Tue Nov 18 18:12:29 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Tue, 18 Nov 2014 18:12:29 +0100 (CET)
Subject: [pypy-commit] creflect default: add a passing test
Message-ID: <20141118171229.6F4BD1C1148@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r78:7282685432ce
Date: 2014-11-18 18:12 +0100
http://bitbucket.org/cffi/creflect/changeset/7282685432ce/

Log:	add a passing test

diff --git a/test/codegen/func-003b.c b/test/codegen/func-003b.c
new file mode 100644
--- /dev/null
+++ b/test/codegen/func-003b.c
@@ -0,0 +1,21 @@
+typedef int(*func_t)(long, ...);
+
+# ____________________________________________________________
+
+void testfunc_003b(crx_builder_t *cb)
+{
+    crx_type_t *t1, *t2, *a3[1], *t4, *t5;
+    {
+        func_t *p1;
+        char *p2;
+        p1 = (void *)&p2;
+        *p1 = (void *)0;    /* check that 'func_t' is a pointer type */
+        t1 = cb->get_signed_type(cb, sizeof(int), "int");
+        t2 = cb->get_signed_type(cb, sizeof(long), "long");
+        a3[0] = t2;
+        t4 = cb->get_ellipsis_function_type(cb, t1, a3, 1);
+        t5 = cb->get_pointer_type(cb, t4);
+        cb->define_type(cb, "func_t", t5);
+#expect TYPEDEF func_t = PTR FUNC( long -> ... -> int )
+    }
+}

From noreply at buildbot.pypy.org  Tue Nov 18 18:32:09 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Tue, 18 Nov 2014 18:32:09 +0100 (CET)
Subject: [pypy-commit] creflect default: typedefs of function pointers:
 check more precisely the type if needed
Message-ID: <20141118173209.943AA1D2889@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r79:b06d3f4392e6
Date: 2014-11-18 18:32 +0100
http://bitbucket.org/cffi/creflect/changeset/b06d3f4392e6/

Log:	typedefs of function pointers: check more precisely the type if
	needed

diff --git a/creflect/model.py b/creflect/model.py
--- a/creflect/model.py
+++ b/creflect/model.py
@@ -204,13 +204,22 @@
     def inspect_type(self, block, inspect):
         # this class overrides inspect_type() instead of
         # inspect_nonconst_type(), to avoid the extra call to get_const_type()
-        inspect.flush()
+
+        # this variable is set if we support slightly misdeclared function
+        # args or return value; this is the case only for typedefs that
+        # declare a pointer to a function without ellipsis.
+        common_case = (not self.ellipsis and isinstance(inspect, TypeInspector)
+                       and inspect.started and inspect.levels == ['*'])
+        if common_case:
+            inspect.flush()
+        else:
+            inspect.flush('(%s)' % self.get_c_name('(*)'),
+                "a function pointer type with exactly the given signature")
 
         if self.ellipsis:
             return self.inspect_type_ellipsis(block, inspect)
 
-        if (isinstance(inspect, TypeInspector) and inspect.started and
-                inspect.levels == ['*']):    # common case
+        if common_case:
             decl = '%s f' % inspect.typename
         else:
             decl = self.get_c_name('(*f)')
@@ -409,7 +418,7 @@
 
 class MissingInspector(object):
 
-    def flush(self):
+    def flush(self, cast=None, comment=None):
         pass
 
 
@@ -418,7 +427,7 @@
         self.block = block
         self.varname = varname
 
-    def flush(self):
+    def flush(self, cast=None, comment=None):
         pass
 
 
@@ -445,9 +454,9 @@
             self.at_end = []
             self.assign_target = None
 
-    def flush(self):
+    def flush(self, cast=None, comment=None):
         if self.started:
-            self.assign_to_p1('0')
+            self.assign_to_p1('0', cast, comment)
 
     def get_comment_type(self, levels_delta, ignore_array, minlevel=0):
         end = len(self.levels)
@@ -488,23 +497,26 @@
             star_p1 = '%sp1->%s' % ('*' * len(self.levels), self.structfield)
         return star_p1
 
-    def assign_to_p1(self, expr):
+    def assign_to_p1(self, expr, cast=None, comment=None):
+        cast = cast or '(void *)'
+        comment = comment or "a pointer type"
         if self.assign_target is None:
             length = len(self.levels)
             while length > 0 and self.levels[length - 1] == '[]':
                 length -= 1
-            comment = self.get_comment(-1, True, "a pointer type")
+            comment = self.get_comment(-1, True, comment)
             spaces = comment and (' ' * (3 - len(expr)))
             self.assign_target = '%sp1' % ('*' * length,)
         else:
             spaces = comment = ''
         if self.structfield is None:
-            line = '%s = (void *)%s;%s%s' % (self.assign_target, expr, spaces,
-                                             comment)
+            line = '%s = %s%s;%s%s' % (self.assign_target, cast, expr, spaces,
+                                       comment)
         else:
             if length > 0:
-                line = '%sp1->%s = (void *)%s;%s%s' % (
-                    '*' * (length - 1), self.structfield, expr, spaces, comment)
+                line = '%sp1->%s = %s%s;%s%s' % (
+                    '*' * (length - 1), self.structfield, cast, expr, spaces,
+                    comment)
             else:
                 line = 'p1 = (void *)(((char *)%s) - o);' % (expr,)
         self.assign_target = None
diff --git a/test/codegen/func-003b.c b/test/codegen/func-003b.c
--- a/test/codegen/func-003b.c
+++ b/test/codegen/func-003b.c
@@ -9,7 +9,7 @@
         func_t *p1;
         char *p2;
         p1 = (void *)&p2;
-        *p1 = (void *)0;    /* check that 'func_t' is a pointer type */
+        *p1 = (int (*)(long, ...))0;    /* check that 'func_t' is a function pointer type with exactly the given signature */
         t1 = cb->get_signed_type(cb, sizeof(int), "int");
         t2 = cb->get_signed_type(cb, sizeof(long), "long");
         a3[0] = t2;
diff --git a/test/codegen/func-004.c b/test/codegen/func-004.c
new file mode 100644
--- /dev/null
+++ b/test/codegen/func-004.c
@@ -0,0 +1,30 @@
+typedef int(**func_t)(long, int);
+
+# ____________________________________________________________
+
+static void testfunc_004__f1(void *func, void *args[], void *result) {
+    int (*f)(long, int) = func;
+    *(int *)result = f(*(long *)args[0], *(int *)args[1]);
+}
+
+void testfunc_004(crx_builder_t *cb)
+{
+    crx_type_t *t1, *t2, *a3[2], *t4, *t5, *t6;
+    {
+        func_t *p1;
+        char *p2;
+        char *p3;
+        p1 = (void *)&p2;
+        *p1 = (void *)&p3;  /* check that 'func_t' is a pointer type */
+        **p1 = (int (*)(long, int))0;    /* check that '*func_t' is a function pointer type with exactly the given signature */
+        t1 = cb->get_signed_type(cb, sizeof(int), "int");
+        t2 = cb->get_signed_type(cb, sizeof(long), "long");
+        a3[0] = t2;
+        a3[1] = t1;
+        t4 = cb->get_function_type(cb, t1, a3, 2, &testfunc_004__f1);
+        t5 = cb->get_pointer_type(cb, t4);
+        t6 = cb->get_pointer_type(cb, t5);
+        cb->define_type(cb, "func_t", t6);
+#expect TYPEDEF func_t = PTR PTR FUNC( long -> int -> int )
+    }
+}

From noreply at buildbot.pypy.org  Tue Nov 18 19:04:10 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Tue, 18 Nov 2014 19:04:10 +0100 (CET)
Subject: [pypy-commit] creflect default: pass simple structs again
Message-ID: <20141118180410.ADCBB1C104D@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r80:9a4131bc9166
Date: 2014-11-18 19:04 +0100
http://bitbucket.org/cffi/creflect/changeset/9a4131bc9166/

Log:	pass simple structs again

diff --git a/creflect/codegen.py b/creflect/codegen.py
--- a/creflect/codegen.py
+++ b/creflect/codegen.py
@@ -7,10 +7,12 @@
         if parent is None:
             self.crx_type_vars = []
             self.crx_type_cache = {}
+            self.crx_field_vars = []
             self.crx_top_level = self
         else:
             self.crx_type_vars = parent.crx_type_vars
             self.crx_type_cache = parent.crx_type_cache
+            self.crx_field_vars = parent.crx_field_vars
             self.crx_top_level = parent.crx_top_level
 
     def writedecl(self, line):
@@ -46,6 +48,11 @@
         else:
             return '0'
 
+    def write_crx_field_var(self, nfields):
+        d1 = 'd%d' % (len(self.crx_field_vars) + 1)
+        self.crx_field_vars.append('%s[%d]' % (d1, nfields))
+        return d1
+
 
 def transform_cdef(csource, crx_func_name):
     outerblock = CodeBlock(parent=None)
@@ -59,6 +66,8 @@
     if funcblock.crx_type_vars:
         funcblock.writedecl("crx_type_t %s;" % (
             ', '.join(funcblock.crx_type_vars),))
+    for d1 in funcblock.crx_field_vars:
+        funcblock.writedecl("crx_field_t %s;" % d1)
 
     outerblock.writeline('void %s(crx_builder_t *cb)' % (crx_func_name,))
     outerblock.write_subblock(funcblock)
diff --git a/creflect/creflect.h b/creflect/creflect.h
--- a/creflect/creflect.h
+++ b/creflect/creflect.h
@@ -2,17 +2,17 @@
 #include 
 
 
-typedef struct {
-    const char *name;
-    void *type;
-    size_t offset;
-    int numbits, bitshift;
-} crx_field_t;
-
 typedef struct crx_type_s crx_type_t;   /* opaque */
 typedef void (*crx_trampoline0_fn)(void *args[], void *res);
 typedef void (*crx_trampoline1_fn)(void *fn, void *args[], void *res);
 
+typedef struct {
+    const char *name;
+    crx_type_t *type;
+    size_t offset;
+    int numbits, bitshift;
+} crx_field_t;
+
 typedef union {
     int as_int;
     long as_long;
diff --git a/creflect/model.py b/creflect/model.py
--- a/creflect/model.py
+++ b/creflect/model.py
@@ -559,20 +559,19 @@
             insptp = '*%s' % ptrtp
             include_alignment = False
         #
-        extra1 = "(long long)sizeof(%s)" % (sizetp,)    # total size
+        extra1 = "sizeof(%s)" % (sizetp,)    # total size
         if include_alignment:
-            extra2 = ("(long long)(((char *)&((struct{char a; %s b;} *)0)->b)"
-                    " - (char *)0)" % (realtp,))          # alignment
-            funcblock.sprintf(tp + r" /*%lld,%lld*/{\n",
-                              extra="%s, %s" % (extra1, extra2),
-                              extralength=40)
+            extra2 = ("(((char *)&((struct{char a; %s b;} *)0)->b)"
+                      " - (char *)0)" % (realtp,))          # alignment
         else:
-            funcblock.sprintf(tp + r" /*%lld*/{\n",
-                              extra=extra1,
-                              extralength=20)
+            extra2 = '0'
+        d1 = funcblock.write_crx_field_var(len(self.fldnames))
+        t1 = funcblock.write_crx_type_var('cb->get_%s_type(cb, "%s")' % (
+            self.type.kind, self.type.name))
         #
-        for fldname, fldtype in zip(self.fldnames, self.fldtypes):
-            block = CodeBlock(funcblock.tr)
+        for i, (fldname, fldtype) in enumerate(
+                zip(self.fldnames, self.fldtypes)):
+            block = CodeBlock(funcblock)
             inspect = TypeInspector(block, insptp, fldname)
             inspect.start()
             # get the offset of the field
@@ -587,22 +586,27 @@
                 if arraylevels >= 1:
                     comment = " (%dx)" % arraylevels
                 comment = inspect.get_comment(0, False, "an array%s" % comment)
-                block.writedecl("long long o = offsetof(%s, %s%s);%s"
+                block.writedecl("size_t o = offsetof(%s, %s%s);%s"
                                 % (realtp, fldname, "[0]" * arraylevels,
                                    comment))
             else:
                 comment = inspect.get_comment(0, False, "not an array")
-                block.writedecl("long long o = ((char *)&((%s)0)->%s)"
+                block.writedecl("size_t o = ((char *)&((%s)0)->%s)"
                                 " - (char *)0;%s" % (ptrtp, fldname, comment))
             #
-            block.sprintf("  /*%lld*/", extra="o", extralength=20)
-            block.sprintf_add_right(r';\n')
-            fldtype.inspect_type(block, inspect)
+            t2 = fldtype.inspect_type(block, inspect)
             inspect.stop()
-            block.sprintf_add_left(fldname)
+            block.writeline('%s[%d].name = "%s";\n' % (d1, i, fldname))
+            block.writeline('%s[%d].type = %s;\n'   % (d1, i, t2))
+            block.writeline('%s[%d].offset = o;\n'  % (d1, i))
+            block.writeline('%s[%d].numbits = -1;\n' % (d1, i))
+            block.writeline('%s[%d].bitshift = -1;\n' % (d1, i))
             funcblock.write_subblock(block)
         #
-        funcblock.sprintf(r"};\n")
+        funcblock.writeline('cb->complete(cb, %s, sizeof(%s),' % (t1, realtp))
+        funcblock.writeline('             ((char *)&((struct{char a; %s b;} *)'
+                            '0)->b) - (char *)0,' % realtp)
+        funcblock.writeline('             %s, %d);' % (d1, len(self.fldnames)))
 
 
 class TypeDefDecl(object):
diff --git a/test/cgcompile.c b/test/cgcompile.c
--- a/test/cgcompile.c
+++ b/test/cgcompile.c
@@ -143,7 +143,11 @@
                          size_t sz, size_t align,
                          crx_field_t fields[], int nfields)
 {
-    abort();
+    int i;
+    printf("%s:\n", t->text);
+    for (i = 0; i < nfields; i++) {
+        printf("| %s: %s\n", fields[i].name, fields[i].type->text);
+    }
 }
 
 static void tst_complete_enum(crx_builder_t *cb, crx_type_t *t,
diff --git a/test/codegen/struct-001.c b/test/codegen/struct-001.c
--- a/test/codegen/struct-001.c
+++ b/test/codegen/struct-001.c
@@ -14,7 +14,6 @@
         struct foo_s *p1;
         size_t o = ((char *)&((struct foo_s *)0)->aa) - (char *)0;  /* check that 'struct foo_s::aa' is not an array */
         char b[sizeof(p1->aa)];
-        r += sprintf(r, "  /*%lld*/", o);
         p1 = (void *)(((char *)b) - o);
         (void)(p1->aa << 1);  /* check that 'struct foo_s::aa' is an integer type */
         p1->aa = -1;  /* check that 'struct foo_s::aa' is not declared 'const' */
@@ -27,20 +26,22 @@
     }
     {
         struct foo_s *p1;
-        long long o = ((char *)&((struct foo_s *)0)->bb) - (char *)0;  /* check that 'struct foo_s::bb' is not an array */
+        size_t o = ((char *)&((struct foo_s *)0)->bb) - (char *)0;  /* check that 'struct foo_s::bb' is not an array */
         char b[sizeof(p1->bb)];
-        r += sprintf(r, "  /*%lld*/", o);
         p1 = (void *)(((char *)b) - o);
         (void)(p1->bb << 1);  /* check that 'struct foo_s::bb' is an integer type */
         p1->bb = -1;  /* check that 'struct foo_s::bb' is not declared 'const' */
-        t3 = CRX_INT_TYPE(cb, p1->bb, "int");
-        d1[0].name = "bb";
-        d1[0].type = t3;
-        d1[0].offset = o;
-        d1[0].numbits = -1;
-        d1[0].bitshift = -1;
+        t3 = CRX_INT_TYPE(cb, p1->bb, "unsigned int");
+        d1[1].name = "bb";
+        d1[1].type = t3;
+        d1[1].offset = o;
+        d1[1].numbits = -1;
+        d1[1].bitshift = -1;
     }
     cb->complete(cb, t1, sizeof(struct foo_s),
-                 offsetof(struct{char a; struct foo_s b;}, b),
+                 ((char *)&((struct{char a; struct foo_s b;} *)0)->b) - (char *)0,
                  d1, 2);
+#expect STRUCT foo_s:
+#expect | aa: int
+#expect | bb: unsigned int
 }

From noreply at buildbot.pypy.org  Tue Nov 18 19:06:52 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Tue, 18 Nov 2014 19:06:52 +0100 (CET)
Subject: [pypy-commit] creflect default: next test
Message-ID: <20141118180652.04BE21C104D@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r81:08ca9b65dbb0
Date: 2014-11-18 19:07 +0100
http://bitbucket.org/cffi/creflect/changeset/08ca9b65dbb0/

Log:	next test

diff --git a/test/codegen/struct-001b.c b/test/codegen/struct-001b.c
--- a/test/codegen/struct-001b.c
+++ b/test/codegen/struct-001b.c
@@ -1,56 +1,32 @@
-struct foo_s /*4,4*/{
-  /*0*/const int aa;
+struct foo_s {
+    const int aa;
 };
 
 # ____________________________________________________________
-#include 
 
-int teststruct_001b(char *r)
+void teststruct_001b(crx_builder_t *cb)
 {
-    if (!r)
-        return 100;
-    r += sprintf(r, "struct foo_s /*%lld,%lld*/{\n", (long long)sizeof(struct foo_s), (long long)(((char *)&((struct{char a; struct foo_s b;} *)0)->b) - (char *)0));
+    crx_type_t *t1, *t2, *t3;
+    crx_field_t d1[1];
+    t1 = cb->get_struct_type(cb, "foo_s");
     {
         struct foo_s *p1;
-        long long o = ((char *)&((struct foo_s *)0)->aa) - (char *)0;  /* check that 'struct foo_s::aa' is not an array */
+        size_t o = ((char *)&((struct foo_s *)0)->aa) - (char *)0;  /* check that 'struct foo_s::aa' is not an array */
         char b[sizeof(p1->aa)];
         memset(b, -1, sizeof(b));
-        r += sprintf(r, "  /*%lld*/", o);
         p1 = (void *)(((char *)b) - o);
         (void)(p1->aa << 1);  /* check that 'struct foo_s::aa' is an integer type */
-        r += sprintf(r, "const ");
-        if (p1->aa > 0) {
-            if (sizeof(p1->aa) == 1 && p1->aa == 1)
-                r += sprintf(r, "_Bool");
-            else if (sizeof(p1->aa) == sizeof(unsigned int))
-                r += sprintf(r, "unsigned int");
-            else if (sizeof(p1->aa) == sizeof(unsigned short))
-                r += sprintf(r, "unsigned short");
-            else if (sizeof(p1->aa) == sizeof(unsigned char))
-                r += sprintf(r, "unsigned char");
-            else if (sizeof(p1->aa) == sizeof(unsigned long))
-                r += sprintf(r, "unsigned long");
-            else if (sizeof(p1->aa) == sizeof(unsigned long long))
-                r += sprintf(r, "unsigned long long");
-            else
-                r += sprintf(r, "uint%u_t", (int)sizeof(p1->aa) * 8);
-        }
-        else {
-            if (sizeof(p1->aa) == sizeof(int))
-                r += sprintf(r, "int");
-            else if (sizeof(p1->aa) == sizeof(short))
-                r += sprintf(r, "short");
-            else if (sizeof(p1->aa) == sizeof(signed char))
-                r += sprintf(r, "signed char");
-            else if (sizeof(p1->aa) == sizeof(long))
-                r += sprintf(r, "long");
-            else if (sizeof(p1->aa) == sizeof(long long))
-                r += sprintf(r, "long long");
-            else
-                r += sprintf(r, "int%u_t", (int)sizeof(p1->aa) * 8);
-        }
-        r += sprintf(r, " aa;\n");
+        t2 = CRX_INT_TYPE(cb, p1->aa, "int");
+        t3 = cb->get_const_type(cb, t2);
+        d1[0].name = "aa";
+        d1[0].type = t3;
+        d1[0].offset = o;
+        d1[0].numbits = -1;
+        d1[0].bitshift = -1;
     }
-    r += sprintf(r, "};\n");
-    return 0;
+    cb->complete(cb, t1, sizeof(struct foo_s),
+                 ((char *)&((struct{char a; struct foo_s b;} *)0)->b) - (char *)0,
+                 d1, 1);
+#expect STRUCT foo_s:
+#expect | aa: CONST int
 }

From noreply at buildbot.pypy.org  Tue Nov 18 19:40:28 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Tue, 18 Nov 2014 19:40:28 +0100 (CET)
Subject: [pypy-commit] creflect default: next test
Message-ID: <20141118184028.2DCAC1C104D@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r82:a20d5500a35a
Date: 2014-11-18 19:10 +0100
http://bitbucket.org/cffi/creflect/changeset/a20d5500a35a/

Log:	next test

diff --git a/test/codegen/struct-002.c b/test/codegen/struct-002.c
--- a/test/codegen/struct-002.c
+++ b/test/codegen/struct-002.c
@@ -4,20 +4,28 @@
 
 # ____________________________________________________________
 
-int teststruct_002(char *r)
+void teststruct_002(crx_builder_t *cb)
 {
-    if (!r)
-        return 100;
-    r += sprintf(r, "struct foo_s /*%lld,%lld*/{\n", (long long)sizeof(struct foo_s), (long long)(((char *)&((struct{char a; struct foo_s b;} *)0)->b) - (char *)0));
+    crx_type_t *t1, *t2, *t3;
+    crx_field_t d1[1];
+    t1 = cb->get_struct_type(cb, "foo_s");
     {
         struct foo_s *p1;
-        long long o = ((char *)&((struct foo_s *)0)->aa) - (char *)0;  /* check that 'struct foo_s::aa' is not an array */
-        char b[sizeof(p1->aa)];
-        r += sprintf(r, "  /*%lld*/", o);
-        p1 = (void *)(((char *)b) - o);
+        size_t o = ((char *)&((struct foo_s *)0)->aa) - (char *)0;  /* check that 'struct foo_s::aa' is not an array */
+        char *p2;
+        p1 = (void *)(((char *)&p2) - o);
         p1->aa = (void *)0;    /* check that 'struct foo_s::aa' is a pointer type */
-        r += sprintf(r, "void *aa;\n");
+        t2 = cb->get_void_type(cb);
+        t3 = cb->get_pointer_type(cb, t2);
+        d1[0].name = "aa";
+        d1[0].type = t3;
+        d1[0].offset = o;
+        d1[0].numbits = -1;
+        d1[0].bitshift = -1;
     }
-    r += sprintf(r, "};\n");
-    return 0;
+    cb->complete(cb, t1, sizeof(struct foo_s),
+                 ((char *)&((struct{char a; struct foo_s b;} *)0)->b) - (char *)0,
+                 d1, 1);
+#expect STRUCT foo_s:
+#expect | aa: PTR void
 }

From noreply at buildbot.pypy.org  Tue Nov 18 19:40:29 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Tue, 18 Nov 2014 19:40:29 +0100 (CET)
Subject: [pypy-commit] creflect default: next test
Message-ID: <20141118184029.3D61F1C104D@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r83:201c2fbd2132
Date: 2014-11-18 19:17 +0100
http://bitbucket.org/cffi/creflect/changeset/201c2fbd2132/

Log:	next test

diff --git a/creflect/model.py b/creflect/model.py
--- a/creflect/model.py
+++ b/creflect/model.py
@@ -586,16 +586,16 @@
                 if arraylevels >= 1:
                     comment = " (%dx)" % arraylevels
                 comment = inspect.get_comment(0, False, "an array%s" % comment)
-                block.writedecl("size_t o = offsetof(%s, %s%s);%s"
-                                % (realtp, fldname, "[0]" * arraylevels,
-                                   comment))
+                o_decl = "size_t o = offsetof(%s, %s%s);%s" % (
+                    realtp, fldname, "[0]" * arraylevels, comment)
             else:
                 comment = inspect.get_comment(0, False, "not an array")
-                block.writedecl("size_t o = ((char *)&((%s)0)->%s)"
-                                " - (char *)0;%s" % (ptrtp, fldname, comment))
+                o_decl = ("size_t o = ((char *)&((%s)0)->%s)"
+                          " - (char *)0;%s" % (ptrtp, fldname, comment))
             #
             t2 = fldtype.inspect_type(block, inspect)
             inspect.stop()
+            block.writedecl(o_decl)
             block.writeline('%s[%d].name = "%s";\n' % (d1, i, fldname))
             block.writeline('%s[%d].type = %s;\n'   % (d1, i, t2))
             block.writeline('%s[%d].offset = o;\n'  % (d1, i))
diff --git a/test/codegen/struct-001.c b/test/codegen/struct-001.c
--- a/test/codegen/struct-001.c
+++ b/test/codegen/struct-001.c
@@ -12,8 +12,8 @@
     t1 = cb->get_struct_type(cb, "foo_s");
     {
         struct foo_s *p1;
+        char b[sizeof(p1->aa)];
         size_t o = ((char *)&((struct foo_s *)0)->aa) - (char *)0;  /* check that 'struct foo_s::aa' is not an array */
-        char b[sizeof(p1->aa)];
         p1 = (void *)(((char *)b) - o);
         (void)(p1->aa << 1);  /* check that 'struct foo_s::aa' is an integer type */
         p1->aa = -1;  /* check that 'struct foo_s::aa' is not declared 'const' */
@@ -26,8 +26,8 @@
     }
     {
         struct foo_s *p1;
+        char b[sizeof(p1->bb)];
         size_t o = ((char *)&((struct foo_s *)0)->bb) - (char *)0;  /* check that 'struct foo_s::bb' is not an array */
-        char b[sizeof(p1->bb)];
         p1 = (void *)(((char *)b) - o);
         (void)(p1->bb << 1);  /* check that 'struct foo_s::bb' is an integer type */
         p1->bb = -1;  /* check that 'struct foo_s::bb' is not declared 'const' */
diff --git a/test/codegen/struct-001b.c b/test/codegen/struct-001b.c
--- a/test/codegen/struct-001b.c
+++ b/test/codegen/struct-001b.c
@@ -11,8 +11,8 @@
     t1 = cb->get_struct_type(cb, "foo_s");
     {
         struct foo_s *p1;
+        char b[sizeof(p1->aa)];
         size_t o = ((char *)&((struct foo_s *)0)->aa) - (char *)0;  /* check that 'struct foo_s::aa' is not an array */
-        char b[sizeof(p1->aa)];
         memset(b, -1, sizeof(b));
         p1 = (void *)(((char *)b) - o);
         (void)(p1->aa << 1);  /* check that 'struct foo_s::aa' is an integer type */
diff --git a/test/codegen/struct-002.c b/test/codegen/struct-002.c
--- a/test/codegen/struct-002.c
+++ b/test/codegen/struct-002.c
@@ -11,8 +11,8 @@
     t1 = cb->get_struct_type(cb, "foo_s");
     {
         struct foo_s *p1;
+        char *p2;
         size_t o = ((char *)&((struct foo_s *)0)->aa) - (char *)0;  /* check that 'struct foo_s::aa' is not an array */
-        char *p2;
         p1 = (void *)(((char *)&p2) - o);
         p1->aa = (void *)0;    /* check that 'struct foo_s::aa' is a pointer type */
         t2 = cb->get_void_type(cb);
diff --git a/test/codegen/struct-003.c b/test/codegen/struct-003.c
--- a/test/codegen/struct-003.c
+++ b/test/codegen/struct-003.c
@@ -4,53 +4,31 @@
 
 # ____________________________________________________________
 
-int teststruct_003(char *r)
+void teststruct_003(crx_builder_t *cb)
 {
-    if (!r)
-        return 100;
-    r += sprintf(r, "struct foo_s /*%lld,%lld*/{\n", (long long)sizeof(struct foo_s), (long long)(((char *)&((struct{char a; struct foo_s b;} *)0)->b) - (char *)0));
+    crx_type_t *t1, *t2, *t3;
+    crx_field_t d1[1];
+    t1 = cb->get_struct_type(cb, "foo_s");
     {
         struct foo_s *p1;
         char *p2;
-        long long o = ((char *)&((struct foo_s *)0)->aa) - (char *)0;  /* check that 'struct foo_s::aa' is not an array */
         char b[sizeof(*p1->aa)];  /* check that '*struct foo_s::aa' is a valid type */
-        r += sprintf(r, "  /*%lld*/", o);
+        size_t o = ((char *)&((struct foo_s *)0)->aa) - (char *)0;  /* check that 'struct foo_s::aa' is not an array */
         p1 = (void *)(((char *)&p2) - o);
         p1->aa = (void *)b;    /* check that 'struct foo_s::aa' is a pointer type */
         (void)(*p1->aa << 1);  /* check that '*struct foo_s::aa' is an integer type */
         *p1->aa = -1;  /* check that '*struct foo_s::aa' is not declared 'const' */
-        if (*p1->aa > 0) {
-            if (sizeof(*p1->aa) == 1 && *p1->aa == 1)
-                r += sprintf(r, "_Bool");
-            else if (sizeof(*p1->aa) == sizeof(unsigned int))
-                r += sprintf(r, "unsigned int");
-            else if (sizeof(*p1->aa) == sizeof(unsigned short))
-                r += sprintf(r, "unsigned short");
-            else if (sizeof(*p1->aa) == sizeof(unsigned char))
-                r += sprintf(r, "unsigned char");
-            else if (sizeof(*p1->aa) == sizeof(unsigned long))
-                r += sprintf(r, "unsigned long");
-            else if (sizeof(*p1->aa) == sizeof(unsigned long long))
-                r += sprintf(r, "unsigned long long");
-            else
-                r += sprintf(r, "uint%u_t", (int)sizeof(*p1->aa) * 8);
-        }
-        else {
-            if (sizeof(*p1->aa) == sizeof(int))
-                r += sprintf(r, "int");
-            else if (sizeof(*p1->aa) == sizeof(short))
-                r += sprintf(r, "short");
-            else if (sizeof(*p1->aa) == sizeof(signed char))
-                r += sprintf(r, "signed char");
-            else if (sizeof(*p1->aa) == sizeof(long))
-                r += sprintf(r, "long");
-            else if (sizeof(*p1->aa) == sizeof(long long))
-                r += sprintf(r, "long long");
-            else
-                r += sprintf(r, "int%u_t", (int)sizeof(*p1->aa) * 8);
-        }
-        r += sprintf(r, " *aa;\n");
+        t2 = CRX_INT_TYPE(cb, *p1->aa, "int");
+        t3 = cb->get_pointer_type(cb, t2);
+        d1[0].name = "aa";
+        d1[0].type = t3;
+        d1[0].offset = o;
+        d1[0].numbits = -1;
+        d1[0].bitshift = -1;
     }
-    r += sprintf(r, "};\n");
-    return 0;
+    cb->complete(cb, t1, sizeof(struct foo_s),
+                 ((char *)&((struct{char a; struct foo_s b;} *)0)->b) - (char *)0,
+                 d1, 1);
+#expect STRUCT foo_s:
+#expect | aa: PTR int
 }

From noreply at buildbot.pypy.org  Tue Nov 18 19:40:30 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Tue, 18 Nov 2014 19:40:30 +0100 (CET)
Subject: [pypy-commit] creflect default: next test
Message-ID: <20141118184030.505511C104D@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r84:5ffaa9b7d90c
Date: 2014-11-18 19:25 +0100
http://bitbucket.org/cffi/creflect/changeset/5ffaa9b7d90c/

Log:	next test

diff --git a/creflect/model.py b/creflect/model.py
--- a/creflect/model.py
+++ b/creflect/model.py
@@ -581,9 +581,8 @@
                 arraylevels += 1
                 checktype = checktype.item
             if arraylevels:
-                block.add_include('stddef.h')
                 comment = ""
-                if arraylevels >= 1:
+                if arraylevels > 1:
                     comment = " (%dx)" % arraylevels
                 comment = inspect.get_comment(0, False, "an array%s" % comment)
                 o_decl = "size_t o = offsetof(%s, %s%s);%s" % (
diff --git a/test/codegen/struct-004.c b/test/codegen/struct-004.c
--- a/test/codegen/struct-004.c
+++ b/test/codegen/struct-004.c
@@ -3,33 +3,34 @@
 };
 
 # ____________________________________________________________
-#include 
 
-static void __creflect1(void r(char *, void**, int))
+void teststruct_004(crx_builder_t *cb)
 {
-    void *a[4];
-    __CREFLECT_PREV(r);
-    a[0] = (void *)sizeof(struct foo_s);  /* size */
-    a[1] = &((struct{char a; struct foo_s b;}*)0)->b;  /* align */
-    r("struct foo_s", a, 2);
+    crx_type_t *t1, *t2, *t3;
+    crx_field_t d1[1];
+    t1 = cb->get_struct_type(cb, "foo_s");
     {
         struct foo_s *p1;
-        void *o = (void *)offsetof(struct foo_s, aa[0]);  /* offset */
         char b[sizeof(*p1->aa)];  /* check that 'struct foo_s::aa[]' is a valid type */
-        a[1] = o;
-        p1 = (void *)(((char *)b) - (long)o);
+        size_t o = offsetof(struct foo_s, aa[0]);  /* check that 'struct foo_s::aa' is an array */
+        p1 = (void *)(((char *)b) - o);
         if ((void *)&p1->aa != (void *)p1->aa) {
-            r("E:type 'struct foo_s::aa' is not an array, but a pointer type", 0, 0);
-            goto f1;
+            cb->error(cb, "type 'struct foo_s::aa' is not an array, but a pointer type");
+            return;
         }
         (void)(*p1->aa << 1);  /* check that 'struct foo_s::aa[]' is an integer type */
-        *p1->aa = -1;
-        a[2] = (void *)(*p1->aa > 0 ? sizeof(*p1->aa) : -sizeof(*p1->aa));
-        a[3] = (void *)(sizeof(p1->aa) / sizeof(*p1->aa));
+        *p1->aa = -1;  /* check that 'struct foo_s::aa[]' is not declared 'const' */
+        t2 = CRX_INT_TYPE(cb, *p1->aa, "int");
+        t3 = cb->get_array_type(cb, t2, sizeof(p1->aa) / sizeof(*p1->aa));
+        d1[0].name = "aa";
+        d1[0].type = t3;
+        d1[0].offset = o;
+        d1[0].numbits = -1;
+        d1[0].bitshift = -1;
     }
-    r("{aa:int?[?]", a, 4);
-  f1:;
+    cb->complete(cb, t1, sizeof(struct foo_s),
+                 ((char *)&((struct{char a; struct foo_s b;} *)0)->b) - (char *)0,
+                 d1, 1);
+#expect STRUCT foo_s:
+#expect | aa: ARRAY[27] int
 }
-
-#expect  struct foo_s  108  4
-#expect  {aa:int?[?]  99  0  -4  27

From noreply at buildbot.pypy.org  Tue Nov 18 19:40:31 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Tue, 18 Nov 2014 19:40:31 +0100 (CET)
Subject: [pypy-commit] creflect default: next test
Message-ID: <20141118184031.66ACE1C104D@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r85:6731bdd907f8
Date: 2014-11-18 19:35 +0100
http://bitbucket.org/cffi/creflect/changeset/6731bdd907f8/

Log:	next test

diff --git a/creflect/model.py b/creflect/model.py
--- a/creflect/model.py
+++ b/creflect/model.py
@@ -412,6 +412,14 @@
         if const:
             self.c_name_with_marker = 'const ' + self.c_name_with_marker
 
+    def get_type_var(self, block):
+        return block.write_crx_type_var('cb->get_%s_type(cb, "%s")' % (
+            self.kind, self.name))
+
+    def inspect_nonconst_type(self, block, inspect):
+        inspect.flush()
+        return self.get_type_var(block)
+
 
 # ____________________________________________________________
 
@@ -566,8 +574,7 @@
         else:
             extra2 = '0'
         d1 = funcblock.write_crx_field_var(len(self.fldnames))
-        t1 = funcblock.write_crx_type_var('cb->get_%s_type(cb, "%s")' % (
-            self.type.kind, self.type.name))
+        t1 = self.type.get_type_var(funcblock)
         #
         for i, (fldname, fldtype) in enumerate(
                 zip(self.fldnames, self.fldtypes)):
@@ -595,11 +602,11 @@
             t2 = fldtype.inspect_type(block, inspect)
             inspect.stop()
             block.writedecl(o_decl)
-            block.writeline('%s[%d].name = "%s";\n' % (d1, i, fldname))
-            block.writeline('%s[%d].type = %s;\n'   % (d1, i, t2))
-            block.writeline('%s[%d].offset = o;\n'  % (d1, i))
-            block.writeline('%s[%d].numbits = -1;\n' % (d1, i))
-            block.writeline('%s[%d].bitshift = -1;\n' % (d1, i))
+            block.writeline('%s[%d].name = "%s";' % (d1, i, fldname))
+            block.writeline('%s[%d].type = %s;'   % (d1, i, t2))
+            block.writeline('%s[%d].offset = o;'  % (d1, i))
+            block.writeline('%s[%d].numbits = -1;' % (d1, i))
+            block.writeline('%s[%d].bitshift = -1;' % (d1, i))
             funcblock.write_subblock(block)
         #
         funcblock.writeline('cb->complete(cb, %s, sizeof(%s),' % (t1, realtp))
diff --git a/test/codegen/struct-005.c b/test/codegen/struct-005.c
--- a/test/codegen/struct-005.c
+++ b/test/codegen/struct-005.c
@@ -2,59 +2,32 @@
 
 # ____________________________________________________________
 
-int teststruct_005(char *r)
+void teststruct_005(crx_builder_t *cb)
 {
-    if (!r)
-        return 70 + 30 + 18 + 6 + 4 + 30;
-    r += sprintf(r, "struct $foo_t /*%lld,%lld*/{\n", (long long)sizeof(foo_t), (long long)(((char *)&((struct{char a; foo_t b;} *)0)->b) - (char *)0));
+    crx_type_t *t1, *t2;
+    crx_field_t d1[1];
+    t1 = cb->get_struct_type(cb, "$foo_t");
     {
         foo_t *p1;
-        long long o = ((char *)&((foo_t *)0)->aa) - (char *)0;  /* check that 'foo_t::aa' is not an array */
         char b[sizeof(p1->aa)];
-        r += sprintf(r, "  /*%lld*/", o);
+        size_t o = ((char *)&((foo_t *)0)->aa) - (char *)0;  /* check that 'foo_t::aa' is not an array */
         p1 = (void *)(((char *)b) - o);
         (void)(p1->aa << 1);  /* check that 'foo_t::aa' is an integer type */
         p1->aa = -1;  /* check that 'foo_t::aa' is not declared 'const' */
-        if (p1->aa > 0) {
-            if (sizeof(p1->aa) == 1 && p1->aa == 1)
-                r += sprintf(r, "_Bool");
-            else if (sizeof(p1->aa) == sizeof(unsigned int))
-                r += sprintf(r, "unsigned int");
-            else if (sizeof(p1->aa) == sizeof(unsigned short))
-                r += sprintf(r, "unsigned short");
-            else if (sizeof(p1->aa) == sizeof(unsigned char))
-                r += sprintf(r, "unsigned char");
-            else if (sizeof(p1->aa) == sizeof(unsigned long))
-                r += sprintf(r, "unsigned long");
-            else if (sizeof(p1->aa) == sizeof(unsigned long long))
-                r += sprintf(r, "unsigned long long");
-            else
-                r += sprintf(r, "uint%u_t", (int)sizeof(p1->aa) * 8);
-        }
-        else {
-            if (sizeof(p1->aa) == sizeof(int))
-                r += sprintf(r, "int");
-            else if (sizeof(p1->aa) == sizeof(short))
-                r += sprintf(r, "short");
-            else if (sizeof(p1->aa) == sizeof(signed char))
-                r += sprintf(r, "signed char");
-            else if (sizeof(p1->aa) == sizeof(long))
-                r += sprintf(r, "long");
-            else if (sizeof(p1->aa) == sizeof(long long))
-                r += sprintf(r, "long long");
-            else
-                r += sprintf(r, "int%u_t", (int)sizeof(p1->aa) * 8);
-        }
-        r += sprintf(r, " aa;\n");
+        t2 = CRX_INT_TYPE(cb, p1->aa, "int");
+        d1[0].name = "aa";
+        d1[0].type = t2;
+        d1[0].offset = o;
+        d1[0].numbits = -1;
+        d1[0].bitshift = -1;
     }
-    r += sprintf(r, "};\n");
+    cb->complete(cb, t1, sizeof(foo_t),
+                 ((char *)&((struct{char a; foo_t b;} *)0)->b) - (char *)0,
+                 d1, 1);
+#expect STRUCT $foo_t:
+#expect | aa: int
     {
-        r += sprintf(r, "typedef struct $foo_t foo_t;\n");
+        cb->define_type(cb, "foo_t", t1);
+#expect TYPEDEF foo_t = STRUCT $foo_t
     }
-    return 0;
 }
-
-#expect struct $foo_t {
-#expect   int aa;
-#expect };
-#expect typedef struct $foo_t foo_t;

From noreply at buildbot.pypy.org  Tue Nov 18 19:40:32 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Tue, 18 Nov 2014 19:40:32 +0100 (CET)
Subject: [pypy-commit] creflect default: final test!
Message-ID: <20141118184032.77C5A1C104D@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r86:f4143d8b619c
Date: 2014-11-18 19:40 +0100
http://bitbucket.org/cffi/creflect/changeset/f4143d8b619c/

Log:	final test!

diff --git a/creflect/model.py b/creflect/model.py
--- a/creflect/model.py
+++ b/creflect/model.py
@@ -562,17 +562,18 @@
             include_alignment = True
         else:
             ptrtp = realtp[1:]
+            del realtp     # don't use any more
             assert not ptrtp.startswith('$')
             sizetp = '*(%s)0' % ptrtp
             insptp = '*%s' % ptrtp
             include_alignment = False
         #
-        extra1 = "sizeof(%s)" % (sizetp,)    # total size
+        text_size = "sizeof(%s)" % (sizetp,)    # total size
         if include_alignment:
-            extra2 = ("(((char *)&((struct{char a; %s b;} *)0)->b)"
-                      " - (char *)0)" % (realtp,))          # alignment
+            text_offset = ("((char *)&((struct{char a; %s b;} *)0)->b)"
+                           " - (char *)0" % (realtp,))          # alignment
         else:
-            extra2 = '0'
+            text_offset = '0'
         d1 = funcblock.write_crx_field_var(len(self.fldnames))
         t1 = self.type.get_type_var(funcblock)
         #
@@ -609,9 +610,8 @@
             block.writeline('%s[%d].bitshift = -1;' % (d1, i))
             funcblock.write_subblock(block)
         #
-        funcblock.writeline('cb->complete(cb, %s, sizeof(%s),' % (t1, realtp))
-        funcblock.writeline('             ((char *)&((struct{char a; %s b;} *)'
-                            '0)->b) - (char *)0,' % realtp)
+        funcblock.writeline('cb->complete(cb, %s, %s,' % (t1, text_size))
+        funcblock.writeline('             %s,' % text_offset)
         funcblock.writeline('             %s, %d);' % (d1, len(self.fldnames)))
 
 
diff --git a/test/codegen/struct-005b.c b/test/codegen/struct-005b.c
--- a/test/codegen/struct-005b.c
+++ b/test/codegen/struct-005b.c
@@ -2,63 +2,37 @@
 
 # ____________________________________________________________
 
-int teststruct_005b(char *r)
+void teststruct_005b(crx_builder_t *cb)
 {
-    if (!r)
-        return 46 + 30 + 18 + 6 + 4 + 32;
-    r += sprintf(r, "struct $$foo_p /*%lld*/{\n", (long long)sizeof(*(foo_p)0));
+    crx_type_t *t1, *t2, *t3;
+    crx_field_t d1[1];
+    t1 = cb->get_struct_type(cb, "$$foo_p");
     {
         foo_p p1;
-        long long o = ((char *)&((foo_p)0)->aa) - (char *)0;  /* check that '*foo_p::aa' is not an array */
         char b[sizeof(p1->aa)];
-        r += sprintf(r, "  /*%lld*/", o);
+        size_t o = ((char *)&((foo_p)0)->aa) - (char *)0;  /* check that '*foo_p::aa' is not an array */
         p1 = (void *)(((char *)b) - o);
         (void)(p1->aa << 1);  /* check that '*foo_p::aa' is an integer type */
         p1->aa = -1;  /* check that '*foo_p::aa' is not declared 'const' */
-        if (p1->aa > 0) {
-            if (sizeof(p1->aa) == 1 && p1->aa == 1)
-                r += sprintf(r, "_Bool");
-            else if (sizeof(p1->aa) == sizeof(unsigned int))
-                r += sprintf(r, "unsigned int");
-            else if (sizeof(p1->aa) == sizeof(unsigned short))
-                r += sprintf(r, "unsigned short");
-            else if (sizeof(p1->aa) == sizeof(unsigned char))
-                r += sprintf(r, "unsigned char");
-            else if (sizeof(p1->aa) == sizeof(unsigned long))
-                r += sprintf(r, "unsigned long");
-            else if (sizeof(p1->aa) == sizeof(unsigned long long))
-                r += sprintf(r, "unsigned long long");
-            else
-                r += sprintf(r, "uint%u_t", (int)sizeof(p1->aa) * 8);
-        }
-        else {
-            if (sizeof(p1->aa) == sizeof(int))
-                r += sprintf(r, "int");
-            else if (sizeof(p1->aa) == sizeof(short))
-                r += sprintf(r, "short");
-            else if (sizeof(p1->aa) == sizeof(signed char))
-                r += sprintf(r, "signed char");
-            else if (sizeof(p1->aa) == sizeof(long))
-                r += sprintf(r, "long");
-            else if (sizeof(p1->aa) == sizeof(long long))
-                r += sprintf(r, "long long");
-            else
-                r += sprintf(r, "int%u_t", (int)sizeof(p1->aa) * 8);
-        }
-        r += sprintf(r, " aa;\n");
+        t2 = CRX_INT_TYPE(cb, p1->aa, "int");
+        d1[0].name = "aa";
+        d1[0].type = t2;
+        d1[0].offset = o;
+        d1[0].numbits = -1;
+        d1[0].bitshift = -1;
     }
-    r += sprintf(r, "};\n");
+    cb->complete(cb, t1, sizeof(*(foo_p)0),
+                 0,
+                 d1, 1);
+#expect STRUCT $$foo_p:
+#expect | aa: int
     {
         foo_p *p1;
         char *p2;
         p1 = (void *)&p2;
         *p1 = (void *)0;    /* check that 'foo_p' is a pointer type */
-        r += sprintf(r, "typedef struct $$foo_p *foo_p;\n");
+        t3 = cb->get_pointer_type(cb, t1);
+        cb->define_type(cb, "foo_p", t3);
+#expect TYPEDEF foo_p = PTR STRUCT $$foo_p
     }
-    return 0;
 }
-
-#expect struct $$foo_p {
-#expect   int aa;
-#expect };
-#expect typedef struct $$foo_p *foo_p;

From noreply at buildbot.pypy.org  Tue Nov 18 19:53:05 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Tue, 18 Nov 2014 19:53:05 +0100 (CET)
Subject: [pypy-commit] creflect default: check for char more precisely
Message-ID: <20141118185305.B2D8E1C104D@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r87:0338c76d71e5
Date: 2014-11-18 19:53 +0100
http://bitbucket.org/cffi/creflect/changeset/0338c76d71e5/

Log:	check for char more precisely

diff --git a/creflect/model.py b/creflect/model.py
--- a/creflect/model.py
+++ b/creflect/model.py
@@ -110,6 +110,12 @@
             if self.is_integer_type():
                 expr = 'CRX_INT_TYPE(cb, %s, "%s")' % (star_p1, self.name)
             elif self.is_char_type():
+                errmsg = "numeric type '%s' is not a char" % (
+                    inspect.get_comment_type(0, False),)
+                block.writeline("if (sizeof(%s) != 1) {" % (star_p1,))
+                block.writeline('    cb->error(cb, "%s");' % (errmsg,))
+                block.writeline("    return;")
+                block.writeline("}")
                 expr = 'cb->get_char_type(cb)'
             elif self.is_float_type():
                 xxx
diff --git a/test/codegen/003e.c b/test/codegen/003e.c
--- a/test/codegen/003e.c
+++ b/test/codegen/003e.c
@@ -11,6 +11,10 @@
         p1 = (void *)b;
         (void)(*p1 << 1);  /* check that 'num_t' is an integer type */
         *p1 = -1;  /* check that 'num_t' is not declared 'const' */
+        if (sizeof(*p1) != 1) {
+            cb->error(cb, "numeric type 'num_t' is not a char");
+            return;
+        }
         t1 = cb->get_char_type(cb);
         cb->define_type(cb, "num_t", t1);
 #expect TYPEDEF num_t = char

From noreply at buildbot.pypy.org  Tue Nov 18 20:16:52 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Tue, 18 Nov 2014 20:16:52 +0100 (CET)
Subject: [pypy-commit] creflect default: basic case for floats
Message-ID: <20141118191652.A044E1C1148@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r88:7056fee0a27a
Date: 2014-11-18 20:13 +0100
http://bitbucket.org/cffi/creflect/changeset/7056fee0a27a/

Log:	basic case for floats

diff --git a/creflect/creflect.h b/creflect/creflect.h
--- a/creflect/creflect.h
+++ b/creflect/creflect.h
@@ -29,11 +29,11 @@
     crx_type_t *(*get_char_type)(CRX_SELF);
     crx_type_t *(*get_bool_type)(CRX_SELF);
     crx_type_t *(*get_signed_type)(CRX_SELF, size_t,
-                                const char *);
+                                   const char *);
     crx_type_t *(*get_unsigned_type)(CRX_SELF, size_t,
+                                     const char *);
+    crx_type_t *(*get_float_type)(CRX_SELF, size_t,
                                   const char *);
-    crx_type_t *(*get_float_type)(CRX_SELF, size_t,
-                               const char *);
     crx_type_t *(*get_function_type)(CRX_SELF, crx_type_t *,
                                      crx_type_t *[], int, crx_trampoline1_fn);
     crx_type_t *(*get_ellipsis_function_type)(CRX_SELF, crx_type_t *,
diff --git a/creflect/model.py b/creflect/model.py
--- a/creflect/model.py
+++ b/creflect/model.py
@@ -98,13 +98,22 @@
         if isinstance(inspect, TypeInspector):
             star_p1 = inspect.fetch_star_p1()
             comment1 = inspect.get_comment(0, False, "a valid type", minlevel=1)
-            comment2 = inspect.get_comment(0, False, "an integer type")
+            if self.is_float_type():
+                comment2 = inspect.get_comment(0, False, "a numeric type")
+            else:
+                comment2 = inspect.get_comment(0, False, "an integer type")
             comment3 = inspect.get_comment(0, False, "not declared 'const'")
             block.writedecl("char b[sizeof(%s)];%s" % (star_p1, comment1))
-            if self.const:
+            if self.is_float_type():
+                block.writeline("memset(b, 0, sizeof(b));")
+            elif self.const:
                 block.writeline("memset(b, -1, sizeof(b));")
             inspect.assign_to_p1("b")
-            block.writeline("(void)(%s << 1);%s" % (star_p1, comment2))
+            if self.is_float_type():
+                op = '/ 2.0'
+            else:
+                op = '<< 1'
+            block.writeline("(void)(%s %s);%s" % (star_p1, op, comment2))
             if not self.const:
                 block.writeline("%s = -1;%s" % (star_p1, comment3))
             if self.is_integer_type():
@@ -118,7 +127,14 @@
                 block.writeline("}")
                 expr = 'cb->get_char_type(cb)'
             elif self.is_float_type():
-                xxx
+                errmsg = "numeric type '%s' is an integer, not a float" % (
+                    inspect.get_comment_type(0, False),)
+                block.writeline("if ((1 + %s * 0) / 2 == 0) {" % (star_p1,))
+                block.writeline('    cb->error(cb, "%s");' % (errmsg,))
+                block.writeline("    return;")
+                block.writeline("}")
+                expr = 'cb->get_float_type(cb, sizeof(%s), "%s")' % (
+                    star_p1, self.name)
             else:
                 raise AssertionError
         else:
diff --git a/test/codegen/003g.c b/test/codegen/003g.c
new file mode 100644
--- /dev/null
+++ b/test/codegen/003g.c
@@ -0,0 +1,23 @@
+typedef double num_t;
+
+# ____________________________________________________________
+
+void test003g(crx_builder_t *cb)
+{
+    crx_type_t *t1;
+    {
+        num_t *p1;
+        char b[sizeof(*p1)];
+        memset(b, 0, sizeof(b));
+        p1 = (void *)b;
+        (void)(*p1 / 2.0);  /* check that 'num_t' is a numeric type */
+        *p1 = -1;  /* check that 'num_t' is not declared 'const' */
+        if ((1 + *p1 * 0) / 2 == 0) {
+            cb->error(cb, "numeric type 'num_t' is an integer, not a float");
+            return;
+        }
+        t1 = cb->get_float_type(cb, sizeof(*p1), "double");
+        cb->define_type(cb, "num_t", t1);
+#expect TYPEDEF num_t = double
+    }
+}

From noreply at buildbot.pypy.org  Tue Nov 18 20:16:53 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Tue, 18 Nov 2014 20:16:53 +0100 (CET)
Subject: [pypy-commit] creflect default: floats, other case
Message-ID: <20141118191653.E3CE31C1148@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r89:2bf9370e2e5a
Date: 2014-11-18 20:17 +0100
http://bitbucket.org/cffi/creflect/changeset/2bf9370e2e5a/

Log:	floats, other case

diff --git a/creflect/model.py b/creflect/model.py
--- a/creflect/model.py
+++ b/creflect/model.py
@@ -153,7 +153,8 @@
             elif self.is_char_type():
                 expr = 'cb->get_char_type(cb)'
             elif self.is_float_type():
-                xxx
+                expr = 'cb->get_float_type(cb, sizeof(%s), "%s")' % (
+                    self.name, self.name)
             else:
                 raise AssertionError
         return block.write_crx_type_var(expr)

From noreply at buildbot.pypy.org  Tue Nov 18 20:33:08 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Tue, 18 Nov 2014 20:33:08 +0100 (CET)
Subject: [pypy-commit] creflect default: float consts
Message-ID: <20141118193308.2586F1C1148@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r90:4a2b32da62a3
Date: 2014-11-18 20:33 +0100
http://bitbucket.org/cffi/creflect/changeset/4a2b32da62a3/

Log:	float consts

diff --git a/creflect/creflect.h b/creflect/creflect.h
--- a/creflect/creflect.h
+++ b/creflect/creflect.h
@@ -21,7 +21,10 @@
     unsigned long as_unsigned_long;
     unsigned long long as_unsigned_long_long;
     char as_char;
-} crx_int_const_t;
+    float as_float;
+    double as_double;
+    long double as_long_double;
+} crx_num_const_t;
 
 #define CRX_SELF struct _crx_builder_s *
 typedef struct _crx_builder_s {
@@ -53,8 +56,8 @@
                        void *);
     void (*define_func)(CRX_SELF, const char *, crx_type_t *,
                         crx_type_t *[], int, crx_trampoline0_fn, void *);
-    void (*define_int_const)(CRX_SELF, const char *, crx_type_t *,
-                             crx_int_const_t *);
+    void (*define_num_const)(CRX_SELF, const char *, crx_type_t *,
+                             crx_num_const_t *);
     void (*error)(CRX_SELF, const char *);
 } crx_builder_t;
 #undef CRX_SELF
@@ -79,6 +82,9 @@
         (expr) == (signed char)(expr) || (expr) == (unsigned char)(expr), \
         (char)(expr))
 
+#define CRX_FLOAT_CONST(cb, expr, vp)                                   \
+    _creflect__float_const(cb, vp, sizeof(expr), (long double)(expr))
+
 __attribute__((unused))
 static crx_type_t *_creflect__int_type(crx_builder_t *cb, int expr_positive,
                                        size_t size_of_expr, int expr_equal_one,
@@ -93,7 +99,7 @@
 }
 
 __attribute__((unused))
-static crx_type_t *_creflect__int_const(crx_builder_t *cb, crx_int_const_t *vp,
+static crx_type_t *_creflect__int_const(crx_builder_t *cb, crx_num_const_t *vp,
                                         int preference_number,
                                         const char *toobig,
                                         int fits_int, int fits_long,
@@ -159,7 +165,7 @@
 }
 
 __attribute__((unused))
-static void _creflect__char_const(crx_builder_t *cb, crx_int_const_t *vp,
+static void _creflect__char_const(crx_builder_t *cb, crx_num_const_t *vp,
                                   const char *toobig,
                                   int fits_char, char value)
 {
@@ -168,3 +174,25 @@
     else
         vp->as_char = value;
 }
+
+__attribute__((unused))
+static crx_type_t *_creflect__float_const(crx_builder_t *cb,
+                                          crx_num_const_t *vp,
+                                          size_t size, long double value)
+{
+    const char *name;
+
+    if (size == sizeof(float)) {
+        vp->as_float = (float)value;
+        name = "float";
+    }
+    else if (size == sizeof(double)) {
+        vp->as_double = (double)value;
+        name = "double";
+    }
+    else {    /* unknown-sized floats are treated as "long double" */
+        vp->as_long_double = value;
+        name = "long double";
+    }
+    return cb->get_float_type(cb, size, name);
+}
diff --git a/creflect/model.py b/creflect/model.py
--- a/creflect/model.py
+++ b/creflect/model.py
@@ -159,13 +159,17 @@
                 raise AssertionError
         return block.write_crx_type_var(expr)
 
-    def write_int_const_decl(self, block, varname):
-        block.writedecl("crx_int_const_t v;")
+    def write_num_const_decl(self, block, varname):
+        block.writedecl("crx_num_const_t v;")
         comment = "an integer"
+        op = "<< 1"
         if self.is_char_type():
             comment += " or char"
-        block.writeline("(void)((%s) << 1);  /* check that '%s' is %s */" % (
-            varname, varname, comment))
+        if self.is_float_type():
+            comment = "a number"
+            op = "/ 2.0"
+        block.writeline("(void)((%s) %s);  /* check that '%s' is %s */" % (
+            varname, op, varname, comment))
         if self.is_integer_type():
             if self.name.endswith('long'):
                 if self.name.endswith('long long'):
@@ -178,10 +182,12 @@
         elif self.is_char_type():
             block.writeline('CRX_CHAR_CONST(cb, %s, &v);' % varname)
             expr = 'cb->get_char_type(cb)'
+        elif self.is_float_type():
+            expr = "CRX_FLOAT_CONST(cb, %s, &v)" % (varname,)
         else:
-            xxxxxxx
+            raise AssertionError
         t1 = block.write_crx_type_var(expr)
-        block.writeline('cb->define_int_const(cb, "%s", %s, &v);' % (
+        block.writeline('cb->define_num_const(cb, "%s", %s, &v);' % (
             varname, t1))
 
 
@@ -681,7 +687,7 @@
     def write_declaration(self, funcblock):
         if isinstance(self.type, PrimitiveType):
             block = CodeBlock(funcblock)
-            self.type.write_int_const_decl(block, self.name)
+            self.type.write_num_const_decl(block, self.name)
             funcblock.write_subblock(block)
         else:
             VarDecl.write_declaration(self, funcblock)
diff --git a/test/cgcompile.c b/test/cgcompile.c
--- a/test/cgcompile.c
+++ b/test/cgcompile.c
@@ -178,8 +178,8 @@
     printf("%s\n", ret->text);
 }
 
-static void tst_define_int_const(crx_builder_t *cb, const char *name,
-                                 crx_type_t *t, crx_int_const_t *value)
+static void tst_define_num_const(crx_builder_t *cb, const char *name,
+                                 crx_type_t *t, crx_num_const_t *value)
 {
     printf("INTCONST %s = %s ", name, t->text);
     if (strcmp(t->text, "int") == 0)
@@ -196,6 +196,12 @@
         printf("%llu\n", value->as_unsigned_long_long);
     else if (strcmp(t->text, "char") == 0)
         printf("'%c'\n", value->as_char);
+    else if (strcmp(t->text, "float") == 0)
+        printf("%f\n", (double)value->as_float);
+    else if (strcmp(t->text, "double") == 0)
+        printf("%f\n", value->as_double);
+    else if (strcmp(t->text, "long double") == 0)
+        printf("%f\n", (double)value->as_double);
     else {
         printf("???\n");
         abort();
@@ -228,7 +234,7 @@
     tst_define_type,
     tst_define_var,
     tst_define_func,
-    tst_define_int_const,
+    tst_define_num_const,
     tst_error,
 };
 
diff --git a/test/codegen/glob-003.c b/test/codegen/glob-003.c
--- a/test/codegen/glob-003.c
+++ b/test/codegen/glob-003.c
@@ -6,10 +6,10 @@
 {
     crx_type_t *t1;
     {
-        crx_int_const_t v;
+        crx_num_const_t v;
         (void)((ab) << 1);  /* check that 'ab' is an integer */
         t1 = CRX_INT_CONST(cb, ab, &v, 1);
-        cb->define_int_const(cb, "ab", t1, &v);
+        cb->define_num_const(cb, "ab", t1, &v);
 #expect INTCONST ab = int -42
     }
 }
diff --git a/test/codegen/glob-003b.c b/test/codegen/glob-003b.c
--- a/test/codegen/glob-003b.c
+++ b/test/codegen/glob-003b.c
@@ -10,10 +10,10 @@
 {
     crx_type_t *t1;
     {
-        crx_int_const_t v;
+        crx_num_const_t v;
         (void)((ab) << 1);  /* check that 'ab' is an integer */
         t1 = CRX_INT_CONST(cb, ab, &v, 1);
-        cb->define_int_const(cb, "ab", t1, &v);
+        cb->define_num_const(cb, "ab", t1, &v);
 #expect INTCONST ab = int 42
     }
 }
diff --git a/test/codegen/glob-003c.c b/test/codegen/glob-003c.c
--- a/test/codegen/glob-003c.c
+++ b/test/codegen/glob-003c.c
@@ -6,10 +6,10 @@
 {
     crx_type_t *t1;
     {
-        crx_int_const_t v;
+        crx_num_const_t v;
         (void)((ab) << 1);  /* check that 'ab' is an integer */
         t1 = CRX_INT_CONST(cb, ab, &v, 1);
-        cb->define_int_const(cb, "ab", t1, &v);
+        cb->define_num_const(cb, "ab", t1, &v);
 #expect INTCONST ab = unsigned int 42
     }
 }
diff --git a/test/codegen/glob-003d.c b/test/codegen/glob-003d.c
--- a/test/codegen/glob-003d.c
+++ b/test/codegen/glob-003d.c
@@ -6,11 +6,11 @@
 {
     crx_type_t *t1;
     {
-        crx_int_const_t v;
+        crx_num_const_t v;
         (void)((ab) << 1);  /* check that 'ab' is an integer or char */
         CRX_CHAR_CONST(cb, ab, &v);
         t1 = cb->get_char_type(cb);
-        cb->define_int_const(cb, "ab", t1, &v);
+        cb->define_num_const(cb, "ab", t1, &v);
 #expect INTCONST ab = char '*'
     }
 }
diff --git a/test/codegen/glob-003e.c b/test/codegen/glob-003e.c
--- a/test/codegen/glob-003e.c
+++ b/test/codegen/glob-003e.c
@@ -6,10 +6,10 @@
 {
     crx_type_t *t1;
     {
-        crx_int_const_t v;
+        crx_num_const_t v;
         (void)((ab) << 1);  /* check that 'ab' is an integer */
         t1 = CRX_INT_CONST(cb, ab, &v, 2);
-        cb->define_int_const(cb, "ab", t1, &v);
+        cb->define_num_const(cb, "ab", t1, &v);
 #expect INTCONST ab = long -42
     }
 }
diff --git a/test/codegen/glob-003f.c b/test/codegen/glob-003f.c
--- a/test/codegen/glob-003f.c
+++ b/test/codegen/glob-003f.c
@@ -6,10 +6,10 @@
 {
     crx_type_t *t1;
     {
-        crx_int_const_t v;
+        crx_num_const_t v;
         (void)((ab) << 1);  /* check that 'ab' is an integer */
         t1 = CRX_INT_CONST(cb, ab, &v, 3);
-        cb->define_int_const(cb, "ab", t1, &v);
+        cb->define_num_const(cb, "ab", t1, &v);
 #expect INTCONST ab = long long -42
     }
 }
diff --git a/test/codegen/glob-003g.c b/test/codegen/glob-003g.c
--- a/test/codegen/glob-003g.c
+++ b/test/codegen/glob-003g.c
@@ -6,10 +6,10 @@
 {
     crx_type_t *t1;
     {
-        crx_int_const_t v;
+        crx_num_const_t v;
         (void)((ab) << 1);  /* check that 'ab' is an integer */
         t1 = CRX_INT_CONST(cb, ab, &v, 1);
-        cb->define_int_const(cb, "ab", t1, &v);
+        cb->define_num_const(cb, "ab", t1, &v);
 #expect INTCONST ab = int -42
     }
 }

From noreply at buildbot.pypy.org  Tue Nov 18 20:35:13 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Tue, 18 Nov 2014 20:35:13 +0100 (CET)
Subject: [pypy-commit] creflect default: rename INTCONST to NUMCONST too
Message-ID: <20141118193513.E4FFA1C1148@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r91:431f4f324789
Date: 2014-11-18 20:35 +0100
http://bitbucket.org/cffi/creflect/changeset/431f4f324789/

Log:	rename INTCONST to NUMCONST too

diff --git a/test/cgcompile.c b/test/cgcompile.c
--- a/test/cgcompile.c
+++ b/test/cgcompile.c
@@ -181,7 +181,7 @@
 static void tst_define_num_const(crx_builder_t *cb, const char *name,
                                  crx_type_t *t, crx_num_const_t *value)
 {
-    printf("INTCONST %s = %s ", name, t->text);
+    printf("NUMCONST %s = %s ", name, t->text);
     if (strcmp(t->text, "int") == 0)
         printf("%d\n", value->as_int);
     else if (strcmp(t->text, "long") == 0)
diff --git a/test/codegen/glob-003.c b/test/codegen/glob-003.c
--- a/test/codegen/glob-003.c
+++ b/test/codegen/glob-003.c
@@ -10,6 +10,6 @@
         (void)((ab) << 1);  /* check that 'ab' is an integer */
         t1 = CRX_INT_CONST(cb, ab, &v, 1);
         cb->define_num_const(cb, "ab", t1, &v);
-#expect INTCONST ab = int -42
+#expect NUMCONST ab = int -42
     }
 }
diff --git a/test/codegen/glob-003b.c b/test/codegen/glob-003b.c
--- a/test/codegen/glob-003b.c
+++ b/test/codegen/glob-003b.c
@@ -14,6 +14,6 @@
         (void)((ab) << 1);  /* check that 'ab' is an integer */
         t1 = CRX_INT_CONST(cb, ab, &v, 1);
         cb->define_num_const(cb, "ab", t1, &v);
-#expect INTCONST ab = int 42
+#expect NUMCONST ab = int 42
     }
 }
diff --git a/test/codegen/glob-003c.c b/test/codegen/glob-003c.c
--- a/test/codegen/glob-003c.c
+++ b/test/codegen/glob-003c.c
@@ -10,6 +10,6 @@
         (void)((ab) << 1);  /* check that 'ab' is an integer */
         t1 = CRX_INT_CONST(cb, ab, &v, 1);
         cb->define_num_const(cb, "ab", t1, &v);
-#expect INTCONST ab = unsigned int 42
+#expect NUMCONST ab = unsigned int 42
     }
 }
diff --git a/test/codegen/glob-003d.c b/test/codegen/glob-003d.c
--- a/test/codegen/glob-003d.c
+++ b/test/codegen/glob-003d.c
@@ -11,6 +11,6 @@
         CRX_CHAR_CONST(cb, ab, &v);
         t1 = cb->get_char_type(cb);
         cb->define_num_const(cb, "ab", t1, &v);
-#expect INTCONST ab = char '*'
+#expect NUMCONST ab = char '*'
     }
 }
diff --git a/test/codegen/glob-003e.c b/test/codegen/glob-003e.c
--- a/test/codegen/glob-003e.c
+++ b/test/codegen/glob-003e.c
@@ -10,6 +10,6 @@
         (void)((ab) << 1);  /* check that 'ab' is an integer */
         t1 = CRX_INT_CONST(cb, ab, &v, 2);
         cb->define_num_const(cb, "ab", t1, &v);
-#expect INTCONST ab = long -42
+#expect NUMCONST ab = long -42
     }
 }
diff --git a/test/codegen/glob-003f.c b/test/codegen/glob-003f.c
--- a/test/codegen/glob-003f.c
+++ b/test/codegen/glob-003f.c
@@ -10,6 +10,6 @@
         (void)((ab) << 1);  /* check that 'ab' is an integer */
         t1 = CRX_INT_CONST(cb, ab, &v, 3);
         cb->define_num_const(cb, "ab", t1, &v);
-#expect INTCONST ab = long long -42
+#expect NUMCONST ab = long long -42
     }
 }
diff --git a/test/codegen/glob-003g.c b/test/codegen/glob-003g.c
--- a/test/codegen/glob-003g.c
+++ b/test/codegen/glob-003g.c
@@ -10,6 +10,6 @@
         (void)((ab) << 1);  /* check that 'ab' is an integer */
         t1 = CRX_INT_CONST(cb, ab, &v, 1);
         cb->define_num_const(cb, "ab", t1, &v);
-#expect INTCONST ab = int -42
+#expect NUMCONST ab = int -42
     }
 }

From noreply at buildbot.pypy.org  Tue Nov 18 20:38:15 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Tue, 18 Nov 2014 20:38:15 +0100 (CET)
Subject: [pypy-commit] creflect default: long double test. Add some older
 test files that I forgot.
Message-ID: <20141118193815.3D3D31C31CF@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r92:f71495c9c2aa
Date: 2014-11-18 20:38 +0100
http://bitbucket.org/cffi/creflect/changeset/f71495c9c2aa/

Log:	long double test. Add some older test files that I forgot.

diff --git a/test/cgcompile.c b/test/cgcompile.c
--- a/test/cgcompile.c
+++ b/test/cgcompile.c
@@ -73,6 +73,7 @@
     int found = 0;
     TT(float);
     TT(double);
+    TT(long double);
     assert(found == sz);
 #undef TT
     return newtype(g);
@@ -201,7 +202,7 @@
     else if (strcmp(t->text, "double") == 0)
         printf("%f\n", value->as_double);
     else if (strcmp(t->text, "long double") == 0)
-        printf("%f\n", (double)value->as_double);
+        printf("%f\n", (double)value->as_long_double);
     else {
         printf("???\n");
         abort();
diff --git a/test/codegen/func-001c.c b/test/codegen/func-001c.c
new file mode 100644
--- /dev/null
+++ b/test/codegen/func-001c.c
@@ -0,0 +1,27 @@
+double f(float);
+
+# ____________________________________________________________
+
+double f(float a) { return a + 0.25f; }
+
+# ____________________________________________________________
+
+static void testfunc_001c__c_f(void *args[], void *result) {
+    *(double *)result = f(*(float *)args[0]);
+}
+
+static double testfunc_001c__d_f(float a0) {
+    return f(a0);
+}
+
+void testfunc_001c(crx_builder_t *cb)
+{
+    crx_type_t *t1, *t2, *a3[1];
+    {
+        t1 = cb->get_float_type(cb, sizeof(double), "double");
+        t2 = cb->get_float_type(cb, sizeof(float), "float");
+        a3[0] = t2;
+        cb->define_func(cb, "f", t1, a3, 1, &testfunc_001c__c_f, &testfunc_001c__d_f);
+#expect FUNC f: float -> double
+    }
+}
diff --git a/test/codegen/glob-003h.c b/test/codegen/glob-003h.c
new file mode 100644
--- /dev/null
+++ b/test/codegen/glob-003h.c
@@ -0,0 +1,23 @@
+float const ab = 42.0;
+double const abd = 42.0;
+
+# ____________________________________________________________
+
+void testglob_003h(crx_builder_t *cb)
+{
+    crx_type_t *t1, *t2;
+    {
+        crx_num_const_t v;
+        (void)((ab) / 2.0);  /* check that 'ab' is a number */
+        t1 = CRX_FLOAT_CONST(cb, ab, &v);
+        cb->define_num_const(cb, "ab", t1, &v);
+#expect NUMCONST ab = float 42.000000
+    }
+    {
+        crx_num_const_t v;
+        (void)((abd) / 2.0);  /* check that 'abd' is a number */
+        t2 = CRX_FLOAT_CONST(cb, abd, &v);
+        cb->define_num_const(cb, "abd", t2, &v);
+#expect NUMCONST abd = double 42.000000
+    }
+}
diff --git a/test/codegen/glob-003i.c b/test/codegen/glob-003i.c
new file mode 100644
--- /dev/null
+++ b/test/codegen/glob-003i.c
@@ -0,0 +1,15 @@
+long double const abl = 42.0;
+
+# ____________________________________________________________
+
+void testglob_003i(crx_builder_t *cb)
+{
+    crx_type_t *t1;
+    {
+        crx_num_const_t v;
+        (void)((abl) / 2.0);  /* check that 'abl' is a number */
+        t1 = CRX_FLOAT_CONST(cb, abl, &v);
+        cb->define_num_const(cb, "abl", t1, &v);
+#expect NUMCONST abl = long double 42.000000
+    }
+}

From noreply at buildbot.pypy.org  Tue Nov 18 21:23:19 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Tue, 18 Nov 2014 21:23:19 +0100 (CET)
Subject: [pypy-commit] creflect default: Small reorganizations until all
	tests pass
Message-ID: <20141118202319.EE74E1C1148@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r93:b04afb735e35
Date: 2014-11-18 21:23 +0100
http://bitbucket.org/cffi/creflect/changeset/b04afb735e35/

Log:	Small reorganizations until all tests pass

diff --git a/creflect/codegen.py b/creflect/codegen.py
--- a/creflect/codegen.py
+++ b/creflect/codegen.py
@@ -56,6 +56,7 @@
 
 def transform_cdef(csource, crx_func_name):
     outerblock = CodeBlock(parent=None)
+    outerblock.writeline('#include "creflect.h"')
     outerblock.crx_func_name = crx_func_name
 
     funcblock = CodeBlock(outerblock)
diff --git a/creflect/driver.py b/creflect/driver.py
--- a/creflect/driver.py
+++ b/creflect/driver.py
@@ -73,22 +73,13 @@
 
 _DEBUG_TEMPLATE1 = '''
 /***** CREFLECT debug code *****/
-#include 
-#include 
+#include "creflect_print.h"
 int main(void) {
 %s    return 0;
 }
 '''
 _DEBUG_TEMPLATE2 = '''\
-    {
-        int size = %(funcname)s((char *)0);
-        char *result = malloc(size);
-        int err = %(funcname)s(result);
-        printf("%%s", result);
-        free(result);
-        if (err != 0)
-            return 1;
-    }
+    %(funcname)s(&maincb);
 '''
 
 def get_debug_code(reflection_func_name):
diff --git a/creflect/model.py b/creflect/model.py
--- a/creflect/model.py
+++ b/creflect/model.py
@@ -117,7 +117,10 @@
             if not self.const:
                 block.writeline("%s = -1;%s" % (star_p1, comment3))
             if self.is_integer_type():
-                expr = 'CRX_INT_TYPE(cb, %s, "%s")' % (star_p1, self.name)
+                hint = self.name.split()
+                if hint[0] in ('signed', 'unsigned'):
+                    hint = hint[1:]
+                expr = 'CRX_INT_TYPE(cb, %s, "%s")' % (star_p1, " ".join(hint))
             elif self.is_char_type():
                 errmsg = "numeric type '%s' is not a char" % (
                     inspect.get_comment_type(0, False),)
diff --git a/creflect/creflect.h b/creflect/src/creflect.h
rename from creflect/creflect.h
rename to creflect/src/creflect.h
--- a/creflect/creflect.h
+++ b/creflect/src/creflect.h
@@ -1,3 +1,6 @@
+#ifndef _CREFLECT_H_
+#define _CREFLECT_H_
+
 #include 
 #include 
 
@@ -123,17 +126,17 @@
         else if (fits_int) {
             vp->as_unsigned_int = (unsigned int)value;
             size = sizeof(unsigned int);
-            name = "unsigned int";
+            name = "int";
         }
         else if (fits_long) {
             vp->as_unsigned_long = (unsigned long)value;
             size = sizeof(unsigned long);
-            name = "unsigned long";
+            name = "long";
         }
         else {
             vp->as_unsigned_long_long = (unsigned long long)value;
             size = sizeof(unsigned long long);
-            name = "unsigned long long";
+            name = "long long";
         }
         return cb->get_unsigned_type(cb, size, name);
     }
@@ -196,3 +199,5 @@
     }
     return cb->get_float_type(cb, size, name);
 }
+
+#endif  /* _CREFLECT_H_ */
diff --git a/test/cgcompile.c b/creflect/src/creflect_print.h
rename from test/cgcompile.c
rename to creflect/src/creflect_print.h
--- a/test/cgcompile.c
+++ b/creflect/src/creflect_print.h
@@ -1,4 +1,5 @@
 #include 
+#include 
 #include 
 #include 
 
@@ -43,40 +44,61 @@
 static crx_type_t *tst_get_signed_type(crx_builder_t *cb, size_t sz,
                                        const char *g)
 {
-#define TT(name)  if (strcmp(g, #name) == 0) { found = sizeof(name); }
-    int found = 0;
+    int skip = 0;
+    if (sizeof(long) == sizeof(long long))
+        if (strcmp(g, "long long") == 0)
+            skip = 4;
+    if (sizeof(int) == sizeof(long))
+        if (strcmp(g, "long") == 0 || strcmp(g, "long long") == 0)
+            skip = 3;
+
+#define TT(name)   if (--skip && sz == sizeof(name)) { return newtype(#name); }
     TT(signed char);
     TT(short);
     TT(int);
     TT(long);
     TT(long long);
-    assert(found == sz);
-    return newtype(g);
+
+    printf("cannot find signed type with %zd bytes\n", sz);
+    abort();
 }
 
 static crx_type_t *tst_get_unsigned_type(crx_builder_t *cb, size_t sz,
                                          const char *g)
 {
-    int found = 0;
+    int skip = 0;
+    if (sizeof(long) == sizeof(long long))
+        if (strcmp(g, "long long") == 0)
+            skip = 4;
+    if (sizeof(int) == sizeof(long))
+        if (strcmp(g, "long") == 0 || strcmp(g, "long long") == 0)
+            skip = 3;
+
     TT(unsigned char);
     TT(unsigned short);
     TT(unsigned int);
     TT(unsigned long);
     TT(unsigned long long);
-    assert(found == sz);
-    return newtype(g);
+
+    printf("cannot find unsigned type with %zd bytes\n", sz);
+    abort();
 }
 
 static crx_type_t *tst_get_float_type(crx_builder_t *cb, size_t sz,
                                       const char *g)
 {
-    int found = 0;
+    int skip = 0;
+    if (sizeof(double) == sizeof(long double))
+        if (strcmp(g, "long double") == 0)
+            skip = 2;
+
     TT(float);
     TT(double);
     TT(long double);
-    assert(found == sz);
 #undef TT
-    return newtype(g);
+
+    printf("cannot find float type with %zd bytes\n", sz);
+    abort();
 }
 
 static crx_type_t *tst_get_function_type(crx_builder_t *cb, crx_type_t *ret,
@@ -238,9 +260,3 @@
     tst_define_num_const,
     tst_error,
 };
-
-int main(void)
-{
-    TESTFN(&maincb);
-    return 0;
-}
diff --git a/test/codegen/003d.c b/test/codegen/003d.c
--- a/test/codegen/003d.c
+++ b/test/codegen/003d.c
@@ -11,7 +11,7 @@
         p1 = (void *)b;
         (void)(*p1 << 1);  /* check that 'num_t' is an integer type */
         *p1 = -1;  /* check that 'num_t' is not declared 'const' */
-        t1 = CRX_INT_TYPE(cb, *p1, "signed char");
+        t1 = CRX_INT_TYPE(cb, *p1, "char");
         cb->define_type(cb, "num_t", t1);
 #expect TYPEDEF num_t = signed char
     }
diff --git a/test/codegen/003f.c b/test/codegen/003f.c
--- a/test/codegen/003f.c
+++ b/test/codegen/003f.c
@@ -11,7 +11,7 @@
         p1 = (void *)b;
         (void)(*p1 << 1);  /* check that 'num_t' is an integer type */
         *p1 = -1;  /* check that 'num_t' is not declared 'const' */
-        t1 = CRX_INT_TYPE(cb, *p1, "unsigned long long");
+        t1 = CRX_INT_TYPE(cb, *p1, "long long");
         cb->define_type(cb, "num_t", t1);
 #expect TYPEDEF num_t = unsigned long long
     }
diff --git a/test/codegen/003h.c b/test/codegen/003h.c
new file mode 100644
--- /dev/null
+++ b/test/codegen/003h.c
@@ -0,0 +1,22 @@
+typedef long long num_t;
+
+# ____________________________________________________________
+
+#define num_t int
+
+# ____________________________________________________________
+
+void test003h(crx_builder_t *cb)
+{
+    crx_type_t *t1;
+    {
+        num_t *p1;
+        char b[sizeof(*p1)];
+        p1 = (void *)b;
+        (void)(*p1 << 1);  /* check that 'num_t' is an integer type */
+        *p1 = -1;  /* check that 'num_t' is not declared 'const' */
+        t1 = CRX_INT_TYPE(cb, *p1, "long long");
+        cb->define_type(cb, "num_t", t1);
+#expect TYPEDEF num_t = int
+    }
+}
diff --git a/test/codegen/006.c b/test/codegen/006.c
--- a/test/codegen/006.c
+++ b/test/codegen/006.c
@@ -11,7 +11,7 @@
         memset(b, -1, sizeof(b));
         p1 = (void *)b;
         (void)(*p1 << 1);  /* check that 'num_t' is an integer type */
-        t1 = CRX_INT_TYPE(cb, *p1, "unsigned int");
+        t1 = CRX_INT_TYPE(cb, *p1, "int");
         t2 = cb->get_const_type(cb, t1);
         cb->define_type(cb, "num_t", t2);
 #expect TYPEDEF num_t = CONST unsigned int
diff --git a/test/codegen/struct-001.c b/test/codegen/struct-001.c
--- a/test/codegen/struct-001.c
+++ b/test/codegen/struct-001.c
@@ -31,7 +31,7 @@
         p1 = (void *)(((char *)b) - o);
         (void)(p1->bb << 1);  /* check that 'struct foo_s::bb' is an integer type */
         p1->bb = -1;  /* check that 'struct foo_s::bb' is not declared 'const' */
-        t3 = CRX_INT_TYPE(cb, p1->bb, "unsigned int");
+        t3 = CRX_INT_TYPE(cb, p1->bb, "int");
         d1[1].name = "bb";
         d1[1].type = t3;
         d1[1].offset = o;
diff --git a/test/test_cgcompile.py b/test/test_cgcompile.py
--- a/test/test_cgcompile.py
+++ b/test/test_cgcompile.py
@@ -21,17 +21,23 @@
     assert infile != outfile
     f = open(infile, 'w')
     f.write('#include \n')
-    f.write('#include "%s/../../creflect/creflect.h"\n' % path)
-    f.write('#define TESTFN(cb) test%s(cb)\n' % (basename.replace('-','_'),))
+    f.write('#include "%s/../../creflect/src/creflect.h"\n' % path)
     f.write('\n')
     for line in inputlines:
         if not (line.startswith('# ') or line.startswith('#expect')):
             f.write(line)
     f.write('\n')
-    f.write('#include "%s/../cgcompile.c"\n' % path)
+    f.write('#include "%s/../../creflect/src/creflect_print.h"\n' % path)
+    f.write('''
+int main(void)
+{
+    test%s(&maincb);
+    return 0;
+}
+''' % (basename.replace('-','_'),))
     f.close()
     #
-    err = os.system("gcc -Werror -Wall '%s' -o '%s'" % (infile, outfile))
+    err = os.system("gcc -g -Werror -Wall '%s' -o '%s'" % (infile, outfile))
     assert err == 0
     #
     g = os.popen("'%s'" % (outfile,), 'r')
diff --git a/test/test_driver.py b/test/test_driver.py
--- a/test/test_driver.py
+++ b/test/test_driver.py
@@ -9,9 +9,11 @@
     print >> f, get_debug_code(funcname)
     f.close()
     #
+    from creflect.driver import __file__
+    main_dir = os.path.abspath(os.path.dirname(__file__))
     err = os.system(
-        "cd '%s' && gcc -fPIC -Werror -Wall %s.c -o %s"
-        % (str(udir), funcname, funcname))
+        "cd '%s' && gcc -fPIC -Werror -Wall %s.c -o %s -I%s/src"
+        % (str(udir), funcname, funcname, main_dir))
     assert err == 0
     #
     g = os.popen("cd '%s' && %s" % (str(udir), funcname), 'r')
@@ -26,7 +28,7 @@
     gen = expand_cdef("typedef long a_t[20];", "test_expand_cdef")
     code = '/* real code */\n%s\n/* generated code */\n%s' % (real_code, gen)
     data = compile_and_run(code, "test_expand_cdef")
-    assert data == real_code
+    assert data == "TYPEDEF a_t = ARRAY[42] short\n"
 
 
 def test_copy_file_and_expand():
@@ -41,4 +43,4 @@
     copy_file_and_expand(f, g)
     #
     data = compile_and_run(g.getvalue(), "inspect1")
-    assert data == "typedef unsigned short a_t[5];\n"
+    assert data == "TYPEDEF a_t = ARRAY[5] unsigned short\n"

From noreply at buildbot.pypy.org  Tue Nov 18 23:45:19 2014
From: noreply at buildbot.pypy.org (mattip)
Date: Tue, 18 Nov 2014 23:45:19 +0100 (CET)
Subject: [pypy-commit] pypy ufuncapi: correct the test so it now fails
Message-ID: <20141118224519.E9A0A1C1147@cobra.cs.uni-duesseldorf.de>

Author: mattip 
Branch: ufuncapi
Changeset: r74574:8932643e5a01
Date: 2014-11-18 21:52 +0200
http://bitbucket.org/pypy/pypy/changeset/8932643e5a01/

Log:	correct the test so it now fails

diff --git a/pypy/module/micronumpy/test/test_ufuncs.py b/pypy/module/micronumpy/test/test_ufuncs.py
--- a/pypy/module/micronumpy/test/test_ufuncs.py
+++ b/pypy/module/micronumpy/test/test_ufuncs.py
@@ -162,10 +162,9 @@
 
     def test_frompyfunc_2d_sig(self):
         def times_2(in_array, out_array):
-            in_flat = in_array.flat
-            out_flat = out_array.flat
-            for i in range(in_array.size):
-                out_flat[i] = in_flat[i] * 2
+            assert len(in_array.shape) == 2
+            assert in_array.shape == out_array.shape
+            out_array[:] = in_array * 2
         from numpy import frompyfunc, dtype, arange
         ufunc = frompyfunc([times_2], 1, 1,
                             signature='(m,m)->(m,m)',
@@ -176,7 +175,7 @@
         exc = raises(ValueError, ufunc, ai[:,:,0])
         assert "mismatch in its core dimension 1" in exc.value.message
         ai2 = ufunc(ai)
-        assert all(ai1 == ai * 2)
+        assert (ai2 == ai * 2).all()
 
     def test_ufunc_kwargs(self):
         from numpy import ufunc, frompyfunc, arange, dtype
diff --git a/pypy/module/micronumpy/ufuncs.py b/pypy/module/micronumpy/ufuncs.py
--- a/pypy/module/micronumpy/ufuncs.py
+++ b/pypy/module/micronumpy/ufuncs.py
@@ -699,7 +699,8 @@
         # TODO parse and handle subok
         # TODO handle flags, op_flags
         w_flags = space.w_None #space.newlist([space.wrap('external_loop')])
-        w_op_flags = space.w_None
+        w_op_flags = space.newtuple([space.wrap(['readonly'])] * len(inargs) + \
+                                    [space.wrap(['readwrite'])] * len(outargs))
         w_op_dtypes = space.w_None
         w_casting = space.w_None
         w_itershape = space.newlist([space.wrap(i) for i in iter_shape]) 
@@ -708,7 +709,6 @@
         # mimic NpyIter_AdvancedNew with a nditer
 
         if self.stack_inputs:
-            inargs = inargs + outargs
             nd_it = W_NDIter(space, space.newlist(inargs + outargs), w_flags,
                           w_op_flags, w_op_dtypes, w_casting, w_op_axes,
                           w_itershape)

From noreply at buildbot.pypy.org  Tue Nov 18 23:45:24 2014
From: noreply at buildbot.pypy.org (mattip)
Date: Tue, 18 Nov 2014 23:45:24 +0100 (CET)
Subject: [pypy-commit] pypy ufuncapi: merge default into branch
Message-ID: <20141118224524.D74A91C1147@cobra.cs.uni-duesseldorf.de>

Author: mattip 
Branch: ufuncapi
Changeset: r74575:9199a7897ae2
Date: 2014-11-18 21:52 +0200
http://bitbucket.org/pypy/pypy/changeset/9199a7897ae2/

Log:	merge default into branch

diff too long, truncating to 2000 out of 8557 lines

diff --git a/README.rst b/README.rst
--- a/README.rst
+++ b/README.rst
@@ -37,4 +37,4 @@
 to use virtualenv with the resulting pypy-c as the interpreter; you can
 find more details about various installation schemes here:
 
-http://doc.pypy.org/en/latest/getting-started.html#installing-pypy
+    http://doc.pypy.org/en/latest/install.html
diff --git a/lib-python/conftest.py b/lib-python/conftest.py
--- a/lib-python/conftest.py
+++ b/lib-python/conftest.py
@@ -59,7 +59,7 @@
     def __init__(self, basename, core=False, compiler=None, usemodules='',
                  skip=None):
         self.basename = basename
-        self._usemodules = usemodules.split() + ['signal', 'rctime', 'itertools', '_socket']
+        self._usemodules = usemodules.split() + ['signal', 'time', 'itertools', '_socket']
         self._compiler = compiler
         self.core = core
         self.skip = skip
diff --git a/lib_pypy/grp.py b/lib_pypy/grp.py
--- a/lib_pypy/grp.py
+++ b/lib_pypy/grp.py
@@ -66,11 +66,12 @@
 
 @builtinify
 def getgrnam(name):
-    if not isinstance(name, str):
+    if not isinstance(name, basestring):
         raise TypeError("expected string")
+    name = str(name)
     res = libc.getgrnam(name)
     if not res:
-        raise KeyError(name)
+        raise KeyError("'getgrnam(): name not found: %s'" % name)
     return _group_from_gstruct(res)
 
 @builtinify
diff --git a/pypy/config/pypyoption.py b/pypy/config/pypyoption.py
--- a/pypy/config/pypyoption.py
+++ b/pypy/config/pypyoption.py
@@ -29,7 +29,7 @@
 # --allworkingmodules
 working_modules = default_modules.copy()
 working_modules.update([
-    "_socket", "unicodedata", "mmap", "fcntl", "_locale", "pwd", "rctime" ,
+    "_socket", "unicodedata", "mmap", "fcntl", "_locale", "pwd", "time" ,
     "select", "zipimport", "_lsprof", "crypt", "signal", "_rawffi", "termios",
     "zlib", "bz2", "struct", "_hashlib", "_md5", "_sha", "_minimal_curses",
     "cStringIO", "thread", "itertools", "pyexpat", "_ssl", "cpyext", "array",
@@ -40,7 +40,7 @@
 
 translation_modules = default_modules.copy()
 translation_modules.update([
-    "fcntl", "rctime", "select", "signal", "_rawffi", "zlib", "struct", "_md5",
+    "fcntl", "time", "select", "signal", "_rawffi", "zlib", "struct", "_md5",
     "cStringIO", "array", "binascii",
     # the following are needed for pyrepl (and hence for the
     # interactive prompt/pdb)
@@ -64,19 +64,15 @@
     default_modules.add("_locale")
 
 if sys.platform == "sunos5":
-    working_modules.remove('mmap')   # depend on ctypes, can't get at c-level 'errono'
-    working_modules.remove('rctime') # depend on ctypes, missing tm_zone/tm_gmtoff
-    working_modules.remove('signal') # depend on ctypes, can't get at c-level 'errono'
     working_modules.remove('fcntl')  # LOCK_NB not defined
     working_modules.remove("_minimal_curses")
     working_modules.remove("termios")
-    working_modules.remove("_multiprocessing")   # depends on rctime
     if "cppyy" in working_modules:
         working_modules.remove("cppyy")  # depends on ctypes
 
 
 module_dependencies = {
-    '_multiprocessing': [('objspace.usemodules.rctime', True),
+    '_multiprocessing': [('objspace.usemodules.time', True),
                          ('objspace.usemodules.thread', True)],
     'cpyext': [('objspace.usemodules.array', True)],
     'cppyy': [('objspace.usemodules.cpyext', True)],
@@ -86,9 +82,10 @@
     # itself needs the interp-level struct module
     # because 'P' is missing from the app-level one
     "_rawffi": [("objspace.usemodules.struct", True)],
-    "cpyext": [("translation.secondaryentrypoints", "cpyext,main"),
-               ("translation.shared", sys.platform == "win32")],
+    "cpyext": [("translation.secondaryentrypoints", "cpyext,main")],
 }
+if sys.platform == "win32":
+    module_suggests["cpyext"].append(("translation.shared", True))
 
 module_import_dependencies = {
     # no _rawffi if importing rpython.rlib.clibffi raises ImportError
diff --git a/pypy/doc/config/objspace.usemodules.rctime.txt b/pypy/doc/config/objspace.usemodules.rctime.txt
deleted file mode 100644
--- a/pypy/doc/config/objspace.usemodules.rctime.txt
+++ /dev/null
@@ -1,7 +0,0 @@
-Use the 'rctime' module. 
-
-'rctime' is our `rffi`_ based implementation of the builtin 'time' module.
-It supersedes the less complete :config:`objspace.usemodules.time`,
-at least for C-like targets (the C and LLVM backends).
-
-.. _`rffi`: ../rffi.html
diff --git a/pypy/doc/config/objspace.usemodules.time.txt b/pypy/doc/config/objspace.usemodules.time.txt
--- a/pypy/doc/config/objspace.usemodules.time.txt
+++ b/pypy/doc/config/objspace.usemodules.time.txt
@@ -1,5 +1,1 @@
 Use the 'time' module. 
-
-Obsolete; use :config:`objspace.usemodules.rctime` for our up-to-date version
-of the application-level 'time' module, at least for C-like targets (the C
-and LLVM backends).
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -43,3 +43,11 @@
 .. branch nditer-external_loop
 
 Implement `external_loop` arguement to numpy's nditer
+
+.. branch kill-rctime
+
+Rename pypy/module/rctime to pypy/module/time, since it contains the implementation of the 'time' module.
+
+.. branch: ssa-flow
+
+Use SSA form for flow graphs inside build_flow() and part of simplify_graph()
diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -522,11 +522,6 @@
                 if name not in modules:
                     modules.append(name)
 
-        # a bit of custom logic: rctime take precedence over time
-        # XXX this could probably be done as a "requires" in the config
-        if 'rctime' in modules and 'time' in modules:
-            modules.remove('time')
-
         self._builtinmodule_list = modules
         return self._builtinmodule_list
 
diff --git a/pypy/interpreter/executioncontext.py b/pypy/interpreter/executioncontext.py
--- a/pypy/interpreter/executioncontext.py
+++ b/pypy/interpreter/executioncontext.py
@@ -32,6 +32,17 @@
         self.compiler = space.createcompiler()
         self.profilefunc = None
         self.w_profilefuncarg = None
+        self.thread_disappeared = False   # might be set to True after os.fork()
+
+    @staticmethod
+    def _mark_thread_disappeared(space):
+        # Called in the child process after os.fork() by interp_posix.py.
+        # Marks all ExecutionContexts except the current one
+        # with 'thread_disappeared = True'.
+        me = space.getexecutioncontext()
+        for ec in space.threadlocals.getallvalues().values():
+            if ec is not me:
+                ec.thread_disappeared = True
 
     def gettopframe(self):
         return self.topframeref()
diff --git a/pypy/module/_file/interp_stream.py b/pypy/module/_file/interp_stream.py
--- a/pypy/module/_file/interp_stream.py
+++ b/pypy/module/_file/interp_stream.py
@@ -34,8 +34,12 @@
         # this function runs with the GIL acquired so there is no race
         # condition in the creation of the lock
         me = self.space.getexecutioncontext()   # used as thread ident
-        if self.slockowner is me:
-            return False    # already acquired by the current thread
+        if self.slockowner is not None:
+            if self.slockowner is me:
+                return False    # already acquired by the current thread
+            if self.slockowner.thread_disappeared:
+                self.slockowner = None
+                self.slock = None
         try:
             if self.slock is None:
                 self.slock = self.space.allocate_lock()
diff --git a/pypy/module/_file/test/test_file_extra.py b/pypy/module/_file/test/test_file_extra.py
--- a/pypy/module/_file/test/test_file_extra.py
+++ b/pypy/module/_file/test/test_file_extra.py
@@ -221,7 +221,7 @@
     expected_filename = str(udir.join('sample'))
     expected_mode = 'rb'
     extra_args = ()
-    spaceconfig = {'usemodules': ['binascii', 'rctime', 'struct']}
+    spaceconfig = {'usemodules': ['binascii', 'time', 'struct']}
 
     def setup_method(self, method):
         space = self.space
@@ -281,7 +281,7 @@
     expected_filename = ''
     expected_mode = 'rb'
     extra_args = ()
-    spaceconfig = {'usemodules': ['binascii', 'rctime', 'struct']}
+    spaceconfig = {'usemodules': ['binascii', 'time', 'struct']}
 
     def setup_method(self, method):
         space = self.space
@@ -359,7 +359,7 @@
 #  A few extra tests
 
 class AppTestAFewExtra:
-    spaceconfig = {'usemodules': ['_socket', 'array', 'binascii', 'rctime',
+    spaceconfig = {'usemodules': ['_socket', 'array', 'binascii', 'time',
                                   'struct']}
 
     def setup_method(self, method):
diff --git a/pypy/module/_lsprof/test/test_cprofile.py b/pypy/module/_lsprof/test/test_cprofile.py
--- a/pypy/module/_lsprof/test/test_cprofile.py
+++ b/pypy/module/_lsprof/test/test_cprofile.py
@@ -1,6 +1,6 @@
 class AppTestCProfile(object):
     spaceconfig = {
-        "usemodules": ['_lsprof', 'rctime'],
+        "usemodules": ['_lsprof', 'time'],
     }
 
     def setup_class(cls):
diff --git a/pypy/module/_md5/test/test_md5.py b/pypy/module/_md5/test/test_md5.py
--- a/pypy/module/_md5/test/test_md5.py
+++ b/pypy/module/_md5/test/test_md5.py
@@ -5,7 +5,7 @@
 
 class AppTestMD5(object):
     spaceconfig = {
-        'usemodules': ['_md5', 'binascii', 'rctime', 'struct'],
+        'usemodules': ['_md5', 'binascii', 'time', 'struct'],
     }
 
     def setup_class(cls):
diff --git a/pypy/module/_multibytecodec/src/cjkcodecs/multibytecodec.h b/pypy/module/_multibytecodec/src/cjkcodecs/multibytecodec.h
--- a/pypy/module/_multibytecodec/src/cjkcodecs/multibytecodec.h
+++ b/pypy/module/_multibytecodec/src/cjkcodecs/multibytecodec.h
@@ -97,24 +97,24 @@
   Py_UNICODE *outbuf_start, *outbuf, *outbuf_end;
 };
 
-RPY_EXPORTED_FOR_TESTS
+RPY_EXTERN
 struct pypy_cjk_dec_s *pypy_cjk_dec_new(const MultibyteCodec *codec);
-RPY_EXPORTED_FOR_TESTS
+RPY_EXTERN
 Py_ssize_t pypy_cjk_dec_init(struct pypy_cjk_dec_s *d,
                              char *inbuf, Py_ssize_t inlen);
-RPY_EXPORTED_FOR_TESTS
+RPY_EXTERN
 void pypy_cjk_dec_free(struct pypy_cjk_dec_s *);
-RPY_EXPORTED_FOR_TESTS
+RPY_EXTERN
 Py_ssize_t pypy_cjk_dec_chunk(struct pypy_cjk_dec_s *);
-RPY_EXPORTED_FOR_TESTS
+RPY_EXTERN
 Py_UNICODE *pypy_cjk_dec_outbuf(struct pypy_cjk_dec_s *);
-RPY_EXPORTED_FOR_TESTS
+RPY_EXTERN
 Py_ssize_t pypy_cjk_dec_outlen(struct pypy_cjk_dec_s *);
-RPY_EXPORTED_FOR_TESTS
+RPY_EXTERN
 Py_ssize_t pypy_cjk_dec_inbuf_remaining(struct pypy_cjk_dec_s *d);
-RPY_EXPORTED_FOR_TESTS
+RPY_EXTERN
 Py_ssize_t pypy_cjk_dec_inbuf_consumed(struct pypy_cjk_dec_s* d);
-RPY_EXPORTED_FOR_TESTS
+RPY_EXTERN
 Py_ssize_t pypy_cjk_dec_replace_on_error(struct pypy_cjk_dec_s* d,
                                          Py_UNICODE *, Py_ssize_t, Py_ssize_t);
 
@@ -125,35 +125,35 @@
   unsigned char *outbuf_start, *outbuf, *outbuf_end;
 };
 
-RPY_EXPORTED_FOR_TESTS
+RPY_EXTERN
 struct pypy_cjk_enc_s *pypy_cjk_enc_new(const MultibyteCodec *codec);
-RPY_EXPORTED_FOR_TESTS
+RPY_EXTERN
 Py_ssize_t pypy_cjk_enc_init(struct pypy_cjk_enc_s *d,
                              Py_UNICODE *inbuf, Py_ssize_t inlen);
-RPY_EXPORTED_FOR_TESTS
+RPY_EXTERN
 void pypy_cjk_enc_free(struct pypy_cjk_enc_s *);
-RPY_EXPORTED_FOR_TESTS
+RPY_EXTERN
 Py_ssize_t pypy_cjk_enc_chunk(struct pypy_cjk_enc_s *, Py_ssize_t);
-RPY_EXPORTED_FOR_TESTS
+RPY_EXTERN
 Py_ssize_t pypy_cjk_enc_reset(struct pypy_cjk_enc_s *);
-RPY_EXPORTED_FOR_TESTS
+RPY_EXTERN
 char *pypy_cjk_enc_outbuf(struct pypy_cjk_enc_s *);
-RPY_EXPORTED_FOR_TESTS
+RPY_EXTERN
 Py_ssize_t pypy_cjk_enc_outlen(struct pypy_cjk_enc_s *);
-RPY_EXPORTED_FOR_TESTS
+RPY_EXTERN
 Py_ssize_t pypy_cjk_enc_inbuf_remaining(struct pypy_cjk_enc_s *d);
-RPY_EXPORTED_FOR_TESTS
+RPY_EXTERN
 Py_ssize_t pypy_cjk_enc_inbuf_consumed(struct pypy_cjk_enc_s* d);
-RPY_EXPORTED_FOR_TESTS
+RPY_EXTERN
 Py_ssize_t pypy_cjk_enc_replace_on_error(struct pypy_cjk_enc_s* d,
                                          char *, Py_ssize_t, Py_ssize_t);
-RPY_EXPORTED_FOR_TESTS
+RPY_EXTERN
 const MultibyteCodec *pypy_cjk_enc_getcodec(struct pypy_cjk_enc_s *);
 
 /* list of codecs defined in the .c files */
 
 #define DEFINE_CODEC(name)                              \
-    RPY_EXPORTED_FOR_TESTS MultibyteCodec *pypy_cjkcodec_##name(void);
+    RPY_EXTERN MultibyteCodec *pypy_cjkcodec_##name(void);
 
 // _codecs_cn
 DEFINE_CODEC(gb2312)
diff --git a/pypy/module/_multiprocessing/interp_semaphore.py b/pypy/module/_multiprocessing/interp_semaphore.py
--- a/pypy/module/_multiprocessing/interp_semaphore.py
+++ b/pypy/module/_multiprocessing/interp_semaphore.py
@@ -254,7 +254,7 @@
         start = _GetTickCount()
 
         while True:
-            from pypy.module.rctime.interp_time import State
+            from pypy.module.time.interp_time import State
             interrupt_event = space.fromcache(State).get_interrupt_event()
             handles = [self.handle, interrupt_event]
 
diff --git a/pypy/module/_random/test/test_random.py b/pypy/module/_random/test/test_random.py
--- a/pypy/module/_random/test/test_random.py
+++ b/pypy/module/_random/test/test_random.py
@@ -1,6 +1,6 @@
 class AppTestRandom:
     spaceconfig = {
-        "usemodules": ['_random', 'rctime'],
+        "usemodules": ['_random', 'time'],
     }
 
     def test_dict(self):
diff --git a/pypy/module/_sha/test/test_sha.py b/pypy/module/_sha/test/test_sha.py
--- a/pypy/module/_sha/test/test_sha.py
+++ b/pypy/module/_sha/test/test_sha.py
@@ -5,7 +5,7 @@
 
 class AppTestSHA(object):
     spaceconfig = {
-        'usemodules': ['_sha', 'binascii', 'rctime', 'struct'],
+        'usemodules': ['_sha', 'binascii', 'time', 'struct'],
     }
 
     def setup_class(cls):
diff --git a/pypy/module/_ssl/thread_lock.py b/pypy/module/_ssl/thread_lock.py
--- a/pypy/module/_ssl/thread_lock.py
+++ b/pypy/module/_ssl/thread_lock.py
@@ -65,7 +65,7 @@
 eci = rthread.eci.merge(ExternalCompilationInfo(
     separate_module_sources=[separate_module_source],
     post_include_bits=[
-        "RPY_EXPORTED_FOR_TESTS int _PyPy_SSL_SetupThreads(void);"],
+        "RPY_EXTERN int _PyPy_SSL_SetupThreads(void);"],
     libraries = libraries,
 ))
 
diff --git a/pypy/module/bz2/test/test_bz2_file.py b/pypy/module/bz2/test/test_bz2_file.py
--- a/pypy/module/bz2/test/test_bz2_file.py
+++ b/pypy/module/bz2/test/test_bz2_file.py
@@ -53,7 +53,7 @@
 
 class AppTestBZ2File(CheckAllocation):
     spaceconfig = {
-        'usemodules': ['bz2', 'binascii', 'rctime', 'struct']
+        'usemodules': ['bz2', 'binascii', 'time', 'struct']
     }
 
     def setup_class(cls):
diff --git a/pypy/module/cppyy/include/capi.h b/pypy/module/cppyy/include/capi.h
--- a/pypy/module/cppyy/include/capi.h
+++ b/pypy/module/cppyy/include/capi.h
@@ -2,6 +2,7 @@
 #define CPPYY_CAPI
 
 #include 
+#include "src/precommondefs.h"
 
 #ifdef __cplusplus
 extern "C" {
@@ -15,102 +16,167 @@
     typedef void* (*cppyy_methptrgetter_t)(cppyy_object_t);
 
     /* name to opaque C++ scope representation -------------------------------- */
+    RPY_EXTERN
     int cppyy_num_scopes(cppyy_scope_t parent);
+    RPY_EXTERN
     char* cppyy_scope_name(cppyy_scope_t parent, int iscope);
 
+    RPY_EXTERN
     char* cppyy_resolve_name(const char* cppitem_name);
+    RPY_EXTERN
     cppyy_scope_t cppyy_get_scope(const char* scope_name);
+    RPY_EXTERN
     cppyy_type_t cppyy_get_template(const char* template_name);
+    RPY_EXTERN
     cppyy_type_t cppyy_actual_class(cppyy_type_t klass, cppyy_object_t obj);
 
     /* memory management ------------------------------------------------------ */
+    RPY_EXTERN
     cppyy_object_t cppyy_allocate(cppyy_type_t type);
+    RPY_EXTERN
     void cppyy_deallocate(cppyy_type_t type, cppyy_object_t self);
+    RPY_EXTERN
     void cppyy_destruct(cppyy_type_t type, cppyy_object_t self);
 
     /* method/function dispatching -------------------------------------------- */
+    RPY_EXTERN
     void   cppyy_call_v(cppyy_method_t method, cppyy_object_t self, int nargs, void* args);
+    RPY_EXTERN
     unsigned char cppyy_call_b(cppyy_method_t method, cppyy_object_t self, int nargs, void* args);
+    RPY_EXTERN
     char   cppyy_call_c(cppyy_method_t method, cppyy_object_t self, int nargs, void* args);
+    RPY_EXTERN
     short  cppyy_call_h(cppyy_method_t method, cppyy_object_t self, int nargs, void* args);
+    RPY_EXTERN
     int    cppyy_call_i(cppyy_method_t method, cppyy_object_t self, int nargs, void* args);
+    RPY_EXTERN
     long   cppyy_call_l(cppyy_method_t method, cppyy_object_t self, int nargs, void* args);
+    RPY_EXTERN
     long long cppyy_call_ll(cppyy_method_t method, cppyy_object_t self, int nargs, void* args);
+    RPY_EXTERN
     float  cppyy_call_f(cppyy_method_t method, cppyy_object_t self, int nargs, void* args);
+    RPY_EXTERN
     double cppyy_call_d(cppyy_method_t method, cppyy_object_t self, int nargs, void* args);
 
+    RPY_EXTERN
     void*  cppyy_call_r(cppyy_method_t method, cppyy_object_t self, int nargs, void* args);
+    RPY_EXTERN
     char*  cppyy_call_s(cppyy_method_t method, cppyy_object_t self, int nargs, void* args);
 
+    RPY_EXTERN
     cppyy_object_t cppyy_constructor(cppyy_method_t method, cppyy_type_t klass, int nargs, void* args);
+    RPY_EXTERN
     cppyy_object_t cppyy_call_o(cppyy_method_t method, cppyy_object_t self, int nargs, void* args, cppyy_type_t result_type);
 
+    RPY_EXTERN
     cppyy_methptrgetter_t cppyy_get_methptr_getter(cppyy_scope_t scope, cppyy_index_t idx);
 
     /* handling of function argument buffer ----------------------------------- */
+    RPY_EXTERN
     void*  cppyy_allocate_function_args(int nargs);
+    RPY_EXTERN
     void   cppyy_deallocate_function_args(void* args);
+    RPY_EXTERN
     size_t cppyy_function_arg_sizeof();
+    RPY_EXTERN
     size_t cppyy_function_arg_typeoffset();
 
     /* scope reflection information ------------------------------------------- */
+    RPY_EXTERN
     int cppyy_is_namespace(cppyy_scope_t scope);
+    RPY_EXTERN
     int cppyy_is_enum(const char* type_name);
 
     /* class reflection information ------------------------------------------- */
+    RPY_EXTERN
     char* cppyy_final_name(cppyy_type_t type);
+    RPY_EXTERN
     char* cppyy_scoped_final_name(cppyy_type_t type);
+    RPY_EXTERN
     int cppyy_has_complex_hierarchy(cppyy_type_t type);
+    RPY_EXTERN
     int cppyy_num_bases(cppyy_type_t type);
+    RPY_EXTERN
     char* cppyy_base_name(cppyy_type_t type, int base_index);
+    RPY_EXTERN
     int cppyy_is_subtype(cppyy_type_t derived, cppyy_type_t base);
 
     /* calculate offsets between declared and actual type, up-cast: direction > 0; down-cast: direction < 0 */
+    RPY_EXTERN
     ptrdiff_t cppyy_base_offset(cppyy_type_t derived, cppyy_type_t base, cppyy_object_t address, int direction);
 
     /* method/function reflection information --------------------------------- */
+    RPY_EXTERN
     int cppyy_num_methods(cppyy_scope_t scope);
+    RPY_EXTERN
     cppyy_index_t cppyy_method_index_at(cppyy_scope_t scope, int imeth);
+    RPY_EXTERN
     cppyy_index_t* cppyy_method_indices_from_name(cppyy_scope_t scope, const char* name);
 
+    RPY_EXTERN
     char* cppyy_method_name(cppyy_scope_t scope, cppyy_index_t idx);
+    RPY_EXTERN
     char* cppyy_method_result_type(cppyy_scope_t scope, cppyy_index_t idx);
+    RPY_EXTERN
     int cppyy_method_num_args(cppyy_scope_t scope, cppyy_index_t idx);
+    RPY_EXTERN
     int cppyy_method_req_args(cppyy_scope_t scope, cppyy_index_t idx);
+    RPY_EXTERN
     char* cppyy_method_arg_type(cppyy_scope_t scope, cppyy_index_t idx, int arg_index);
+    RPY_EXTERN
     char* cppyy_method_arg_default(cppyy_scope_t scope, cppyy_index_t idx, int arg_index);
+    RPY_EXTERN
     char* cppyy_method_signature(cppyy_scope_t scope, cppyy_index_t idx);
 
+    RPY_EXTERN
     int cppyy_method_is_template(cppyy_scope_t scope, cppyy_index_t idx);
+    RPY_EXTERN
     int cppyy_method_num_template_args(cppyy_scope_t scope, cppyy_index_t idx);
+    RPY_EXTERN
     char* cppyy_method_template_arg_name(cppyy_scope_t scope, cppyy_index_t idx, cppyy_index_t iarg);
 
+    RPY_EXTERN
     cppyy_method_t cppyy_get_method(cppyy_scope_t scope, cppyy_index_t idx);
+    RPY_EXTERN
     cppyy_index_t cppyy_get_global_operator(
         cppyy_scope_t scope, cppyy_scope_t lc, cppyy_scope_t rc, const char* op);
 
     /* method properties ------------------------------------------------------ */
+    RPY_EXTERN
     int cppyy_is_constructor(cppyy_type_t type, cppyy_index_t idx);
+    RPY_EXTERN
     int cppyy_is_staticmethod(cppyy_type_t type, cppyy_index_t idx);
 
     /* data member reflection information ------------------------------------- */
+    RPY_EXTERN
     int cppyy_num_datamembers(cppyy_scope_t scope);
+    RPY_EXTERN
     char* cppyy_datamember_name(cppyy_scope_t scope, int datamember_index);
+    RPY_EXTERN
     char* cppyy_datamember_type(cppyy_scope_t scope, int datamember_index);
+    RPY_EXTERN
     ptrdiff_t cppyy_datamember_offset(cppyy_scope_t scope, int datamember_index);
 
+    RPY_EXTERN
     int cppyy_datamember_index(cppyy_scope_t scope, const char* name);
 
     /* data member properties ------------------------------------------------- */
+    RPY_EXTERN
     int cppyy_is_publicdata(cppyy_type_t type, int datamember_index);
+    RPY_EXTERN
     int cppyy_is_staticdata(cppyy_type_t type, int datamember_index);
 
     /* misc helpers ----------------------------------------------------------- */
+    RPY_EXTERN
     long long cppyy_strtoll(const char* str);
+    RPY_EXTERN
     unsigned long long cppyy_strtoull(const char* str);
+    RPY_EXTERN
     void cppyy_free(void* ptr);
 
+    RPY_EXTERN
     cppyy_object_t cppyy_charp2stdstring(const char* str);
+    RPY_EXTERN
     cppyy_object_t cppyy_stdstring2stdstring(cppyy_object_t ptr);
 
 #ifdef __cplusplus
diff --git a/pypy/module/cppyy/src/dummy_backend.cxx b/pypy/module/cppyy/src/dummy_backend.cxx
--- a/pypy/module/cppyy/src/dummy_backend.cxx
+++ b/pypy/module/cppyy/src/dummy_backend.cxx
@@ -1,4 +1,3 @@
-#include "src/precommondefs.h"
 #include "cppyy.h"
 #include "capi.h"
 
@@ -349,29 +348,24 @@
 
 
 /* name to opaque C++ scope representation -------------------------------- */
-RPY_EXPORTED_FOR_TESTS
 int cppyy_num_scopes(cppyy_scope_t handle) {
     return 0;
 }
 
-RPY_EXPORTED_FOR_TESTS
 char* cppyy_resolve_name(const char* cppitem_name) {
     return cppstring_to_cstring(cppitem_name);
 }
 
-RPY_EXPORTED_FOR_TESTS
 cppyy_scope_t cppyy_get_scope(const char* scope_name) {
     return s_handles[scope_name];  // lookup failure will return 0 (== error)
 }
 
-RPY_EXPORTED_FOR_TESTS
 cppyy_type_t cppyy_actual_class(cppyy_type_t klass, cppyy_object_t /* obj */) {
     return klass;
 }
 
 
 /* memory management ------------------------------------------------------ */
-RPY_EXPORTED_FOR_TESTS
 void cppyy_destruct(cppyy_type_t handle, cppyy_object_t self) {
     if (handle == s_handles["example01"])
        delete (dummy::example01*)self;
@@ -379,7 +373,6 @@
 
 
 /* method/function dispatching -------------------------------------------- */
-RPY_EXPORTED_FOR_TESTS
 void cppyy_call_v(cppyy_method_t method, cppyy_object_t self, int nargs, void* args) {
     long idx = (long)method;
     if (idx == s_methods["static_example01::staticSetPayload_payload*_double"]) {
@@ -469,7 +462,6 @@
     }
 }
 
-RPY_EXPORTED_FOR_TESTS
 unsigned char cppyy_call_b(cppyy_method_t method, cppyy_object_t self, int nargs, void* args) {
     unsigned char result = 0;
     const long idx = (long)method;
@@ -482,7 +474,6 @@
     return result;
 }
 
-RPY_EXPORTED_FOR_TESTS
 char cppyy_call_c(cppyy_method_t method, cppyy_object_t self, int nargs, void* args) {
     char result = 0;
     const long idx = (long)method;
@@ -498,7 +489,6 @@
     return result;
 }
 
-RPY_EXPORTED_FOR_TESTS
 short cppyy_call_h(cppyy_method_t method, cppyy_object_t self, int nargs, void* args) {
     short result = 0;
     const long idx = (long)method; 
@@ -514,7 +504,6 @@
     return result;
 }
 
-RPY_EXPORTED_FOR_TESTS
 int cppyy_call_i(cppyy_method_t method, cppyy_object_t self, int nargs, void* args) {
     int result = 0;
     const long idx = (long)method;
@@ -547,7 +536,6 @@
     return result;
 }
 
-RPY_EXPORTED_FOR_TESTS
 long cppyy_call_l(cppyy_method_t method, cppyy_object_t self, int nargs, void* args) {
     long result = 0;
     const long idx = (long)method;
@@ -689,7 +677,6 @@
     return result;
 }
 
-RPY_EXPORTED_FOR_TESTS
 long long cppyy_call_ll(cppyy_method_t method, cppyy_object_t self, int nargs, void* args) {
     long long result = 0;
     const long idx = (long)method;
@@ -705,7 +692,6 @@
     return result;
 }   
 
-RPY_EXPORTED_FOR_TESTS
 float cppyy_call_f(cppyy_method_t method, cppyy_object_t self, int nargs, void* args) {
     float result = 0;
     const long idx = (long)method;
@@ -718,7 +704,6 @@
     return result;
 }   
 
-RPY_EXPORTED_FOR_TESTS
 double cppyy_call_d(cppyy_method_t method, cppyy_object_t self, int nargs, void* args) {
     double result = 0.;
     const long idx = (long)method;
@@ -740,7 +725,6 @@
     return result;
 }
 
-RPY_EXPORTED_FOR_TESTS
 char* cppyy_call_s(cppyy_method_t method, cppyy_object_t self, int nargs, void* args) {
     char* result = 0;
     const long idx = (long)method;
@@ -753,7 +737,6 @@
     return result;
 }
 
-RPY_EXPORTED_FOR_TESTS
 cppyy_object_t cppyy_constructor(cppyy_method_t method, cppyy_type_t handle, int nargs, void* args) {
     void* result = 0;
     const long idx = (long)method;
@@ -776,14 +759,12 @@
     return (cppyy_object_t)result;
 }
 
-RPY_EXPORTED_FOR_TESTS
 cppyy_methptrgetter_t cppyy_get_methptr_getter(cppyy_type_t /* handle */, cppyy_index_t /* method_index */) {
     return (cppyy_methptrgetter_t)0;
 }
 
 
 /* handling of function argument buffer ----------------------------------- */
-RPY_EXPORTED_FOR_TESTS
 void* cppyy_allocate_function_args(int nargs) {
     CPPYY_G__value* args = (CPPYY_G__value*)malloc(nargs*sizeof(CPPYY_G__value));
     for (int i = 0; i < nargs; ++i)
@@ -793,36 +774,30 @@
 
 
 /* handling of function argument buffer ----------------------------------- */
-RPY_EXPORTED_FOR_TESTS
 void cppyy_deallocate_function_args(void* args) {
     free(args);
 }
 
-RPY_EXPORTED_FOR_TESTS
 size_t cppyy_function_arg_sizeof() {
     return sizeof(CPPYY_G__value);
 }
 
-RPY_EXPORTED_FOR_TESTS
 size_t cppyy_function_arg_typeoffset() {
     return offsetof(CPPYY_G__value, type);
 }
 
 
 /* scope reflection information ------------------------------------------- */
-RPY_EXPORTED_FOR_TESTS
 int cppyy_is_namespace(cppyy_scope_t /* handle */) {
     return 0;
 }   
 
-RPY_EXPORTED_FOR_TESTS
 int cppyy_is_enum(const char* /* type_name */) {
     return 0;
 }
     
     
 /* class reflection information ------------------------------------------- */
-RPY_EXPORTED_FOR_TESTS
 char* cppyy_final_name(cppyy_type_t handle) {
     for (Handles_t::iterator isp = s_handles.begin(); isp != s_handles.end(); ++isp) {
         if (isp->second == handle)
@@ -831,75 +806,61 @@
     return cppstring_to_cstring("");
 }
 
-RPY_EXPORTED_FOR_TESTS
 char* cppyy_scoped_final_name(cppyy_type_t handle) {
     return cppyy_final_name(handle);
 }   
 
-RPY_EXPORTED_FOR_TESTS
 int cppyy_has_complex_hierarchy(cppyy_type_t /* handle */) {
     return 0;
 }
 
-RPY_EXPORTED_FOR_TESTS
 int cppyy_num_bases(cppyy_type_t /*handle*/) {
    return 0;
 }
 
 
 /* method/function reflection information --------------------------------- */
-RPY_EXPORTED_FOR_TESTS
 int cppyy_num_methods(cppyy_scope_t handle) {
     return s_scopes[handle].m_methods.size();
 }
 
-RPY_EXPORTED_FOR_TESTS
 cppyy_index_t cppyy_method_index_at(cppyy_scope_t /* scope */, int imeth) {
     return (cppyy_index_t)imeth;
 }
 
-RPY_EXPORTED_FOR_TESTS
 char* cppyy_method_name(cppyy_scope_t handle, cppyy_index_t method_index) {
     return cppstring_to_cstring(s_scopes[handle].m_methods[(int)method_index].m_name);
 }
 
-RPY_EXPORTED_FOR_TESTS
 char* cppyy_method_result_type(cppyy_scope_t handle, cppyy_index_t method_index) {
     return cppstring_to_cstring(s_scopes[handle].m_methods[method_index].m_returntype);
 }
     
-RPY_EXPORTED_FOR_TESTS
 int cppyy_method_num_args(cppyy_scope_t handle, cppyy_index_t method_index) {
     return s_scopes[handle].m_methods[method_index].m_argtypes.size();
 }
 
-RPY_EXPORTED_FOR_TESTS
 int cppyy_method_req_args(cppyy_scope_t handle, cppyy_index_t method_index) {
     return cppyy_method_num_args(handle, method_index);
 }
 
-RPY_EXPORTED_FOR_TESTS
 char* cppyy_method_arg_type(cppyy_scope_t handle, cppyy_index_t method_index, int arg_index) {
     return cppstring_to_cstring(s_scopes[handle].m_methods[method_index].m_argtypes[arg_index]);
 }
 
-RPY_EXPORTED_FOR_TESTS
 char* cppyy_method_arg_default(
         cppyy_scope_t /* handle */, cppyy_index_t /* method_index */, int /* arg_index */) {
     return cppstring_to_cstring("");
 }
 
-RPY_EXPORTED_FOR_TESTS
 char* cppyy_method_signature(cppyy_scope_t /* handle */, cppyy_index_t /* method_index */) {
     return cppstring_to_cstring("");
 }
 
-RPY_EXPORTED_FOR_TESTS
 int cppyy_method_is_template(cppyy_scope_t /* handle */, cppyy_index_t /* method_index */) {
     return 0;
 }
     
-RPY_EXPORTED_FOR_TESTS
 cppyy_method_t cppyy_get_method(cppyy_scope_t handle, cppyy_index_t method_index) {
     if (s_scopes.find(handle) != s_scopes.end()) {
         long id = s_scopes[handle].m_method_offset + (long)method_index;
@@ -911,7 +872,6 @@
 
 
 /* method properties -----------------------------------------------------  */
-RPY_EXPORTED_FOR_TESTS
 int cppyy_is_constructor(cppyy_type_t handle, cppyy_index_t method_index) {
     if (s_scopes.find(handle) != s_scopes.end())
         return s_scopes[handle].m_methods[method_index].m_type == kConstructor;
@@ -919,7 +879,6 @@
     return 0;
 }
 
-RPY_EXPORTED_FOR_TESTS
 int cppyy_is_staticmethod(cppyy_type_t handle, cppyy_index_t method_index) {
     if (s_scopes.find(handle) != s_scopes.end())
         return s_scopes[handle].m_methods[method_index].m_type == kStatic;
@@ -929,34 +888,28 @@
 
 
 /* data member reflection information ------------------------------------- */
-RPY_EXPORTED_FOR_TESTS
 int cppyy_num_datamembers(cppyy_scope_t handle) {
     return s_scopes[handle].m_datambrs.size();
 }
 
-RPY_EXPORTED_FOR_TESTS
 char* cppyy_datamember_name(cppyy_scope_t handle, int idatambr) {
     return cppstring_to_cstring(s_scopes[handle].m_datambrs[idatambr].m_name);
 }
 
-RPY_EXPORTED_FOR_TESTS
 char* cppyy_datamember_type(cppyy_scope_t handle, int idatambr) {
     return cppstring_to_cstring(s_scopes[handle].m_datambrs[idatambr].m_type);
 }
 
-RPY_EXPORTED_FOR_TESTS
 ptrdiff_t cppyy_datamember_offset(cppyy_scope_t handle, int idatambr) {
     return s_scopes[handle].m_datambrs[idatambr].m_offset;
 }
 
 
 /* data member properties ------------------------------------------------  */
-RPY_EXPORTED_FOR_TESTS
 int cppyy_is_publicdata(cppyy_scope_t handle, int idatambr) {
     return 1;
 }
 
-RPY_EXPORTED_FOR_TESTS
 int cppyy_is_staticdata(cppyy_scope_t handle, int idatambr) {
     return s_scopes[handle].m_datambrs[idatambr].m_isstatic;
 }
@@ -964,44 +917,37 @@
 
 /* misc helpers ----------------------------------------------------------- */
 #if defined(_MSC_VER)
-RPY_EXPORTED_FOR_TESTS
 long long cppyy_strtoll(const char* str) {
     return _strtoi64(str, NULL, 0);
 }
 
 extern "C" {
-RPY_EXPORTED_FOR_TESTS
 unsigned long long cppyy_strtoull(const char* str) {
     return _strtoui64(str, NULL, 0);
 }
 }
 #else
-RPY_EXPORTED_FOR_TESTS
 long long cppyy_strtoll(const char* str) {
     return strtoll(str, NULL, 0);
 }
 
 extern "C" {
-RPY_EXPORTED_FOR_TESTS
 unsigned long long cppyy_strtoull(const char* str) {
     return strtoull(str, NULL, 0);
 }
 }
 #endif
 
-RPY_EXPORTED_FOR_TESTS
 void cppyy_free(void* ptr) {
     free(ptr);
 }
 
-RPY_EXPORTED_FOR_TESTS
 cppyy_object_t cppyy_charp2stdstring(const char* str) {
     void* arena = new char[sizeof(std::string)];
     new (arena) std::string(str);
     return (cppyy_object_t)arena;
 }
 
-RPY_EXPORTED_FOR_TESTS
 cppyy_object_t cppyy_stdstring2stdstring(cppyy_object_t ptr) {
     void* arena = new char[sizeof(std::string)];
     new (arena) std::string(*(std::string*)ptr);
diff --git a/pypy/module/cppyy/test/conftest.py b/pypy/module/cppyy/test/conftest.py
--- a/pypy/module/cppyy/test/conftest.py
+++ b/pypy/module/cppyy/test/conftest.py
@@ -50,7 +50,7 @@
             eci = ExternalCompilationInfo(
                 separate_module_files=[srcpath.join('dummy_backend.cxx')],
                 include_dirs=[incpath, tstpath, cdir],
-                compile_extra=['-DRPY_EXPORTED_FOR_TESTS=RPY_EXPORTED'],
+                compile_extra=['-DRPY_EXTERN=RPY_EXPORTED'],
                 use_cpp_linker=True,
             )
 
diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -775,8 +775,7 @@
     struct PyPyAPI {
     %(members)s
     } _pypyAPI;
-    RPY_EXPORTED_FOR_TESTS
-    struct PyPyAPI* pypyAPI = &_pypyAPI;
+    RPY_EXTERN struct PyPyAPI* pypyAPI = &_pypyAPI;
     """ % dict(members=structmembers)
 
     functions = generate_decls_and_callbacks(db, export_symbols)
@@ -947,7 +946,7 @@
         name_no_star = process_va_name(name)
         header = ('%s pypy_va_get_%s(va_list* vp)' %
                   (name, name_no_star))
-        pypy_decls.append('RPY_EXPORTED_FOR_TESTS ' + header + ';')
+        pypy_decls.append('RPY_EXTERN ' + header + ';')
         functions.append(header + '\n{return va_arg(*vp, %s);}\n' % name)
 
     for name, (typ, expr) in GLOBALS.iteritems():
@@ -1007,7 +1006,7 @@
     if sys.platform == 'win32':
         get_pythonapi_source = '''
         #include 
-        RPY_EXPORTED_FOR_TESTS
+        RPY_EXTERN
         HANDLE pypy_get_pythonapi_handle() {
             MEMORY_BASIC_INFORMATION  mi;
             memset(&mi, 0, sizeof(mi));
diff --git a/pypy/module/cpyext/include/modsupport.h b/pypy/module/cpyext/include/modsupport.h
--- a/pypy/module/cpyext/include/modsupport.h
+++ b/pypy/module/cpyext/include/modsupport.h
@@ -78,11 +78,20 @@
 /*
  * This is from pyport.h.  Perhaps it belongs elsewhere.
  */
+#ifdef _WIN32
+/* explicitly export since PyAPI_FUNC is usually dllimport */
+#ifdef __cplusplus
+#define PyMODINIT_FUNC extern "C" __declspec(dllexport) void
+#else
+#define PyMODINIT_FUNC __declspec(dllexport) void
+#endif
+#else
 #ifdef __cplusplus
 #define PyMODINIT_FUNC extern "C" PyAPI_FUNC(void)
 #else
 #define PyMODINIT_FUNC PyAPI_FUNC(void)
 #endif
+#endif /* WIN32 */
 
 PyAPI_DATA(char *) _Py_PackageContext;
 
diff --git a/pypy/module/cpyext/test/conftest.py b/pypy/module/cpyext/test/conftest.py
--- a/pypy/module/cpyext/test/conftest.py
+++ b/pypy/module/cpyext/test/conftest.py
@@ -7,7 +7,7 @@
     # it's necessary to run "import time" at least once before any
     # other cpyext test, otherwise the same statement will fail in
     # test_datetime.py.
-    space = gettestobjspace(usemodules=['rctime'])
+    space = gettestobjspace(usemodules=['time'])
     space.getbuiltinmodule("time")
 
 def pytest_ignore_collect(path, config):
diff --git a/pypy/module/cpyext/test/test_cpyext.py b/pypy/module/cpyext/test/test_cpyext.py
--- a/pypy/module/cpyext/test/test_cpyext.py
+++ b/pypy/module/cpyext/test/test_cpyext.py
@@ -102,7 +102,7 @@
 class LeakCheckingTest(object):
     """Base class for all cpyext tests."""
     spaceconfig = dict(usemodules=['cpyext', 'thread', '_rawffi', 'array',
-                                   'itertools', 'rctime', 'binascii', 'micronumpy'])
+                                   'itertools', 'time', 'binascii', 'micronumpy'])
     spaceconfig['std.withmethodcache'] = True
 
     enable_leak_checking = True
diff --git a/pypy/module/fcntl/test/test_fcntl.py b/pypy/module/fcntl/test/test_fcntl.py
--- a/pypy/module/fcntl/test/test_fcntl.py
+++ b/pypy/module/fcntl/test/test_fcntl.py
@@ -12,7 +12,7 @@
 
 class AppTestFcntl:
     spaceconfig = dict(usemodules=('fcntl', 'array', 'struct', 'termios',
-                                   'select', 'rctime'))
+                                   'select', 'time'))
 
     def setup_class(cls):
         tmpprefix = str(udir.ensure('test_fcntl', dir=1).join('tmp_'))
diff --git a/pypy/module/imp/test/test_app.py b/pypy/module/imp/test/test_app.py
--- a/pypy/module/imp/test/test_app.py
+++ b/pypy/module/imp/test/test_app.py
@@ -4,7 +4,7 @@
 
 class AppTestImpModule:
     spaceconfig = {
-        'usemodules': ['binascii', 'imp', 'itertools', 'rctime', 'struct'],
+        'usemodules': ['binascii', 'imp', 'itertools', 'time', 'struct'],
     }
 
     def setup_class(cls):
diff --git a/pypy/module/imp/test/test_import.py b/pypy/module/imp/test/test_import.py
--- a/pypy/module/imp/test/test_import.py
+++ b/pypy/module/imp/test/test_import.py
@@ -149,7 +149,7 @@
 
 class AppTestImport:
     spaceconfig = {
-        "usemodules": ['_md5', 'rctime'],
+        "usemodules": ['_md5', 'time'],
     }
 
     def setup_class(cls):
@@ -1044,7 +1044,7 @@
 
 class AppTestImportHooks(object):
     spaceconfig = {
-        "usemodules": ['struct', 'itertools', 'rctime'],
+        "usemodules": ['struct', 'itertools', 'time'],
     }
 
     def setup_class(cls):
@@ -1304,7 +1304,7 @@
 
 
 class AppTestMultithreadedImp(object):
-    spaceconfig = dict(usemodules=['thread', 'rctime'])
+    spaceconfig = dict(usemodules=['thread', 'time'])
 
     def setup_class(cls):
         #if not conftest.option.runappdirect:
diff --git a/pypy/module/math/test/test_math.py b/pypy/module/math/test/test_math.py
--- a/pypy/module/math/test/test_math.py
+++ b/pypy/module/math/test/test_math.py
@@ -7,7 +7,7 @@
 
 class AppTestMath:
     spaceconfig = {
-        "usemodules": ['math', 'struct', 'itertools', 'rctime', 'binascii'],
+        "usemodules": ['math', 'struct', 'itertools', 'time', 'binascii'],
     }
 
     def setup_class(cls):
diff --git a/pypy/module/micronumpy/nditer.py b/pypy/module/micronumpy/nditer.py
--- a/pypy/module/micronumpy/nditer.py
+++ b/pypy/module/micronumpy/nditer.py
@@ -89,10 +89,9 @@
     for w_item in lst:
         if not space.isinstance_w(w_item, space.w_str) and not \
                 space.isinstance_w(w_item, space.w_unicode):
-            typename = space.type(w_item).getname(space)
             raise oefmt(space.w_TypeError,
-                        'expected string or Unicode object, %s found',
-                        typename)
+                        "expected string or Unicode object, %T found",
+                        w_item)
         item = space.str_w(w_item)
         if item == 'external_loop':
             nditer.external_loop = True
diff --git a/pypy/module/mmap/interp_mmap.py b/pypy/module/mmap/interp_mmap.py
--- a/pypy/module/mmap/interp_mmap.py
+++ b/pypy/module/mmap/interp_mmap.py
@@ -5,6 +5,7 @@
 from rpython.rlib import rmmap, rarithmetic
 from rpython.rlib.buffer import Buffer
 from rpython.rlib.rmmap import RValueError, RTypeError, RMMapError
+from rpython.rlib.rstring import StringBuilder
 
 if rmmap.HAVE_LARGEFILE_SUPPORT:
     OFF_T = rarithmetic.r_longlong
@@ -163,17 +164,18 @@
         self.check_valid()
 
         space = self.space
-        start, stop, step = space.decode_index(w_index, self.mmap.size)
+        start, stop, step, length = space.decode_index4(w_index, self.mmap.size)
         if step == 0:  # index only
             return space.wrap(self.mmap.getitem(start))
         elif step == 1:
             if stop - start < 0:
                 return space.wrap("")
-            return space.wrap(self.mmap.getslice(start, stop - start))
+            return space.wrap(self.mmap.getslice(start, length))
         else:
-            res = "".join([self.mmap.getitem(i)
-                           for i in range(start, stop, step)])
-            return space.wrap(res)
+            b = StringBuilder(length)
+            for i in range(start, stop, step):
+                b.append(self.mmap.getitem(i))
+            return space.wrap(b.build())
 
     def descr_setitem(self, w_index, w_value):
         space = self.space
diff --git a/pypy/module/operator/tscmp.h b/pypy/module/operator/tscmp.h
--- a/pypy/module/operator/tscmp.h
+++ b/pypy/module/operator/tscmp.h
@@ -1,2 +1,2 @@
-RPY_EXPORTED_FOR_TESTS int pypy_tscmp(const char *, const char *, long, long);
-RPY_EXPORTED_FOR_TESTS int pypy_tscmp_wide(const wchar_t *, const wchar_t *, long, long);
+RPY_EXTERN int pypy_tscmp(const char *, const char *, long, long);
+RPY_EXTERN int pypy_tscmp_wide(const wchar_t *, const wchar_t *, long, long);
diff --git a/pypy/module/posix/interp_posix.py b/pypy/module/posix/interp_posix.py
--- a/pypy/module/posix/interp_posix.py
+++ b/pypy/module/posix/interp_posix.py
@@ -10,6 +10,7 @@
 
 from pypy.interpreter.gateway import unwrap_spec
 from pypy.interpreter.error import OperationError, wrap_oserror, wrap_oserror2
+from pypy.interpreter.executioncontext import ExecutionContext
 from pypy.module.sys.interp_encoding import getfilesystemencoding
 
 
@@ -721,6 +722,8 @@
     "NOT_RPYTHON"
     get_fork_hooks(where).append(hook)
 
+add_fork_hook('child', ExecutionContext._mark_thread_disappeared)
+
 @specialize.arg(0)
 def run_fork_hooks(where, space):
     for hook in get_fork_hooks(where):
diff --git a/pypy/module/posix/test/test_posix2.py b/pypy/module/posix/test/test_posix2.py
--- a/pypy/module/posix/test/test_posix2.py
+++ b/pypy/module/posix/test/test_posix2.py
@@ -14,7 +14,7 @@
 import signal
 
 def setup_module(mod):
-    usemodules = ['binascii', 'posix', 'struct', 'rctime']
+    usemodules = ['binascii', 'posix', 'struct', 'time']
     if os.name != 'nt':
         usemodules += ['fcntl']
     else:
diff --git a/pypy/module/posix/test/test_posix_libfile.py b/pypy/module/posix/test/test_posix_libfile.py
--- a/pypy/module/posix/test/test_posix_libfile.py
+++ b/pypy/module/posix/test/test_posix_libfile.py
@@ -10,7 +10,7 @@
 
 class AppTestPosix:
     spaceconfig = {
-        "usemodules": ['posix', 'rctime'],
+        "usemodules": ['posix', 'time'],
     }
 
     def setup_class(cls):
diff --git a/pypy/module/pyexpat/test/test_parser.py b/pypy/module/pyexpat/test/test_parser.py
--- a/pypy/module/pyexpat/test/test_parser.py
+++ b/pypy/module/pyexpat/test/test_parser.py
@@ -177,7 +177,7 @@
 
 class AppTestPyexpat2:
     spaceconfig = dict(usemodules=['pyexpat', 'itertools', '_socket',
-                                   'rctime', 'struct', 'binascii'])
+                                   'time', 'struct', 'binascii'])
 
     def test_django_bug(self):
         xml_str = ''
diff --git a/pypy/module/pypyjit/test/test_policy.py b/pypy/module/pypyjit/test/test_policy.py
--- a/pypy/module/pypyjit/test/test_policy.py
+++ b/pypy/module/pypyjit/test/test_policy.py
@@ -29,7 +29,7 @@
     assert pypypolicy.look_inside_function(get_ident)
 
 def test_time():
-    from pypy.module.rctime.interp_time import time
+    from pypy.module.time.interp_time import time
     assert pypypolicy.look_inside_function(time)
 
 def test_io():
diff --git a/pypy/module/rctime/__init__.py b/pypy/module/rctime/__init__.py
deleted file mode 100644
--- a/pypy/module/rctime/__init__.py
+++ /dev/null
@@ -1,42 +0,0 @@
-
-from pypy.interpreter.mixedmodule import MixedModule
-import os
-
-_WIN = os.name == "nt"
-
-class Module(MixedModule):
-    applevel_name = 'time'
-
-    interpleveldefs = {
-        'time': 'interp_time.time',
-        'clock': 'interp_time.clock',
-        'ctime': 'interp_time.ctime',
-        'asctime': 'interp_time.asctime',
-        'gmtime': 'interp_time.gmtime',
-        'localtime': 'interp_time.localtime',
-        'mktime': 'interp_time.mktime',
-        'strftime': 'interp_time.strftime',
-        'sleep' : 'interp_time.sleep',
-    }
-
-    if os.name == "posix":
-        interpleveldefs['tzset'] = 'interp_time.tzset'
-
-    appleveldefs = {
-        'struct_time': 'app_time.struct_time',
-        '__doc__': 'app_time.__doc__',
-        'strptime': 'app_time.strptime',
-    }
-
-    def startup(self, space):
-        if _WIN:
-            from pypy.module.rctime.interp_time import State
-            space.fromcache(State).startup(space)
-
-        # this machinery is needed to expose constants
-        # that have to be initialized one time only
-        from pypy.module.rctime import interp_time
-
-        interp_time._init_timezone(space)
-        interp_time._init_accept2dyear(space)
-
diff --git a/pypy/module/rctime/interp_time.py b/pypy/module/rctime/interp_time.py
deleted file mode 100644
--- a/pypy/module/rctime/interp_time.py
+++ /dev/null
@@ -1,670 +0,0 @@
-from rpython.rtyper.tool import rffi_platform as platform
-from rpython.rtyper.lltypesystem import rffi
-from pypy.interpreter.error import OperationError, oefmt
-from pypy.interpreter.gateway import unwrap_spec
-from rpython.rtyper.lltypesystem import lltype
-from rpython.rlib.rarithmetic import intmask
-from rpython.rlib import rposix
-from rpython.translator.tool.cbuild import ExternalCompilationInfo
-import os
-import sys
-import time as pytime
-
-_POSIX = os.name == "posix"
-_WIN = os.name == "nt"
-_CYGWIN = sys.platform == "cygwin"
-
-_time_zones = []
-if _CYGWIN:
-    _time_zones = ["GMT-12", "GMT-11", "GMT-10", "GMT-9", "GMT-8", "GMT-7",
-                   "GMT-6", "GMT-5", "GMT-4", "GMT-3", "GMT-2", "GMT-1",
-                   "GMT",  "GMT+1", "GMT+2", "GMT+3", "GMT+4", "GMT+5",
-                   "GMT+6",  "GMT+7", "GMT+8", "GMT+9", "GMT+10", "GMT+11",
-                   "GMT+12",  "GMT+13", "GMT+14"]
-
-if _WIN:
-    # Interruptible sleeps on Windows:
-    # We install a specific Console Ctrl Handler which sets an 'event'.
-    # time.sleep() will actually call WaitForSingleObject with the desired
-    # timeout.  On Ctrl-C, the signal handler is called, the event is set,
-    # and the wait function exits.
-    from rpython.rlib import rwin32
-    from pypy.interpreter.error import wrap_windowserror, wrap_oserror
-    from rpython.rlib import rthread as thread
-
-    eci = ExternalCompilationInfo(
-        includes = ['windows.h'],
-        post_include_bits = [
-            "RPY_EXPORTED_FOR_TESTS\n"
-            "BOOL pypy_timemodule_setCtrlHandler(HANDLE event);"],
-        separate_module_sources=['''
-            static HANDLE interrupt_event;
-
-            static BOOL WINAPI CtrlHandlerRoutine(
-              DWORD dwCtrlType)
-            {
-                SetEvent(interrupt_event);
-                /* allow other default handlers to be called.
-                 * Default Python handler will setup the
-                 * KeyboardInterrupt exception.
-                 */
-                return 0;
-            }
-
-            BOOL pypy_timemodule_setCtrlHandler(HANDLE event)
-            {
-                interrupt_event = event;
-                return SetConsoleCtrlHandler(CtrlHandlerRoutine, TRUE);
-            }
-
-        '''],
-        )
-    _setCtrlHandlerRoutine = rffi.llexternal(
-        'pypy_timemodule_setCtrlHandler',
-        [rwin32.HANDLE], rwin32.BOOL,
-        compilation_info=eci)
-
-    class GlobalState:
-        def __init__(self):
-            self.init()
-
-        def init(self):
-            self.interrupt_event = rwin32.NULL_HANDLE
-
-        def startup(self, space):
-            # Initialize the event handle used to signal Ctrl-C
-            try:
-                globalState.interrupt_event = rwin32.CreateEvent(
-                    rffi.NULL, True, False, rffi.NULL)
-            except WindowsError, e:
-                raise wrap_windowserror(space, e)
-            if not _setCtrlHandlerRoutine(globalState.interrupt_event):
-                raise wrap_windowserror(
-                    space, rwin32.lastWindowsError("SetConsoleCtrlHandler"))
-
-    globalState = GlobalState()
-
-    class State:
-        def __init__(self, space):
-            self.main_thread = 0
-
-        def _cleanup_(self):
-            self.main_thread = 0
-            globalState.init()
-
-        def startup(self, space):
-            self.main_thread = thread.get_ident()
-            globalState.startup(space)
-
-        def get_interrupt_event(self):
-            return globalState.interrupt_event
-
-
-_includes = ["time.h"]
-if _POSIX:
-    _includes.append('sys/time.h')
-
-class CConfig:
-    _compilation_info_ = ExternalCompilationInfo(
-        includes = _includes
-    )
-    CLOCKS_PER_SEC = platform.ConstantInteger("CLOCKS_PER_SEC")
-    clock_t = platform.SimpleType("clock_t", rffi.ULONG)
-    has_gettimeofday = platform.Has('gettimeofday')
-
-if _POSIX:
-    calling_conv = 'c'
-    CConfig.timeval = platform.Struct("struct timeval",
-                                      [("tv_sec", rffi.INT),
-                                       ("tv_usec", rffi.INT)])
-    if _CYGWIN:
-        CConfig.tm = platform.Struct("struct tm", [("tm_sec", rffi.INT),
-            ("tm_min", rffi.INT), ("tm_hour", rffi.INT), ("tm_mday", rffi.INT),
-            ("tm_mon", rffi.INT), ("tm_year", rffi.INT), ("tm_wday", rffi.INT),
-            ("tm_yday", rffi.INT), ("tm_isdst", rffi.INT)])
-    else:
-        CConfig.tm = platform.Struct("struct tm", [("tm_sec", rffi.INT),
-            ("tm_min", rffi.INT), ("tm_hour", rffi.INT), ("tm_mday", rffi.INT),
-            ("tm_mon", rffi.INT), ("tm_year", rffi.INT), ("tm_wday", rffi.INT),
-            ("tm_yday", rffi.INT), ("tm_isdst", rffi.INT), ("tm_gmtoff", rffi.LONG),
-            ("tm_zone", rffi.CCHARP)])
-elif _WIN:
-    calling_conv = 'win'
-    CConfig.tm = platform.Struct("struct tm", [("tm_sec", rffi.INT),
-        ("tm_min", rffi.INT), ("tm_hour", rffi.INT), ("tm_mday", rffi.INT),
-        ("tm_mon", rffi.INT), ("tm_year", rffi.INT), ("tm_wday", rffi.INT),
-        ("tm_yday", rffi.INT), ("tm_isdst", rffi.INT)])
-
-class cConfig:
-    pass
-
-for k, v in platform.configure(CConfig).items():
-    setattr(cConfig, k, v)
-cConfig.tm.__name__ = "_tm"
-
-def external(name, args, result, eci=CConfig._compilation_info_):
-    if _WIN and rffi.sizeof(rffi.TIME_T) == 8:
-        # Recent Microsoft compilers use 64bit time_t and
-        # the corresponding functions are named differently
-        if (rffi.TIME_T in args or rffi.TIME_TP in args
-            or result in (rffi.TIME_T, rffi.TIME_TP)):
-            name = '_' + name + '64'
-    return rffi.llexternal(name, args, result,
-                           compilation_info=eci,
-                           calling_conv=calling_conv,
-                           releasegil=False)
-
-if _POSIX:
-    cConfig.timeval.__name__ = "_timeval"
-    timeval = cConfig.timeval
-
-CLOCKS_PER_SEC = cConfig.CLOCKS_PER_SEC
-clock_t = cConfig.clock_t
-tm = cConfig.tm
-glob_buf = lltype.malloc(tm, flavor='raw', zero=True, immortal=True)
-
-if cConfig.has_gettimeofday:
-    c_gettimeofday = external('gettimeofday', [rffi.VOIDP, rffi.VOIDP], rffi.INT)
-TM_P = lltype.Ptr(tm)
-c_clock = external('clock', [rffi.TIME_TP], clock_t)
-c_time = external('time', [rffi.TIME_TP], rffi.TIME_T)
-c_ctime = external('ctime', [rffi.TIME_TP], rffi.CCHARP)
-c_gmtime = external('gmtime', [rffi.TIME_TP], TM_P)
-c_mktime = external('mktime', [TM_P], rffi.TIME_T)
-c_asctime = external('asctime', [TM_P], rffi.CCHARP)
-c_localtime = external('localtime', [rffi.TIME_TP], TM_P)
-if _POSIX:
-    c_tzset = external('tzset', [], lltype.Void)
-if _WIN:
-    win_eci = ExternalCompilationInfo(
-        includes = ["time.h"],
-        post_include_bits = ["RPY_EXPORTED_FOR_TESTS\n"
-                             "long pypy_get_timezone();\n"
-                             "RPY_EXPORTED_FOR_TESTS\n"
-                             "int pypy_get_daylight();\n"
-                             "RPY_EXPORTED_FOR_TESTS\n"
-                             "char** pypy_get_tzname();\n"
-                             "RPY_EXPORTED_FOR_TESTS\n"
-                             "void* pypy__tzset();"],
-        separate_module_sources = ["""
-        long pypy_get_timezone() { return timezone; }
-        int pypy_get_daylight() { return daylight; }
-        char** pypy_get_tzname() { return tzname; }
-        void pypy__tzset() { return _tzset(); }
-        """])
-    # Ensure sure that we use _tzset() and timezone from the same C Runtime.
-    c_tzset = external('pypy__tzset', [], lltype.Void, win_eci)
-    c_get_timezone = external('pypy_get_timezone', [], rffi.LONG, win_eci)
-    c_get_daylight = external('pypy_get_daylight', [], rffi.INT, win_eci)
-    c_get_tzname = external('pypy_get_tzname', [], rffi.CCHARPP, win_eci)
-
-c_strftime = external('strftime', [rffi.CCHARP, rffi.SIZE_T, rffi.CCHARP, TM_P],
-                      rffi.SIZE_T)
-
-def _init_accept2dyear(space):
-    if os.environ.get("PYTHONY2K"):
-        accept2dyear = 0
-    else:
-        accept2dyear = 1
-    _set_module_object(space, "accept2dyear", space.wrap(accept2dyear))
-
-def _init_timezone(space):
-    timezone = daylight = altzone = 0
-    tzname = ["", ""]
-
-    if _WIN:
-        c_tzset()
-        timezone = c_get_timezone()
-        altzone = timezone - 3600
-        daylight = c_get_daylight()
-        tzname_ptr = c_get_tzname()
-        tzname = rffi.charp2str(tzname_ptr[0]), rffi.charp2str(tzname_ptr[1])
-
-    if _POSIX:
-        if _CYGWIN:
-            YEAR = (365 * 24 + 6) * 3600
-
-            # about January 11th
-            t = (((c_time(lltype.nullptr(rffi.TIME_TP.TO))) / YEAR) * YEAR + 10 * 24 * 3600)
-            # we cannot have reference to stack variable, put it on the heap
-            t_ref = lltype.malloc(rffi.TIME_TP.TO, 1, flavor='raw')
-            t_ref[0] = rffi.cast(rffi.TIME_T, t)
-            p = c_localtime(t_ref)
-            q = c_gmtime(t_ref)
-            janzone = (p.c_tm_hour + 24 * p.c_tm_mday) - (q.c_tm_hour + 24 * q.c_tm_mday)
-            if janzone < -12:
-                janname = "   "
-            elif janzone > 14:
-                janname = "   "
-            else:
-                janname = _time_zones[janzone - 12]
-            janzone = janzone * 3600
-            # about July 11th
-            tt = t + YEAR / 2
-            t_ref[0] = rffi.cast(rffi.TIME_T, tt)
-            p = c_localtime(t_ref)
-            q = c_gmtime(t_ref)
-            julyzone = (p.c_tm_hour + 24 * p.c_tm_mday) - (q.c_tm_hour + 24 * q.c_tm_mday)
-            if julyzone < -12:
-                julyname = "   "
-            elif julyzone > 14:
-                julyname = "   "
-            else:
-                julyname = _time_zones[julyzone - 12]
-            julyzone = julyzone * 3600
-            lltype.free(t_ref, flavor='raw')
-
-            if janzone < julyzone:
-                # DST is reversed in the southern hemisphere
-                timezone = julyzone
-                altzone = janzone
-                daylight = int(janzone != julyzone)
-                tzname = [julyname, janname]
-            else:
-                timezone = janzone
-                altzone = julyzone
-                daylight = int(janzone != julyzone)
-                tzname = [janname, julyname]
-
-        else:
-            YEAR = (365 * 24 + 6) * 3600
-
-            t = (((c_time(lltype.nullptr(rffi.TIME_TP.TO))) / YEAR) * YEAR)
-            # we cannot have reference to stack variable, put it on the heap
-            t_ref = lltype.malloc(rffi.TIME_TP.TO, 1, flavor='raw')
-            t_ref[0] = rffi.cast(rffi.TIME_T, t)
-            p = c_localtime(t_ref)
-            janzone = -p.c_tm_gmtoff
-            tm_zone = rffi.charp2str(p.c_tm_zone)
-            janname = ["   ", tm_zone][bool(tm_zone)]
-            tt = t + YEAR / 2
-            t_ref[0] = rffi.cast(rffi.TIME_T, tt)
-            p = c_localtime(t_ref)
-            lltype.free(t_ref, flavor='raw')
-            tm_zone = rffi.charp2str(p.c_tm_zone)
-            julyzone = -p.c_tm_gmtoff
-            julyname = ["   ", tm_zone][bool(tm_zone)]
-
-            if janzone < julyzone:
-                # DST is reversed in the southern hemisphere
-                timezone = julyzone
-                altzone = janzone
-                daylight = int(janzone != julyzone)
-                tzname = [julyname, janname]
-            else:
-                timezone = janzone
-                altzone = julyzone
-                daylight = int(janzone != julyzone)
-                tzname = [janname, julyname]
-
-    _set_module_object(space, "timezone", space.wrap(timezone))
-    _set_module_object(space, 'daylight', space.wrap(daylight))
-    tzname_w = [space.wrap(tzname[0]), space.wrap(tzname[1])]
-    _set_module_object(space, 'tzname', space.newtuple(tzname_w))
-    _set_module_object(space, 'altzone', space.wrap(altzone))
-
-def _get_error_msg():
-    errno = rposix.get_errno()
-    return os.strerror(errno)
-
-if sys.platform != 'win32':
-    @unwrap_spec(secs=float)
-    def sleep(space, secs):
-        if secs < 0:
-            raise OperationError(space.w_IOError,
-                                 space.wrap("Invalid argument: negative time in sleep"))
-        pytime.sleep(secs)
-else:
-    from rpython.rlib import rwin32
-    from errno import EINTR
-    def _simple_sleep(space, secs, interruptible):
-        if secs == 0.0 or not interruptible:
-            pytime.sleep(secs)
-        else:
-            millisecs = int(secs * 1000)
-            interrupt_event = space.fromcache(State).get_interrupt_event()
-            rwin32.ResetEvent(interrupt_event)
-            rc = rwin32.WaitForSingleObject(interrupt_event, millisecs)
-            if rc == rwin32.WAIT_OBJECT_0:
-                # Yield to make sure real Python signal handler
-                # called.
-                pytime.sleep(0.001)
-                raise wrap_oserror(space,
-                                   OSError(EINTR, "sleep() interrupted"))
-    @unwrap_spec(secs=float)
-    def sleep(space, secs):
-        if secs < 0:
-            raise OperationError(space.w_IOError,
-                                 space.wrap("Invalid argument: negative time in sleep"))
-        # as decreed by Guido, only the main thread can be
-        # interrupted.
-        main_thread = space.fromcache(State).main_thread
-        interruptible = (main_thread == thread.get_ident())
-        MAX = sys.maxint / 1000.0 # > 24 days
-        while secs > MAX:
-            _simple_sleep(space, MAX, interruptible)
-            secs -= MAX
-        _simple_sleep(space, secs, interruptible)
-
-def _get_module_object(space, obj_name):
-    w_module = space.getbuiltinmodule('time')
-    w_obj = space.getattr(w_module, space.wrap(obj_name))
-    return w_obj
-
-def _set_module_object(space, obj_name, w_obj_value):
-    w_module = space.getbuiltinmodule('time')
-    space.setattr(w_module, space.wrap(obj_name), w_obj_value)
-
-def _get_inttime(space, w_seconds):
-    # w_seconds can be a wrapped None (it will be automatically wrapped
-    # in the callers, so we never get a real None here).
-    if space.is_none(w_seconds):
-        seconds = pytime.time()
-    else:
-        seconds = space.float_w(w_seconds)
-    #
-    t = rffi.cast(rffi.TIME_T, seconds)
-    #
-    # Logic from CPython: How much info did we lose?  We assume that
-    # time_t is an integral type.  If we lost a second or more, the
-    # input doesn't fit in a time_t; call it an error.
-    diff = seconds - rffi.cast(lltype.Float, t)
-    if diff <= -1.0 or diff >= 1.0:
-        raise OperationError(space.w_ValueError,
-                      space.wrap("timestamp out of range for platform time_t"))
-    return t
-
-def _tm_to_tuple(space, t):
-    time_tuple = [
-        space.wrap(rffi.getintfield(t, 'c_tm_year') + 1900),
-        space.wrap(rffi.getintfield(t, 'c_tm_mon') + 1), # want january == 1
-        space.wrap(rffi.getintfield(t, 'c_tm_mday')),
-        space.wrap(rffi.getintfield(t, 'c_tm_hour')),
-        space.wrap(rffi.getintfield(t, 'c_tm_min')),
-        space.wrap(rffi.getintfield(t, 'c_tm_sec')),
-        space.wrap((rffi.getintfield(t, 'c_tm_wday') + 6) % 7), # want monday == 0
-        space.wrap(rffi.getintfield(t, 'c_tm_yday') + 1), # want january, 1 == 1
-        space.wrap(rffi.getintfield(t, 'c_tm_isdst'))]
-
-    w_struct_time = _get_module_object(space, 'struct_time')
-    w_time_tuple = space.newtuple(time_tuple)
-    return space.call_function(w_struct_time, w_time_tuple)
-
-def _gettmarg(space, w_tup, allowNone=True):
-    if space.is_none(w_tup):
-        if not allowNone:
-            raise OperationError(space.w_TypeError,
-                                 space.wrap("tuple expected"))
-        # default to the current local time
-        tt = rffi.r_time_t(int(pytime.time()))
-        t_ref = lltype.malloc(rffi.TIME_TP.TO, 1, flavor='raw')
-        t_ref[0] = tt
-        pbuf = c_localtime(t_ref)
-        lltype.free(t_ref, flavor='raw')
-        if not pbuf:
-            raise OperationError(space.w_ValueError,
-                space.wrap(_get_error_msg()))
-        return pbuf
-
-    tup_w = space.fixedview(w_tup)
-    if len(tup_w) != 9:
-        raise oefmt(space.w_TypeError,
-                    "argument must be sequence of length 9, not %d",
-                    len(tup_w))
-
-    y = space.int_w(tup_w[0])
-    tm_mon = space.int_w(tup_w[1])
-    if tm_mon == 0:
-        tm_mon = 1
-    tm_mday = space.int_w(tup_w[2])
-    if tm_mday == 0:
-        tm_mday = 1
-    tm_yday = space.int_w(tup_w[7])
-    if tm_yday == 0:
-        tm_yday = 1
-    rffi.setintfield(glob_buf, 'c_tm_mon', tm_mon)
-    rffi.setintfield(glob_buf, 'c_tm_mday', tm_mday)
-    rffi.setintfield(glob_buf, 'c_tm_hour', space.int_w(tup_w[3]))
-    rffi.setintfield(glob_buf, 'c_tm_min', space.int_w(tup_w[4]))
-    rffi.setintfield(glob_buf, 'c_tm_sec', space.int_w(tup_w[5]))
-    rffi.setintfield(glob_buf, 'c_tm_wday', space.int_w(tup_w[6]))
-    rffi.setintfield(glob_buf, 'c_tm_yday', tm_yday)
-    rffi.setintfield(glob_buf, 'c_tm_isdst', space.int_w(tup_w[8]))
-    if _POSIX:
-        if _CYGWIN:
-            pass
-        else:
-            # actually never happens, but makes annotator happy
-            glob_buf.c_tm_zone = lltype.nullptr(rffi.CCHARP.TO)
-            rffi.setintfield(glob_buf, 'c_tm_gmtoff', 0)
-
-    if y < 1900:
-        w_accept2dyear = _get_module_object(space, "accept2dyear")
-        accept2dyear = space.int_w(w_accept2dyear)
-
-        if not accept2dyear:
-            raise OperationError(space.w_ValueError,
-                space.wrap("year >= 1900 required"))
-
-        if 69 <= y <= 99:
-            y += 1900
-        elif 0 <= y <= 68:
-            y += 2000
-        else:
-            raise OperationError(space.w_ValueError,
-                space.wrap("year out of range"))
-
-    # tm_wday does not need checking of its upper-bound since taking "%
-    #  7" in gettmarg() automatically restricts the range.
-    if rffi.getintfield(glob_buf, 'c_tm_wday') < -1:
-        raise OperationError(space.w_ValueError,
-                             space.wrap("day of week out of range"))
-
-    rffi.setintfield(glob_buf, 'c_tm_year', y - 1900)
-    rffi.setintfield(glob_buf, 'c_tm_mon',
-                     rffi.getintfield(glob_buf, 'c_tm_mon') - 1)
-    rffi.setintfield(glob_buf, 'c_tm_wday',
-                     (rffi.getintfield(glob_buf, 'c_tm_wday') + 1) % 7)
-    rffi.setintfield(glob_buf, 'c_tm_yday',
-                     rffi.getintfield(glob_buf, 'c_tm_yday') - 1)
-
-    return glob_buf
-
-def time(space):
-    """time() -> floating point number
-
-    Return the current time in seconds since the Epoch.
-    Fractions of a second may be present if the system clock provides them."""
-
-    secs = pytime.time()
-    return space.wrap(secs)
-
-if _WIN:
-    class PCCache:
-        pass
-    pccache = PCCache()
-    pccache.divisor = 0.0
-    pccache.ctrStart = 0
-
-def clock(space):
-    """clock() -> floating point number
-
-    Return the CPU time or real time since the start of the process or since
-    the first call to clock().  This has as much precision as the system
-    records."""
-
-    return space.wrap(pytime.clock())
-
-def ctime(space, w_seconds=None):
-    """ctime([seconds]) -> string
-
-    Convert a time in seconds since the Epoch to a string in local time.
-    This is equivalent to asctime(localtime(seconds)). When the time tuple is
-    not present, current time as returned by localtime() is used."""
-
-    seconds = _get_inttime(space, w_seconds)
-
-    t_ref = lltype.malloc(rffi.TIME_TP.TO, 1, flavor='raw')
-    t_ref[0] = seconds
-    p = c_ctime(t_ref)
-    lltype.free(t_ref, flavor='raw')
-    if not p:
-        raise OperationError(space.w_ValueError,
-            space.wrap("unconvertible time"))
-
-    return space.wrap(rffi.charp2str(p)[:-1]) # get rid of new line
-
-# by now w_tup is an optional argument (and not *args)
-# because of the ext. compiler bugs in handling such arguments (*args, **kwds)
-def asctime(space, w_tup=None):
-    """asctime([tuple]) -> string
-
-    Convert a time tuple to a string, e.g. 'Sat Jun 06 16:26:11 1998'.
-    When the time tuple is not present, current time as returned by localtime()
-    is used."""
-    buf_value = _gettmarg(space, w_tup)
-    p = c_asctime(buf_value)
-    if not p:
-        raise OperationError(space.w_ValueError,
-            space.wrap("unconvertible time"))
-
-    return space.wrap(rffi.charp2str(p)[:-1]) # get rid of new line
-
-def gmtime(space, w_seconds=None):
-    """gmtime([seconds]) -> (tm_year, tm_mon, tm_day, tm_hour, tm_min,
-                          tm_sec, tm_wday, tm_yday, tm_isdst)
-
-    Convert seconds since the Epoch to a time tuple expressing UTC (a.k.a.
-    GMT).  When 'seconds' is not passed in, convert the current time instead.
-    """
-
-    # rpython does not support that a variable has two incompatible builtins
-    # as value so we have to duplicate the code. NOT GOOD! see localtime() too
-    seconds = _get_inttime(space, w_seconds)
-    t_ref = lltype.malloc(rffi.TIME_TP.TO, 1, flavor='raw')
-    t_ref[0] = seconds
-    p = c_gmtime(t_ref)
-    lltype.free(t_ref, flavor='raw')
-
-    if not p:
-        raise OperationError(space.w_ValueError, space.wrap(_get_error_msg()))
-    return _tm_to_tuple(space, p)
-
-def localtime(space, w_seconds=None):
-    """localtime([seconds]) -> (tm_year, tm_mon, tm_day, tm_hour, tm_min,
-                             tm_sec, tm_wday, tm_yday, tm_isdst)
-
-    Convert seconds since the Epoch to a time tuple expressing local time.
-    When 'seconds' is not passed in, convert the current time instead."""
-
-    seconds = _get_inttime(space, w_seconds)
-    t_ref = lltype.malloc(rffi.TIME_TP.TO, 1, flavor='raw')
-    t_ref[0] = seconds
-    p = c_localtime(t_ref)
-    lltype.free(t_ref, flavor='raw')
-
-    if not p:
-        raise OperationError(space.w_ValueError, space.wrap(_get_error_msg()))
-    return _tm_to_tuple(space, p)
-
-def mktime(space, w_tup):
-    """mktime(tuple) -> floating point number
-
-    Convert a time tuple in local time to seconds since the Epoch."""
-
-    buf = _gettmarg(space, w_tup, allowNone=False)
-    rffi.setintfield(buf, "c_tm_wday", -1)
-    tt = c_mktime(buf)
-    # A return value of -1 does not necessarily mean an error, but tm_wday
-    # cannot remain set to -1 if mktime succeeds.
-    if tt == -1 and rffi.getintfield(buf, "c_tm_wday") == -1:
-        raise OperationError(space.w_OverflowError,
-            space.wrap("mktime argument out of range"))
-
-    return space.wrap(float(tt))
-
-if _POSIX:
-    def tzset(space):
-        """tzset()
-
-        Initialize, or reinitialize, the local timezone to the value stored in
-        os.environ['TZ']. The TZ environment variable should be specified in
-        standard Unix timezone format as documented in the tzset man page
-        (eg. 'US/Eastern', 'Europe/Amsterdam'). Unknown timezones will silently
-        fall back to UTC. If the TZ environment variable is not set, the local
-        timezone is set to the systems best guess of wallclock time.
-        Changing the TZ environment variable without calling tzset *may* change
-        the local timezone used by methods such as localtime, but this behaviour
-        should not be relied on"""
-
-        c_tzset()
-
-        # reset timezone, altzone, daylight and tzname
-        _init_timezone(space)
-
- at unwrap_spec(format=str)
-def strftime(space, format, w_tup=None):
-    """strftime(format[, tuple]) -> string
-
-    Convert a time tuple to a string according to a format specification.
-    See the library reference manual for formatting codes. When the time tuple
-    is not present, current time as returned by localtime() is used."""
-    buf_value = _gettmarg(space, w_tup)
-
-    # Checks added to make sure strftime() does not crash Python by
-    # indexing blindly into some array for a textual representation
-    # by some bad index (fixes bug #897625).
-    # No check for year since handled in gettmarg().
-    if rffi.getintfield(buf_value, 'c_tm_mon') < 0 or rffi.getintfield(buf_value, 'c_tm_mon') > 11:
-        raise OperationError(space.w_ValueError,
-                             space.wrap("month out of range"))
-    if rffi.getintfield(buf_value, 'c_tm_mday') < 1 or rffi.getintfield(buf_value, 'c_tm_mday') > 31:
-        raise OperationError(space.w_ValueError,
-                             space.wrap("day of month out of range"))
-    if rffi.getintfield(buf_value, 'c_tm_hour') < 0 or rffi.getintfield(buf_value, 'c_tm_hour') > 23:
-        raise OperationError(space.w_ValueError,
-                             space.wrap("hour out of range"))
-    if rffi.getintfield(buf_value, 'c_tm_min') < 0 or rffi.getintfield(buf_value, 'c_tm_min') > 59:
-        raise OperationError(space.w_ValueError,
-                             space.wrap("minute out of range"))
-    if rffi.getintfield(buf_value, 'c_tm_sec') < 0 or rffi.getintfield(buf_value, 'c_tm_sec') > 61:
-        raise OperationError(space.w_ValueError,
-                             space.wrap("seconds out of range"))
-    if rffi.getintfield(buf_value, 'c_tm_yday') < 0 or rffi.getintfield(buf_value, 'c_tm_yday') > 365:
-        raise OperationError(space.w_ValueError,
-                             space.wrap("day of year out of range"))
-    if rffi.getintfield(buf_value, 'c_tm_isdst') < -1 or rffi.getintfield(buf_value, 'c_tm_isdst') > 1:
-        raise OperationError(space.w_ValueError,
-                             space.wrap("daylight savings flag out of range"))
-
-    if _WIN:
-        # check that the format string contains only valid directives
-        length = len(format)
-        i = 0
-        while i < length:
-            if format[i] == '%':
-                i += 1
-                if i < length and format[i] == '#':
-                    # not documented by python
-                    i += 1
-                if i >= length or format[i] not in "aAbBcdHIjmMpSUwWxXyYzZ%":
-                    raise OperationError(space.w_ValueError,
-                                         space.wrap("invalid format string"))
-            i += 1
-
-    i = 1024
-    while True:
-        outbuf = lltype.malloc(rffi.CCHARP.TO, i, flavor='raw')
-        try:
-            buflen = c_strftime(outbuf, i, format, buf_value)
-            if buflen > 0 or i >= 256 * len(format):
-                # if the buffer is 256 times as long as the format,
-                # it's probably not failing for lack of room!
-                # More likely, the format yields an empty result,
-                # e.g. an empty format, or %Z when the timezone
-                # is unknown.
-                result = rffi.charp2strn(outbuf, intmask(buflen))
-                return space.wrap(result)
-        finally:
-            lltype.free(outbuf, flavor='raw')
-        i += i
diff --git a/pypy/module/rctime/test/test_rctime.py b/pypy/module/rctime/test/test_rctime.py
deleted file mode 100644
--- a/pypy/module/rctime/test/test_rctime.py
+++ /dev/null
@@ -1,333 +0,0 @@
-class AppTestRCTime:
-    spaceconfig = {
-        "usemodules": ['rctime', 'struct', 'binascii'],
-    }
-
-    def test_attributes(self):
-        import time as rctime
-        assert isinstance(rctime.accept2dyear, int)
-        assert isinstance(rctime.altzone, int)
-        assert isinstance(rctime.daylight, int)
-        assert isinstance(rctime.timezone, int)
-        assert isinstance(rctime.tzname, tuple)
-        assert isinstance(rctime.__doc__, str)
-
-    def test_sleep(self):
-        import time as rctime
-        import sys
-        import os
-        raises(TypeError, rctime.sleep, "foo")
-        rctime.sleep(0.12345)
-        raises(IOError, rctime.sleep, -1.0)
-
-    def test_clock(self):
-        import time as rctime
-        rctime.clock()
-        assert isinstance(rctime.clock(), float)
-
-    def test_time(self):
-        import time as rctime
-        t1 = rctime.time()
-        assert isinstance(rctime.time(), float)
-        assert rctime.time() != 0.0 # 0.0 means failure
-        rctime.sleep(0.02)
-        t2 = rctime.time()
-        assert t1 != t2       # the resolution should be at least 0.01 secs
-
-    def test_ctime(self):
-        import time as rctime
-        raises(TypeError, rctime.ctime, "foo")
-        rctime.ctime(None)
-        rctime.ctime()
-        res = rctime.ctime(0)
-        assert isinstance(res, str)
-        rctime.ctime(rctime.time())
-        raises(ValueError, rctime.ctime, 1E200)
-        raises(OverflowError, rctime.ctime, 10**900)
-
-    def test_gmtime(self):
-        import time as rctime
-        raises(TypeError, rctime.gmtime, "foo")
-        rctime.gmtime()
-        rctime.gmtime(None)
-        rctime.gmtime(0)
-        res = rctime.gmtime(rctime.time())
-        assert isinstance(res, rctime.struct_time)
-        assert res[-1] == 0 # DST is always zero in gmtime()
-        t0 = rctime.mktime(rctime.gmtime())
-        t1 = rctime.mktime(rctime.gmtime(None))
-        assert 0 <= (t1 - t0) < 1.2
-        t = rctime.time()
-        assert rctime.gmtime(t) == rctime.gmtime(t)
-        raises(ValueError, rctime.gmtime, 2**64)
-        raises(ValueError, rctime.gmtime, -2**64)
-
-    def test_localtime(self):
-        import time as rctime
-        import os
-        raises(TypeError, rctime.localtime, "foo")
-        rctime.localtime()
-        rctime.localtime(None)
-        rctime.localtime(0)
-        res = rctime.localtime(rctime.time())
-        assert isinstance(res, rctime.struct_time)
-        t0 = rctime.mktime(rctime.localtime())
-        t1 = rctime.mktime(rctime.localtime(None))
-        assert 0 <= (t1 - t0) < 1.2
-        t = rctime.time()
-        assert rctime.localtime(t) == rctime.localtime(t)
-        if os.name == 'nt':

From noreply at buildbot.pypy.org  Tue Nov 18 23:45:26 2014
From: noreply at buildbot.pypy.org (mattip)
Date: Tue, 18 Nov 2014 23:45:26 +0100 (CET)
Subject: [pypy-commit] pypy ufuncapi: refactor and use coalescing to pass a
	test
Message-ID: <20141118224526.3ADBE1C1147@cobra.cs.uni-duesseldorf.de>

Author: mattip 
Branch: ufuncapi
Changeset: r74576:27f4ddb2db05
Date: 2014-11-19 00:22 +0200
http://bitbucket.org/pypy/pypy/changeset/27f4ddb2db05/

Log:	refactor and use coalescing to pass a test

diff --git a/pypy/module/micronumpy/iterators.py b/pypy/module/micronumpy/iterators.py
--- a/pypy/module/micronumpy/iterators.py
+++ b/pypy/module/micronumpy/iterators.py
@@ -117,11 +117,9 @@
         self.shape_m1 = [s - 1 for s in shape]
         self.strides = strides
         self.backstrides = backstrides
-        self.slice_shape = 1
-        self.slice_stride = -1
-        if strides:
-            self.slice_stride = strides[-1]
-        self.slice_backstride = 1
+        self.slice_shape = []
+        self.slice_stride = []
+        self.slice_backstride = []
         self.slice_operand_type = concrete.SliceArray
 
         ndim = len(shape)
@@ -280,6 +278,6 @@
     def getoperand(self, state, base):
         assert state.iterator is self
         impl = self.slice_operand_type
-        arr = impl(state.offset, [self.slice_stride], [self.slice_backstride],
-                   [self.slice_shape], self.array, self.base)
+        arr = impl(state.offset, self.slice_stride, self.slice_backstride,
+                   self.slice_shape, self.array, self.base)
         return arr
diff --git a/pypy/module/micronumpy/nditer.py b/pypy/module/micronumpy/nditer.py
--- a/pypy/module/micronumpy/nditer.py
+++ b/pypy/module/micronumpy/nditer.py
@@ -200,27 +200,8 @@
                 break
         if can_coalesce:
             for i in range(len(it.iters)):
-                old_iter = it.iters[i][0]
-                shape = [s+1 for s in old_iter.shape_m1]
-                strides = old_iter.strides
-                backstrides = old_iter.backstrides
-                if it.order == 'F':
-                    new_shape = shape[1:]
-                    new_strides = strides[1:]
-                    new_backstrides = backstrides[1:]
-                    _stride = min(strides[0], old_iter.slice_stride)
-                else:
-                    new_shape = shape[:-1]
-                    new_strides = strides[:-1]
-                    new_backstrides = backstrides[:-1]
-                    _stride = old_iter.slice_stride
-                # We always want the "fastest" iterator in external loops
-                _shape = shape[fastest] * old_iter.slice_shape
-                _backstride = (_shape - 1) * _stride
-                new_iter = SliceIter(old_iter.array, old_iter.size / shape[fastest],
-                            new_shape, new_strides, new_backstrides,
-                            _shape, _stride, _backstride,
-                            it.op_flags[i], it)
+                new_iter = coalesce_iter(it.iters[i][0], it.op_flags[i], it,
+                                         it.order, fastest)
                 it.iters[i] = (new_iter, new_iter.reset())
             if len(it.shape) > 1:
                 if it.order == 'F':
@@ -229,26 +210,12 @@
                     it.shape = it.shape[:-1]
             else:
                 it.shape = [1]
+
         else:
             break
     # Always coalesce at least one
     for i in range(len(it.iters)):
-        old_iter = it.iters[i][0]
-        shape = [s+1 for s in old_iter.shape_m1]
-        strides = old_iter.strides
-        backstrides = old_iter.backstrides
-        new_shape = shape[:-1]
-        new_strides = strides[:-1]
-        new_backstrides = backstrides[:-1]
-        _shape = shape[-1] * old_iter.slice_shape
-        # use the operand's iterator's rightmost stride,
-        # even if it is not the fastest (for 'F' or swapped axis)
-        _stride = old_iter.slice_stride
-        _backstride = (_shape - 1) * _stride
-        new_iter = SliceIter(old_iter.array, old_iter.size / shape[-1],
-                    new_shape, new_strides, new_backstrides,
-                    _shape, _stride, _backstride,
-                    it.op_flags[i], it)
+        new_iter = coalesce_iter(it.iters[i][0], it.op_flags[i], it, 'C', -1)
         it.iters[i] = (new_iter, new_iter.reset())
     if len(it.shape) > 1:
         if it.order == 'F':
@@ -258,6 +225,43 @@
     else:
         it.shape = [1]
 
+
+def coalesce_iter(old_iter, op_flags, it, order, fastest=-1, flat=True):
+    '''
+    We usually iterate through an array one value at a time.
+    But after coalesce(), getoperand() will return a slice by removing
+    the fastest varying dimension from the beginning or end of the shape.
+    XXX - what happens on swapaxis arrays?
+    If flat is true, then the slice will be 1d, otherwise stack up the shape of
+    the fastest varying dimension in the slice, so an iterator of a  'C' array 
+    of shape (2,4,3) after two calls to coalesce will iterate 2 times over a slice
+    of shape (4,3) by setting the offset to the beginning of the data at each iteration
+    '''
+    shape = [s+1 for s in old_iter.shape_m1]
+    strides = old_iter.strides
+    backstrides = old_iter.backstrides
+    if order == 'F':
+        new_shape = shape[1:]
+        new_strides = strides[1:]
+        new_backstrides = backstrides[1:]
+        _stride = old_iter.slice_stride + [strides[0]]
+    else:
+        new_shape = shape[:-1]
+        new_strides = strides[:-1]
+        new_backstrides = backstrides[:-1]
+        # use the operand's iterator's rightmost stride,
+        # even if it is not the fastest (for 'F' or swapped axis)
+        _stride = [strides[-1]] + old_iter.slice_stride
+    _shape = [shape[fastest]]  + old_iter.slice_shape
+    _backstride = [(_shape[fastest] - 1) * _stride[0]] + old_iter.slice_backstride
+    if flat:
+        _shape = [support.product(_shape)]
+        _stride = [min(_stride)]
+        _backstride = [(shape[0] - 1) * _stride[0]]
+    return SliceIter(old_iter.array, old_iter.size / shape[fastest],
+                new_shape, new_strides, new_backstrides,
+                _shape, _stride, _backstride, op_flags, it)
+
 class IndexIterator(object):
     def __init__(self, shape, backward=False):
         self.shape = shape
diff --git a/pypy/module/micronumpy/ufuncs.py b/pypy/module/micronumpy/ufuncs.py
--- a/pypy/module/micronumpy/ufuncs.py
+++ b/pypy/module/micronumpy/ufuncs.py
@@ -9,7 +9,7 @@
 from pypy.module.micronumpy import boxes, descriptor, loop, constants as NPY
 from pypy.module.micronumpy.base import convert_to_array, W_NDimArray
 from pypy.module.micronumpy.ctors import numpify
-from pypy.module.micronumpy.nditer import W_NDIter
+from pypy.module.micronumpy.nditer import W_NDIter, coalesce_iter
 from pypy.module.micronumpy.strides import shape_agreement
 from pypy.module.micronumpy.support import _parse_signature
 from rpython.rlib.rawstorage import (raw_storage_setitem, free_raw_storage,
@@ -698,7 +698,7 @@
 
         # TODO parse and handle subok
         # TODO handle flags, op_flags
-        w_flags = space.w_None #space.newlist([space.wrap('external_loop')])
+        w_flags = space.w_None # NOT 'external_loop', we do coalescing by core_num_dims
         w_op_flags = space.newtuple([space.wrap(['readonly'])] * len(inargs) + \
                                     [space.wrap(['readwrite'])] * len(outargs))
         w_op_dtypes = space.w_None
@@ -706,14 +706,25 @@
         w_itershape = space.newlist([space.wrap(i) for i in iter_shape]) 
         w_op_axes = space.w_None
 
-        # mimic NpyIter_AdvancedNew with a nditer
 
         if self.stack_inputs:
+            # mimic NpyIter_AdvancedNew with a nditer
             nd_it = W_NDIter(space, space.newlist(inargs + outargs), w_flags,
                           w_op_flags, w_op_dtypes, w_casting, w_op_axes,
                           w_itershape)
-            # XXX coalesce each iterators, according to inner_dimensions
+            # coalesce each iterators, according to inner_dimensions
+            if nd_it.order == 'F':
+                fastest = 0
+            else:
+                fastest = -1
+            for i in range(len(inargs) + len(outargs)):
+                for j in range(self.core_num_dims[i]):
+                    new_iter = coalesce_iter(nd_it.iters[i][0], nd_it.op_flags[i],
+                                    nd_it, nd_it.order, fastest, flat=False)
+                    nd_it.iters[i] = (new_iter, new_iter.reset())
+            # do the iteration
             while not nd_it.done:
+                # XXX jit me
                 for it, st in nd_it.iters:
                     if not it.done(st):
                         break

From noreply at buildbot.pypy.org  Tue Nov 18 23:45:27 2014
From: noreply at buildbot.pypy.org (mattip)
Date: Tue, 18 Nov 2014 23:45:27 +0100 (CET)
Subject: [pypy-commit] pypy ufuncapi: fix test_ztranslation
Message-ID: <20141118224527.785BF1C1147@cobra.cs.uni-duesseldorf.de>

Author: mattip 
Branch: ufuncapi
Changeset: r74577:53935a16ebc8
Date: 2014-11-19 00:44 +0200
http://bitbucket.org/pypy/pypy/changeset/53935a16ebc8/

Log:	fix test_ztranslation

diff --git a/pypy/module/micronumpy/iterators.py b/pypy/module/micronumpy/iterators.py
--- a/pypy/module/micronumpy/iterators.py
+++ b/pypy/module/micronumpy/iterators.py
@@ -99,7 +99,7 @@
 class ArrayIter(object):
     _immutable_fields_ = ['contiguous', 'array', 'size', 'ndim_m1', 'shape_m1[*]',
                           'strides[*]', 'backstrides[*]', 'factors[*]',
-                          'slice_shape', 'slice_stride', 'slice_backstride',
+                          'slice_shape[*]', 'slice_stride[*]', 'slice_backstride[*]',
                           'track_index', 'operand_type', 'slice_operand_type']
 
     track_index = True
@@ -248,7 +248,7 @@
     used with external loops, getitem and setitem return a SliceArray
     view into the original array
     '''
-    _immutable_fields_ = ['base', 'slice_shape[*]', 'slice_stride[*]', 'slice_backstride[*]']
+    _immutable_fields_ = ['base']
 
     def __init__(self, array, size, shape, strides, backstrides, slice_shape,
                  slice_stride, slice_backstride, op_flags, base):
diff --git a/pypy/module/micronumpy/nditer.py b/pypy/module/micronumpy/nditer.py
--- a/pypy/module/micronumpy/nditer.py
+++ b/pypy/module/micronumpy/nditer.py
@@ -256,7 +256,8 @@
     _backstride = [(_shape[fastest] - 1) * _stride[0]] + old_iter.slice_backstride
     if flat:
         _shape = [support.product(_shape)]
-        _stride = [min(_stride)]
+        assert len(_stride) == 2
+        _stride = [min(_stride[0], _stride[1])]
         _backstride = [(shape[0] - 1) * _stride[0]]
     return SliceIter(old_iter.array, old_iter.size / shape[fastest],
                 new_shape, new_strides, new_backstrides,
diff --git a/pypy/module/micronumpy/ufuncs.py b/pypy/module/micronumpy/ufuncs.py
--- a/pypy/module/micronumpy/ufuncs.py
+++ b/pypy/module/micronumpy/ufuncs.py
@@ -699,8 +699,8 @@
         # TODO parse and handle subok
         # TODO handle flags, op_flags
         w_flags = space.w_None # NOT 'external_loop', we do coalescing by core_num_dims
-        w_op_flags = space.newtuple([space.wrap(['readonly'])] * len(inargs) + \
-                                    [space.wrap(['readwrite'])] * len(outargs))
+        w_op_flags = space.newtuple([space.wrap(r) for r in ['readonly'] * len(inargs)] + \
+                                    [space.wrap(r) for r in ['readwrite'] * len(outargs)])
         w_op_dtypes = space.w_None
         w_casting = space.w_None
         w_itershape = space.newlist([space.wrap(i) for i in iter_shape]) 

From noreply at buildbot.pypy.org  Wed Nov 19 00:22:02 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Wed, 19 Nov 2014 00:22:02 +0100 (CET)
Subject: [pypy-commit] pypy stmgc-c7: Tweaks to reduce the diff from default
Message-ID: <20141118232202.8B4DB1D2487@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: stmgc-c7
Changeset: r74578:d4f40d9d3bff
Date: 2014-11-15 11:48 +0100
http://bitbucket.org/pypy/pypy/changeset/d4f40d9d3bff/

Log:	Tweaks to reduce the diff from default

diff --git a/rpython/rlib/rthread.py b/rpython/rlib/rthread.py
--- a/rpython/rlib/rthread.py
+++ b/rpython/rlib/rthread.py
@@ -79,9 +79,6 @@
 
 
 def allocate_lock():
-    # Add some memory pressure for the size of the lock because it is an
-    # Opaque object
-    rgc.add_memory_pressure(TLOCKP_SIZE)
     return Lock(allocate_ll_lock())
 
 @specialize.arg(0)
@@ -139,9 +136,6 @@
         self._lock = ll_lock
 
     def acquire(self, flag):
-        return self._acquire(flag)    # indirection for the STMLock subclass
-
-    def _acquire(self, flag):
         res = c_thread_acquirelock(self._lock, int(flag))
         res = rffi.cast(lltype.Signed, res)
         return bool(res)
@@ -155,7 +149,7 @@
 
     def release(self):
         # Sanity check: the lock must be locked
-        if self._acquire(False):
+        if self.acquire(False):
             c_thread_releaselock(self._lock)
             raise error("bad lock")
         else:
@@ -204,6 +198,9 @@
     if rffi.cast(lltype.Signed, res) <= 0:
         lltype.free(ll_lock, flavor='raw', track_allocation=False)
         raise error("out of resources")
+    # Add some memory pressure for the size of the lock because it is an
+    # Opaque object
+    rgc.add_memory_pressure(TLOCKP_SIZE)
     return ll_lock
 
 def free_ll_lock(ll_lock):
@@ -322,8 +319,8 @@
                 from rpython.rlib.objectmodel import running_on_llinterp
                 ptr = cast_instance_to_base_ptr(value)
                 if not running_on_llinterp:
-                    if ptr:
-                        gcref = lltype.cast_opaque_ptr(llmemory.GCREF, ptr)
+                    gcref = lltype.cast_opaque_ptr(llmemory.GCREF, ptr)
+                    if gcref:
                         _make_sure_does_not_move(gcref)
                 llop.threadlocalref_set(lltype.Void, opaque_id, ptr)
                 ensure_threadlocal()

From noreply at buildbot.pypy.org  Wed Nov 19 00:22:10 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Wed, 19 Nov 2014 00:22:10 +0100 (CET)
Subject: [pypy-commit] pypy stmgc-c7: hg merge default
Message-ID: <20141118232210.DC4C81D2487@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: stmgc-c7
Changeset: r74579:3032699725ae
Date: 2014-11-19 00:13 +0100
http://bitbucket.org/pypy/pypy/changeset/3032699725ae/

Log:	hg merge default

diff too long, truncating to 2000 out of 45831 lines

diff --git a/.hgignore b/.hgignore
--- a/.hgignore
+++ b/.hgignore
@@ -89,6 +89,7 @@
 ^pypy/doc/image/lattice3\.png$
 ^pypy/doc/image/stackless_informal\.png$
 ^pypy/doc/image/parsing_example.+\.png$
+^rpython/doc/_build/.*$
 ^pypy/module/test_lib_pypy/ctypes_tests/_ctypes_test\.o$
 ^compiled
 ^.git/
diff --git a/README.rst b/README.rst
--- a/README.rst
+++ b/README.rst
@@ -23,6 +23,7 @@
 
     the pypy-dev team 
 
+
 Building
 ========
 
@@ -36,4 +37,4 @@
 to use virtualenv with the resulting pypy-c as the interpreter; you can
 find more details about various installation schemes here:
 
-http://doc.pypy.org/en/latest/getting-started.html#installing-pypy
+    http://doc.pypy.org/en/latest/install.html
diff --git a/ctypes_configure/doc/configure.txt b/ctypes_configure/doc/configure.txt
--- a/ctypes_configure/doc/configure.txt
+++ b/ctypes_configure/doc/configure.txt
@@ -19,6 +19,4 @@
 usage
 =====
 
-`sample.py`_ explains in details how to use it.
-
-.. _`sample.py`: http://codespeak.net/svn/pypy/dist/ctypes_configure/doc/sample.py
+:source:`sample.py ` explains in details how to use it.
diff --git a/lib-python/conftest.py b/lib-python/conftest.py
--- a/lib-python/conftest.py
+++ b/lib-python/conftest.py
@@ -59,7 +59,7 @@
     def __init__(self, basename, core=False, compiler=None, usemodules='',
                  skip=None):
         self.basename = basename
-        self._usemodules = usemodules.split() + ['signal', 'rctime', 'itertools', '_socket']
+        self._usemodules = usemodules.split() + ['signal', 'time', 'itertools', '_socket']
         self._compiler = compiler
         self.core = core
         self.skip = skip
diff --git a/lib_pypy/grp.py b/lib_pypy/grp.py
--- a/lib_pypy/grp.py
+++ b/lib_pypy/grp.py
@@ -66,11 +66,12 @@
 
 @builtinify
 def getgrnam(name):
-    if not isinstance(name, str):
+    if not isinstance(name, basestring):
         raise TypeError("expected string")
+    name = str(name)
     res = libc.getgrnam(name)
     if not res:
-        raise KeyError(name)
+        raise KeyError("'getgrnam(): name not found: %s'" % name)
     return _group_from_gstruct(res)
 
 @builtinify
diff --git a/pypy/config/pypyoption.py b/pypy/config/pypyoption.py
--- a/pypy/config/pypyoption.py
+++ b/pypy/config/pypyoption.py
@@ -29,7 +29,7 @@
 # --allworkingmodules
 working_modules = default_modules.copy()
 working_modules.update([
-    "_socket", "unicodedata", "mmap", "fcntl", "_locale", "pwd", "rctime" ,
+    "_socket", "unicodedata", "mmap", "fcntl", "_locale", "pwd", "time" ,
     "select", "zipimport", "_lsprof", "crypt", "signal", "_rawffi", "termios",
     "zlib", "bz2", "struct", "_hashlib", "_md5", "_sha", "_minimal_curses",
     "cStringIO", "thread", "itertools", "pyexpat", "_ssl", "cpyext", "array",
@@ -40,7 +40,7 @@
 
 translation_modules = default_modules.copy()
 translation_modules.update([
-    "fcntl", "rctime", "select", "signal", "_rawffi", "zlib", "struct", "_md5",
+    "fcntl", "time", "select", "signal", "_rawffi", "zlib", "struct", "_md5",
     "cStringIO", "array", "binascii",
     # the following are needed for pyrepl (and hence for the
     # interactive prompt/pdb)
@@ -64,19 +64,15 @@
     default_modules.add("_locale")
 
 if sys.platform == "sunos5":
-    working_modules.remove('mmap')   # depend on ctypes, can't get at c-level 'errono'
-    working_modules.remove('rctime') # depend on ctypes, missing tm_zone/tm_gmtoff
-    working_modules.remove('signal') # depend on ctypes, can't get at c-level 'errono'
     working_modules.remove('fcntl')  # LOCK_NB not defined
     working_modules.remove("_minimal_curses")
     working_modules.remove("termios")
-    working_modules.remove("_multiprocessing")   # depends on rctime
     if "cppyy" in working_modules:
         working_modules.remove("cppyy")  # depends on ctypes
 
 
 module_dependencies = {
-    '_multiprocessing': [('objspace.usemodules.rctime', True),
+    '_multiprocessing': [('objspace.usemodules.time', True),
                          ('objspace.usemodules.thread', True)],
     'cpyext': [('objspace.usemodules.array', True)],
     'cppyy': [('objspace.usemodules.cpyext', True)],
@@ -86,9 +82,10 @@
     # itself needs the interp-level struct module
     # because 'P' is missing from the app-level one
     "_rawffi": [("objspace.usemodules.struct", True)],
-    "cpyext": [("translation.secondaryentrypoints", "cpyext,main"),
-               ("translation.shared", sys.platform == "win32")],
+    "cpyext": [("translation.secondaryentrypoints", "cpyext,main")],
 }
+if sys.platform == "win32":
+    module_suggests["cpyext"].append(("translation.shared", True))
 
 module_import_dependencies = {
     # no _rawffi if importing rpython.rlib.clibffi raises ImportError
@@ -255,10 +252,6 @@
         BoolOption("optimized_list_getitem",
                    "special case the 'list[integer]' expressions",
                    default=False),
-        BoolOption("builtinshortcut",
-                   "a shortcut for operations between built-in types. XXX: "
-                   "deprecated, not really a shortcut any more.",
-                   default=False),
         BoolOption("getattributeshortcut",
                    "track types that override __getattribute__",
                    default=False,
@@ -270,9 +263,6 @@
                    # weakrefs needed, because of get_subclasses()
                    requires=[("translation.rweakref", True)]),
 
-        ChoiceOption("multimethods", "the multimethod implementation to use",
-                     ["doubledispatch", "mrd"],
-                     default="mrd"),
         BoolOption("withidentitydict",
                    "track types that override __hash__, __eq__ or __cmp__ and use a special dict strategy for those which do not",
                    default=False,
diff --git a/pypy/config/test/test_pypyoption.py b/pypy/config/test/test_pypyoption.py
--- a/pypy/config/test/test_pypyoption.py
+++ b/pypy/config/test/test_pypyoption.py
@@ -64,7 +64,7 @@
     def check_file_exists(fn):
         assert configdocdir.join(fn).check()
 
-    from pypy.doc.config.confrest import all_optiondescrs
+    from pypy.doc.config.generate import all_optiondescrs
     configdocdir = thisdir.dirpath().dirpath().join("doc", "config")
     for descr in all_optiondescrs:
         prefix = descr._name
diff --git a/pypy/doc/TODO b/pypy/doc/TODO
new file mode 100644
--- /dev/null
+++ b/pypy/doc/TODO
@@ -0,0 +1,40 @@
+Documentation TODO
+==================
+
+General
+-------
+
+* architecture documents don't really show the separation between PyPy and
+  RPython
+  * architecture.rst is duplicate (both pypy and rpython)
+* Consider moving information from pypy/doc/{build,windows}.rst to rpython/doc
+
+
+Cleanup
+~~~~~~~
+
+* remove documentation on removed features
+  * various object spaces
+* update / remove dead links
+
+
+Meta
+~~~~
+
+* work on configuration/options documentation generation
+
+
+PyPy
+----
+
+* Update coding guide
+* Move links from project-documentation.rst to index.rst and consider killing
+  it.
+
+
+RPython
+-------
+
+* make translation.rst a high-level overview and move details in their own
+  documents
+* redo various outdated pictures in translation.rst
diff --git a/pypy/doc/__pypy__-module.rst b/pypy/doc/__pypy__-module.rst
--- a/pypy/doc/__pypy__-module.rst
+++ b/pypy/doc/__pypy__-module.rst
@@ -1,20 +1,17 @@
-
 .. comment: this document is very incomplete, should we generate
             it automatically?
 
-=======================
 The ``__pypy__`` module
 =======================
 
 The ``__pypy__`` module is the main entry point to special features provided
-by PyPy's standard interpreter. Its content depends on `configuration options`_ 
-which may add new functionality and functions whose existence or non-existence 
-indicates the presence of such features. 
+by PyPy's standard interpreter. Its content depends on :doc:`configuration options `
+which may add new functionality and functions whose existence or non-existence
+indicates the presence of such features.
 
-.. _`configuration options`: config/index.html
 
 Generally available functionality
-=================================
+---------------------------------
 
  - ``internal_repr(obj)``: return the interpreter-level representation of an
    object.
@@ -22,28 +19,22 @@
    It works like a simplified array of characters (actually, depending on the
    configuration the ``array`` module internally uses this).
 
+
 Transparent Proxy Functionality
-===============================
+-------------------------------
 
-If `transparent proxies`_ are enabled (with :config:`objspace.std.withtproxy`)
+If :ref:`transparent proxies ` are enabled (with :config:`objspace.std.withtproxy`)
 the following functions are put into ``__pypy__``:
 
  - ``tproxy(typ, controller)``: Return something that looks like it is of type
    typ. Its behaviour is completely controlled by the controller. See the docs
-   about `transparent proxies`_ for detail.
-
+   about :ref:`transparent proxies ` for detail.
  - ``get_tproxy_controller(obj)``: If obj is really a transparent proxy, return
    its controller. Otherwise return None.
 
-.. _`transparent proxies`: objspace-proxies.html#tproxy
-
 
 Functionality available on py.py (not after translation)
-========================================================
+--------------------------------------------------------
 
  - ``isfake(obj)``: returns True if ``obj`` is faked.
-
  - ``interp_pdb()``: start a pdb at interpreter-level.
-
-
-
diff --git a/pypy/doc/_ref.txt b/pypy/doc/_ref.txt
deleted file mode 100644
--- a/pypy/doc/_ref.txt
+++ /dev/null
@@ -1,115 +0,0 @@
-.. This file is generated automatically by makeref.py script,
-   which in turn is run manually.
-
-.. _`ctypes_configure/doc/sample.py`: https://bitbucket.org/pypy/pypy/src/default/ctypes_configure/doc/sample.py
-.. _`dotviewer/`: https://bitbucket.org/pypy/pypy/src/default/dotviewer/
-.. _`lib-python/`: https://bitbucket.org/pypy/pypy/src/default/lib-python/
-.. _`lib-python/2.7/dis.py`: https://bitbucket.org/pypy/pypy/src/default/lib-python/2.7/dis.py
-.. _`lib_pypy/`: https://bitbucket.org/pypy/pypy/src/default/lib_pypy/
-.. _`lib_pypy/greenlet.py`: https://bitbucket.org/pypy/pypy/src/default/lib_pypy/greenlet.py
-.. _`lib_pypy/tputil.py`: https://bitbucket.org/pypy/pypy/src/default/lib_pypy/tputil.py
-.. _`lib_pypy/transaction.py`: https://bitbucket.org/pypy/pypy/src/default/lib_pypy/transaction.py
-.. _`pypy/bin/`: https://bitbucket.org/pypy/pypy/src/default/pypy/bin/
-.. _`pypy/bin/pyinteractive.py`: https://bitbucket.org/pypy/pypy/src/default/pypy/bin/pyinteractive.py
-.. _`pypy/config/`: https://bitbucket.org/pypy/pypy/src/default/pypy/config/
-.. _`pypy/config/pypyoption.py`: https://bitbucket.org/pypy/pypy/src/default/pypy/config/pypyoption.py
-.. _`pypy/doc/`: https://bitbucket.org/pypy/pypy/src/default/pypy/doc/
-.. _`pypy/doc/config/`: https://bitbucket.org/pypy/pypy/src/default/pypy/doc/config/
-.. _`pypy/doc/discussion/`: https://bitbucket.org/pypy/pypy/src/default/pypy/doc/discussion/
-.. _`pypy/goal/`: https://bitbucket.org/pypy/pypy/src/default/pypy/goal/
-.. _`pypy/interpreter`:
-.. _`pypy/interpreter/`: https://bitbucket.org/pypy/pypy/src/default/pypy/interpreter/
-.. _`pypy/interpreter/argument.py`: https://bitbucket.org/pypy/pypy/src/default/pypy/interpreter/argument.py
-.. _`pypy/interpreter/astcompiler`:
-.. _`pypy/interpreter/astcompiler/`: https://bitbucket.org/pypy/pypy/src/default/pypy/interpreter/astcompiler/
-.. _`pypy/interpreter/astcompiler/assemble.py`: https://bitbucket.org/pypy/pypy/src/default/pypy/interpreter/astcompiler/assemble.py
-.. _`pypy/interpreter/astcompiler/ast.py`: https://bitbucket.org/pypy/pypy/src/default/pypy/interpreter/astcompiler/ast.py
-.. _`pypy/interpreter/astcompiler/astbuilder.py`: https://bitbucket.org/pypy/pypy/src/default/pypy/interpreter/astcompiler/astbuilder.py
-.. _`pypy/interpreter/astcompiler/asthelpers.py`: https://bitbucket.org/pypy/pypy/src/default/pypy/interpreter/astcompiler/asthelpers.py
-.. _`pypy/interpreter/astcompiler/codegen.py`: https://bitbucket.org/pypy/pypy/src/default/pypy/interpreter/astcompiler/codegen.py
-.. _`pypy/interpreter/astcompiler/optimize.py`: https://bitbucket.org/pypy/pypy/src/default/pypy/interpreter/astcompiler/optimize.py
-.. _`pypy/interpreter/astcompiler/symtable.py`: https://bitbucket.org/pypy/pypy/src/default/pypy/interpreter/astcompiler/symtable.py
-.. _`pypy/interpreter/astcompiler/tools/Python.asdl`: https://bitbucket.org/pypy/pypy/src/default/pypy/interpreter/astcompiler/tools/Python.asdl
-.. _`pypy/interpreter/astcompiler/tools/asdl_py.py`: https://bitbucket.org/pypy/pypy/src/default/pypy/interpreter/astcompiler/tools/asdl_py.py
-.. _`pypy/interpreter/baseobjspace.py`: https://bitbucket.org/pypy/pypy/src/default/pypy/interpreter/baseobjspace.py
-.. _`pypy/interpreter/eval.py`: https://bitbucket.org/pypy/pypy/src/default/pypy/interpreter/eval.py
-.. _`pypy/interpreter/executioncontext.py`: https://bitbucket.org/pypy/pypy/src/default/pypy/interpreter/executioncontext.py
-.. _`pypy/interpreter/function.py`: https://bitbucket.org/pypy/pypy/src/default/pypy/interpreter/function.py
-.. _`pypy/interpreter/gateway.py`: https://bitbucket.org/pypy/pypy/src/default/pypy/interpreter/gateway.py
-.. _`pypy/interpreter/mixedmodule.py`: https://bitbucket.org/pypy/pypy/src/default/pypy/interpreter/mixedmodule.py
-.. _`pypy/interpreter/module.py`: https://bitbucket.org/pypy/pypy/src/default/pypy/interpreter/module.py
-.. _`pypy/interpreter/pyframe.py`: https://bitbucket.org/pypy/pypy/src/default/pypy/interpreter/pyframe.py
-.. _`pypy/interpreter/pyopcode.py`: https://bitbucket.org/pypy/pypy/src/default/pypy/interpreter/pyopcode.py
-.. _`pypy/interpreter/pyparser`:
-.. _`pypy/interpreter/pyparser/`: https://bitbucket.org/pypy/pypy/src/default/pypy/interpreter/pyparser/
-.. _`pypy/interpreter/pyparser/future.py`: https://bitbucket.org/pypy/pypy/src/default/pypy/interpreter/pyparser/future.py
-.. _`pypy/interpreter/pyparser/metaparser.py`: https://bitbucket.org/pypy/pypy/src/default/pypy/interpreter/pyparser/metaparser.py
-.. _`pypy/interpreter/pyparser/parser.py`: https://bitbucket.org/pypy/pypy/src/default/pypy/interpreter/pyparser/parser.py
-.. _`pypy/interpreter/pyparser/pyparse.py`: https://bitbucket.org/pypy/pypy/src/default/pypy/interpreter/pyparser/pyparse.py
-.. _`pypy/interpreter/pyparser/pytokenizer.py`: https://bitbucket.org/pypy/pypy/src/default/pypy/interpreter/pyparser/pytokenizer.py
-.. _`pypy/interpreter/typedef.py`: https://bitbucket.org/pypy/pypy/src/default/pypy/interpreter/typedef.py
-.. _`pypy/module`:
-.. _`pypy/module/`: https://bitbucket.org/pypy/pypy/src/default/pypy/module/
-.. _`pypy/module/__builtin__/__init__.py`: https://bitbucket.org/pypy/pypy/src/default/pypy/module/__builtin__/__init__.py
-.. _`pypy/module/cppyy/capi/__init__.py`: https://bitbucket.org/pypy/pypy/src/default/pypy/module/cppyy/capi/__init__.py
-.. _`pypy/module/cppyy/capi/builtin_capi.py`: https://bitbucket.org/pypy/pypy/src/default/pypy/module/cppyy/capi/builtin_capi.py
-.. _`pypy/module/cppyy/include/capi.h`: https://bitbucket.org/pypy/pypy/src/default/pypy/module/cppyy/include/capi.h
-.. _`pypy/module/test_lib_pypy/`: https://bitbucket.org/pypy/pypy/src/default/pypy/module/test_lib_pypy/
-.. _`pypy/objspace/`: https://bitbucket.org/pypy/pypy/src/default/pypy/objspace/
-.. _`pypy/objspace/std`:
-.. _`pypy/objspace/std/`: https://bitbucket.org/pypy/pypy/src/default/pypy/objspace/std/
-.. _`pypy/objspace/std/bytesobject.py`: https://bitbucket.org/pypy/pypy/src/default/pypy/objspace/std/bytesobject.py
-.. _`pypy/objspace/std/multimethod.py`: https://bitbucket.org/pypy/pypy/src/default/pypy/objspace/std/multimethod.py
-.. _`pypy/objspace/std/objspace.py`: https://bitbucket.org/pypy/pypy/src/default/pypy/objspace/std/objspace.py
-.. _`pypy/objspace/std/proxy_helpers.py`: https://bitbucket.org/pypy/pypy/src/default/pypy/objspace/std/proxy_helpers.py
-.. _`pypy/objspace/std/proxyobject.py`: https://bitbucket.org/pypy/pypy/src/default/pypy/objspace/std/proxyobject.py
-.. _`pypy/objspace/std/strbufobject.py`: https://bitbucket.org/pypy/pypy/src/default/pypy/objspace/std/strbufobject.py
-.. _`pypy/objspace/std/transparent.py`: https://bitbucket.org/pypy/pypy/src/default/pypy/objspace/std/transparent.py
-.. _`pypy/tool/`: https://bitbucket.org/pypy/pypy/src/default/pypy/tool/
-.. _`pypy/tool/pytest/`: https://bitbucket.org/pypy/pypy/src/default/pypy/tool/pytest/
-.. _`rpython/annotator`:
-.. _`rpython/annotator/`: https://bitbucket.org/pypy/pypy/src/default/rpython/annotator/
-.. _`rpython/annotator/annrpython.py`: https://bitbucket.org/pypy/pypy/src/default/rpython/annotator/annrpython.py
-.. _`rpython/annotator/binaryop.py`: https://bitbucket.org/pypy/pypy/src/default/rpython/annotator/binaryop.py
-.. _`rpython/annotator/builtin.py`: https://bitbucket.org/pypy/pypy/src/default/rpython/annotator/builtin.py
-.. _`rpython/bin/translatorshell.py`: https://bitbucket.org/pypy/pypy/src/default/rpython/bin/translatorshell.py
-.. _`rpython/config/`: https://bitbucket.org/pypy/pypy/src/default/rpython/config/
-.. _`rpython/config/translationoption.py`: https://bitbucket.org/pypy/pypy/src/default/rpython/config/translationoption.py
-.. _`rpython/flowspace/`: https://bitbucket.org/pypy/pypy/src/default/rpython/flowspace/
-.. _`rpython/flowspace/model.py`: https://bitbucket.org/pypy/pypy/src/default/rpython/flowspace/model.py
-.. _`rpython/memory/`: https://bitbucket.org/pypy/pypy/src/default/rpython/memory/
-.. _`rpython/memory/gc/generation.py`: https://bitbucket.org/pypy/pypy/src/default/rpython/memory/gc/generation.py
-.. _`rpython/memory/gc/hybrid.py`: https://bitbucket.org/pypy/pypy/src/default/rpython/memory/gc/hybrid.py
-.. _`rpython/memory/gc/minimarkpage.py`: https://bitbucket.org/pypy/pypy/src/default/rpython/memory/gc/minimarkpage.py
-.. _`rpython/memory/gc/semispace.py`: https://bitbucket.org/pypy/pypy/src/default/rpython/memory/gc/semispace.py
-.. _`rpython/rlib`:
-.. _`rpython/rlib/`: https://bitbucket.org/pypy/pypy/src/default/rpython/rlib/
-.. _`rpython/rlib/listsort.py`: https://bitbucket.org/pypy/pypy/src/default/rpython/rlib/listsort.py
-.. _`rpython/rlib/nonconst.py`: https://bitbucket.org/pypy/pypy/src/default/rpython/rlib/nonconst.py
-.. _`rpython/rlib/objectmodel.py`: https://bitbucket.org/pypy/pypy/src/default/rpython/rlib/objectmodel.py
-.. _`rpython/rlib/parsing/`: https://bitbucket.org/pypy/pypy/src/default/rpython/rlib/parsing/
-.. _`rpython/rlib/parsing/tree.py`: https://bitbucket.org/pypy/pypy/src/default/rpython/rlib/parsing/tree.py
-.. _`rpython/rlib/rarithmetic.py`: https://bitbucket.org/pypy/pypy/src/default/rpython/rlib/rarithmetic.py
-.. _`rpython/rlib/rbigint.py`: https://bitbucket.org/pypy/pypy/src/default/rpython/rlib/rbigint.py
-.. _`rpython/rlib/rrandom.py`: https://bitbucket.org/pypy/pypy/src/default/rpython/rlib/rrandom.py
-.. _`rpython/rlib/rsocket.py`: https://bitbucket.org/pypy/pypy/src/default/rpython/rlib/rsocket.py
-.. _`rpython/rlib/streamio.py`: https://bitbucket.org/pypy/pypy/src/default/rpython/rlib/streamio.py
-.. _`rpython/rlib/test`: https://bitbucket.org/pypy/pypy/src/default/rpython/rlib/test/
-.. _`rpython/rlib/unroll.py`: https://bitbucket.org/pypy/pypy/src/default/rpython/rlib/unroll.py
-.. _`rpython/rtyper`:
-.. _`rpython/rtyper/`: https://bitbucket.org/pypy/pypy/src/default/rpython/rtyper/
-.. _`rpython/rtyper/lltypesystem/`: https://bitbucket.org/pypy/pypy/src/default/rpython/rtyper/lltypesystem/
-.. _`rpython/rtyper/lltypesystem/lltype.py`: https://bitbucket.org/pypy/pypy/src/default/rpython/rtyper/lltypesystem/lltype.py
-.. _`rpython/rtyper/rint.py`: https://bitbucket.org/pypy/pypy/src/default/rpython/rtyper/rint.py
-.. _`rpython/rtyper/rlist.py`: https://bitbucket.org/pypy/pypy/src/default/rpython/rtyper/rlist.py
-.. _`rpython/rtyper/rmodel.py`: https://bitbucket.org/pypy/pypy/src/default/rpython/rtyper/rmodel.py
-.. _`rpython/rtyper/rtyper.py`: https://bitbucket.org/pypy/pypy/src/default/rpython/rtyper/rtyper.py
-.. _`rpython/rtyper/test/test_llinterp.py`: https://bitbucket.org/pypy/pypy/src/default/rpython/rtyper/test/test_llinterp.py
-.. _`rpython/tool/algo/`: https://bitbucket.org/pypy/pypy/src/default/rpython/tool/algo/
-.. _`rpython/translator`:
-.. _`rpython/translator/`: https://bitbucket.org/pypy/pypy/src/default/rpython/translator/
-.. _`rpython/translator/backendopt/`: https://bitbucket.org/pypy/pypy/src/default/rpython/translator/backendopt/
-.. _`rpython/translator/c/`: https://bitbucket.org/pypy/pypy/src/default/rpython/translator/c/
-.. _`rpython/translator/c/src/stacklet/`: https://bitbucket.org/pypy/pypy/src/default/rpython/translator/c/src/stacklet/
-.. _`rpython/translator/c/src/stacklet/stacklet.h`: https://bitbucket.org/pypy/pypy/src/default/rpython/translator/c/src/stacklet/stacklet.h
-.. _`rpython/translator/tool/`: https://bitbucket.org/pypy/pypy/src/default/rpython/translator/tool/
diff --git a/pypy/doc/architecture.rst b/pypy/doc/architecture.rst
--- a/pypy/doc/architecture.rst
+++ b/pypy/doc/architecture.rst
@@ -1,93 +1,29 @@
-==================================================
-Goals and Architecture Overview 
-==================================================
+Goals and Architecture Overview
+===============================
 
 .. contents::
 
+This document gives an overview of the goals and architecture of PyPy. If you're
+interested in :ref:`using PyPy ` or :ref:`hacking on it `,
+have a look at our :ref:`getting started ` section.
 
-This document gives an overview of the goals and architecture of PyPy.
-See `getting started`_ for a practical introduction and starting points. 
 
-Mission statement 
-====================
+Mission statement
+-----------------
 
-We aim to provide:
+We aim to provide a compliant, flexible and fast implementation of the Python_
+Language which uses the RPython toolchain to enable new advanced high-level
+features without having to encode the low-level details.  We call this PyPy.
 
- * a common translation and support framework for producing
-   implementations of dynamic languages, emphasizing a clean
-   separation between language specification and implementation
-   aspects.  We call this the `RPython toolchain`_.
+.. _Python: http://docs.python.org/reference/
 
- * a compliant, flexible and fast implementation of the Python_ Language 
-   which uses the above toolchain to enable new advanced high-level features 
-   without having to encode the low-level details.  We call this PyPy.
-
-By separating concerns in this way, our implementation
-of Python - and other dynamic languages - is able to automatically
-generate a Just-in-Time compiler for any dynamic language.  It also
-allows a mix-and-match approach to implementation decisions, including
-many that have historically been outside of a user's control, such as
-target platform, memory and 
-threading models, garbage collection strategies, and optimizations applied, 
-including whether or not to have a JIT in the first place.
 
 High Level Goals
-=============================
-
-RPython - the Translation Toolchain
------------------------------------------------
-
-Traditionally, language interpreters are written in a target platform language
-such as C/Posix, Java or C#.  Each implementation provides 
-a fundamental mapping between application source code and the target 
-environment.  One of 
-the goals of the "all-encompassing" environments, such as the .NET framework
-and to some extent the Java virtual machine, is to provide standardized
-and higher level functionalities in order to support language implementers
-for writing language implementations. 
-
-PyPy is experimenting with a more ambitious approach.  We are using a
-subset of the high-level language Python, called RPython_, in which we
-write languages as simple interpreters with few references to and
-dependencies on lower level details.  The `RPython toolchain`_
-produces a concrete virtual machine for the platform of our choice by
-inserting appropriate lower level aspects.  The result can be customized
-by selecting other feature and platform configurations.
-
-Our goal is to provide a possible solution to the problem of language
-implementers: having to write ``l * o * p`` interpreters for ``l``
-dynamic languages and ``p`` platforms with ``o`` crucial design
-decisions.  PyPy aims at making it possible to change each of these
-variables independently such that:
-
-* ``l``: the language that we analyze can be evolved or entirely replaced;
-
-* ``o``: we can tweak and optimize the translation process to produce 
-  platform specific code based on different models and trade-offs;
-
-* ``p``: we can write new translator back-ends to target different
-  physical and virtual platforms.
-
-By contrast, a standardized target environment - say .NET -
-enforces ``p=1`` as far as it's concerned.  This helps making ``o`` a
-bit smaller by providing a higher-level base to build upon.  Still,
-we believe that enforcing the use of one common environment 
-is not necessary.  PyPy's goal is to give weight to this claim - at least 
-as far as language implementation is concerned - showing an approach
-to the ``l * o * p`` problem that does not rely on standardization.
-
-The most ambitious part of this goal is to `generate Just-In-Time
-Compilers`_ in a language-independent way, instead of only translating
-the source interpreter into an interpreter for the target platform.
-This is an area of language implementation that is commonly considered
-very challenging because of the involved complexity.
-
-
-PyPy - the Python Interpreter
---------------------------------------------
+----------------
 
 Our main motivation for developing the translation framework is to
-provide a full featured, customizable, fast_ and `very compliant`_ Python
+provide a full featured, customizable, :ref:`fast ` and
+:doc:`very compliant ` Python
 implementation, working on and interacting with a large variety of
 platforms and allowing the quick introduction of new advanced language
 features.
@@ -106,88 +42,22 @@
 and fully orthogonal to the interpreter source code.
 
 
-PyPy Architecture 
-===========================
-
-As you would expect from a project implemented using ideas from the world
-of `Extreme Programming`_, the architecture of PyPy has evolved over time
-and continues to evolve.  Nevertheless, the high level architecture is 
-stable. As described above, there are two rather independent basic
-subsystems: the `PyPy Python Interpreter`_ and the `RPython Translation Toolchain`_.
-
-.. _`translation framework`:
-
-RPython Translation Toolchain
--------------------------
-
-The job of the RPython toolchain is to translate RPython_ programs
-into an efficient version of that program for one of the various target
-platforms, generally one that is considerably lower-level than Python.
-
-The approach we have taken is to reduce the level of abstraction of the
-source RPython program in several steps, from the high level down to the
-level of the target platform, whatever that may be.  Currently we
-support two broad flavours of target platforms: the ones that assume a
-C-like memory model with structures and pointers, and the ones that
-assume an object-oriented model with classes, instances and methods (as,
-for example, the Java and .NET virtual machines do).
-
-The RPython toolchain never sees the RPython source code or syntax
-trees, but rather starts with the *code objects* that define the
-behaviour of the function objects one gives it as input.  It can be
-considered as "freezing" a pre-imported RPython program into an
-executable form suitable for the target platform.
-
-The steps of the translation process can be summarized as follows:
-
-* The code object of each source functions is converted to a `control
-  flow graph` by the `Flow Object Space`_.
-
-* The control flow graphs are processed by the Annotator_, which
-  performs whole-program type inference to annotate each variable of
-  the control flow graph with the types it may take at run-time.
-
-* The information provided by the annotator is used by the RTyper_ to
-  convert the high level operations of the control flow graphs into
-  operations closer to the abstraction level of the target platform.
-
-* Optionally, `various transformations`_ can then be applied which, for
-  example, perform optimizations such as inlining, add capabilities
-  such as stackless-style concurrency, or insert code for the
-  `garbage collector`_.
-
-* Then, the graphs are converted to source code for the target platform
-  and compiled into an executable.
-
-This process is described in much more detail in the `document about
-the RPython toolchain`_ and in the paper `Compiling dynamic language
-implementations`_.
-
-.. _`control flow graph`: translation.html#the-flow-model
-.. _`Flow Object Space`: objspace.html#the-flow-object-space
-.. _Annotator: translation.html#the-annotation-pass
-.. _RTyper: rtyper.html#overview
-.. _`various transformations`: translation.html#the-optional-transformations
-.. _`document about the RPython toolchain`: translation.html
-.. _`garbage collector`: garbage_collection.html
-.. _`RPython toolchain`: translation.html
-.. _`standard interpreter`: 
-.. _`python interpreter`: 
+.. _python-interpreter:
 
 PyPy Python Interpreter
--------------------------------------
+-----------------------
 
 PyPy's *Python Interpreter* is written in RPython and implements the
 full Python language.  This interpreter very closely emulates the
 behavior of CPython.  It contains the following key components:
 
-- a bytecode compiler responsible for producing Python code objects 
+- a bytecode compiler responsible for producing Python code objects
   from the source code of a user application;
 
-- a `bytecode evaluator`_ responsible for interpreting 
+- a :doc:`bytecode evaluator ` responsible for interpreting
   Python code objects;
 
-- a `standard object space`_, responsible for creating and manipulating
+- a :ref:`standard object space `, responsible for creating and manipulating
   the Python objects seen by the application.
 
 The *bytecode compiler* is the preprocessing phase that produces a
@@ -200,64 +70,6 @@
 lists, as well as the operations between them, like addition or
 truth-value-testing.
 
-This division between bytecode evaluator and object space is very
-important, as it gives a lot of flexibility.  One can plug in 
-different `object spaces`_ to get different or enriched behaviours 
-of the Python objects.  Additionally, a special more abstract object
-space, the `flow object space`_, allows us to reuse the bytecode
-evaluator for our translation framework.
-
-.. _`bytecode evaluator`: interpreter.html
-.. _`standard object space`: objspace.html#the-standard-object-space
-.. _`object spaces`: objspace.html
-.. _`flow object space`: objspace.html#the-flow-object-space
-
-.. _`the translation framework`:
-
-
-Further reading
-===============
-
-All of PyPy's documentation can be reached from the `documentation
-index`_.  Of particular interest after reading this document might be:
-
- * `getting-started`_: a hands-on guide to getting involved with the
-   PyPy source code.
-
- * `PyPy's approach to virtual machine construction`_: a paper
-   presented to the Dynamic Languages Symposium attached to OOPSLA
-   2006.
-
- * `The translation document`_: a detailed description of our
-   translation process.
-
- * `JIT Generation in PyPy`_, describing how we produce a Just-in-time
-   Compiler from an interpreter.
-
- * A tutorial of how to use the `RPython toolchain`_ to `implement your own
-   interpreter`_.
-
-.. _`documentation index`: index.html#project-documentation
-.. _`getting-started`: getting-started-dev.html
-.. _`PyPy's approach to virtual machine construction`: https://bitbucket.org/pypy/extradoc/raw/tip/talk/dls2006/pypy-vm-construction.pdf
-.. _`the translation document`: translation.html
-.. _`RPython toolchain`: translation.html
-.. _`Compiling dynamic language implementations`: https://bitbucket.org/pypy/extradoc/raw/tip/eu-report/D05.1_Publish_on_translating_a_very-high-level_description.pdf
-.. _`Technical reports`: index-report.html
-
-.. _`getting started`: getting-started.html
-.. _`Extreme Programming`: http://www.extremeprogramming.org/
-
-.. _fast: faq.html#how-fast-is-pypy
-.. _`very compliant`: cpython_differences.html
-
-.. _`RPython`: coding-guide.html#rpython
-
-.. _Python: http://docs.python.org/reference/
-.. _Psyco: http://psyco.sourceforge.net
-.. _`generate Just-In-Time Compilers`: jit/index.html
-.. _`JIT Generation in PyPy`: jit/index.html
-.. _`implement your own interpreter`: http://morepypy.blogspot.com/2011/04/tutorial-writing-interpreter-with-pypy.html
-
-.. include:: _ref.txt
-
+This division between bytecode evaluator and object space gives a lot of
+flexibility.  One can plug in different :doc:`object spaces ` to get
+different or enriched behaviours of the Python objects.
diff --git a/pypy/doc/arm.rst b/pypy/doc/arm.rst
deleted file mode 100644
--- a/pypy/doc/arm.rst
+++ /dev/null
@@ -1,166 +0,0 @@
-=========================
-Cross-translating for ARM
-=========================
-
-
-Here we describe the setup required and the steps needed to follow to translate
-an interpreter using the RPython translator to target ARM using a cross
-compilation toolchain.
-
-To translate an RPython program for ARM we can either
-translate directly on an ARM device following the normal translation steps. Unfortunately this is not really feasible on most ARM machines. The alternative is to cross-translate using a cross-compilation toolchain.
-
-To cross-translate we run the translation on a more powerful (usually
-x86) machine and generate a binary for ARM using a cross-compiler to compile
-the generated C code. There are several constraints when doing this. In
-particular we currently only support Linux as translation host and target
-platforms (tested on Ubuntu). Also we need a 32-bit environment to run the
-translation. This can be done either on a 32bit host or in 32bit chroot.
-
-
-Requirements
-------------
-
-The tools required to cross translate from a Linux based host to an ARM based Linux target are:
-
-- A checkout of PyPy (default branch).
-- The GCC ARM cross compiler (on Ubuntu it is the ``gcc-arm-linux-gnueabi package``) but other toolchains should also work.
-- Scratchbox 2, a cross-compilation engine (``scratchbox2`` Ubuntu package).
-- A 32-bit PyPy or Python.
-- And the following (or corresponding) packages need to be installed to create an ARM based chroot:
-
-  * ``debootstrap`` 
-  * ``schroot``
-  * ``binfmt-support``
-  * ``qemu-system``
-  * ``qemu-user-static``
-
-- The dependencies above are in addition to the ones needed for a regular
-  translation, `listed here`_.
-
-.. _`listed here`: getting-started-python.html#translating-the-pypy-python-interpreter
-
-
-Creating a Qemu based ARM chroot
---------------------------------
-
-First we will need to create a rootfs containing the packages and dependencies
-required in order to translate PyPy or other interpreters. We are going to
-assume, that the files will be placed in ``/srv/chroot/precise_arm``.
-
-Create the rootfs by calling:
-
-::
-
-   mkdir -p /srv/chroot/precise_arm
-   qemu-debootstrap --variant=buildd --arch=armel precise /srv/chroot/precise_arm/  http://ports.ubuntu.com/ubuntu-ports/
-
-Next, copy the qemu-arm-static binary to the rootfs.
-
-:: 
-
-  cp /usr/bin/qemu-arm-static /srv/chroot/precise_arm/usr/bin/qemu-arm-static
-
-For easier configuration and management we will create a schroot pointing to
-the rootfs. We need to add a configuration block (like the one below) to the
-schroot configuration file in /etc/schroot/schroot.conf.
-
-
-::
-
-  [precise_arm]
-  directory=/srv/chroot/precise_arm
-  users=USERNAME
-  root-users=USERNAME
-  groups=users
-  aliases=default
-  type=directory
-
-
-To verify that everything is working in the chroot, running ``schroot -c
-precise_arm`` should start a shell running in the schroot environment using
-qemu-arm to execute the ARM binaries. Running ``uname -m`` in the chroot should 
-yeild a result like ``armv7l``. Showing that we are emulating an ARM system.
-
-Start the schroot as the user root in order to configure the apt sources and
-to install the following packages:
-
-
-::
-
-  schroot -c precise_arm -u root
-  echo "deb http://ports.ubuntu.com/ubuntu-ports/ precise main universe restricted" > /etc/apt/sources.list
-  apt-get update
-  apt-get install libffi-dev libgc-dev python-dev build-essential libncurses5-dev libbz2-dev
-
-
-Now all dependencies should be in place and we can exit the schroot environment.
-
-
-Configuring scratchbox2
------------------------
-
-To configure the scratchbox we need to cd into the root directory of the rootfs
-we created before. From there we can call the sb2 configuration tools which
-will take the current directory as the base directory for the scratchbox2
-environment.
-
-::
-
-  cd /srv/chroot/precise_arm
-  sb2-init -c `which qemu-arm` ARM `which arm-linux-gnueabi-gcc`
-
-This will create a scratchbox2 based environment called ARM that maps calls to
-gcc done within the scratchbox to the arm-linux-gnueabi-gcc outside the
-scratchbox. Now we should have a working cross compilation toolchain in place
-and can start cross-translating programs for ARM.
-
-Translation
------------
-
-Having performed all the preliminary steps we should now be able to cross
-translate a program for ARM.  You can use this_ minimal
-target to test your setup before applying it to a larger project.
-
-Before starting the translator we need to set two environment variables, so the
-translator knows how to use the scratchbox environment. We need to set the
-**SB2** environment variable to point to the rootfs and the **SB2OPT** should
-contain the command line options for the sb2 command. If our rootfs is in the
-folder /srv/chroot/precise_arm and the scratchbox environment is called "ARM",
-the variables would be defined as follows.
-
-
-::
-
-  export SB2=/srv/chroot/precise_arm
-  export SB2OPT='-t ARM'
-
-Once this is set, you can call the translator. For example save this file 
-
-::
-
-  def main(args):
-      print "Hello World"
-      return 0
-
-  def target(*args):
-      return main, None
-
-and call the translator
-
-::
-
-  pypy ~/path_to_pypy_checkout/rpython/bin/rpython -O1 --platform=arm target.py
-
-If everything worked correctly this should yield an ARM binary. Running this binary in the ARM chroot or on an ARM device should produce the output ``"Hello World"``.
-
-To translate the full python pypy interpreter with a jit, you can cd into pypy/goal and call      
-
-::
-
-  pypy /rpython/bin/rpython -Ojit --platform=arm --gcrootfinder=shadowstack --jit-backend=arm targetpypystandalone.py
-
-The gcrootfinder option is needed to work around `issue 1377`_ and the jit-backend works around `issue 1376`_
-
-.. _`issue 1377`: https://bugs.pypy.org/issue1377
-.. _`issue 1376`: https://bugs.pypy.org/issue1376
diff --git a/pypy/doc/build.rst b/pypy/doc/build.rst
new file mode 100644
--- /dev/null
+++ b/pypy/doc/build.rst
@@ -0,0 +1,170 @@
+Building PyPy from Source
+=========================
+
+For building PyPy, we recommend installing a pre-built PyPy first (see
+:doc:`install`). It is possible to build PyPy with CPython, but it will take a
+lot longer to run -- depending on your architecture, between two and three
+times as long.
+
+Even when using PyPy to build PyPy, translation is time-consuming -- 30
+minutes on a fast machine -- and RAM-hungry.  You will need **at least** 2 GB
+of memory on a 32-bit machine and 4GB on a 64-bit machine.
+
+
+Clone the repository
+--------------------
+
+If you prefer to compile your own PyPy, or if you want to modify it, you
+will need to obtain a copy of the sources.  This can be done either by
+`downloading them from the download page`_ or by checking them out from the
+repository using mercurial.  We suggest using mercurial if you want to access
+the current development.
+
+.. _downloading them from the download page: http://pypy.org/download.html
+
+You must issue the following command on your
+command line, DOS box, or terminal::
+
+    hg clone http://bitbucket.org/pypy/pypy pypy
+
+This will clone the repository and place it into a directory
+named ``pypy``, and will get you the PyPy source in ``pypy/pypy`` and
+documentation files in ``pypy/pypy/doc``.
+We try to ensure that the tip is always stable, but it might
+occasionally be broken.  You may want to check out `our nightly tests`_:
+find a revision (12-chars alphanumeric string, e.g. "963e808156b3")
+that passed at least the
+``{linux32}`` tests (corresponding to a ``+`` sign on the
+line ``success``) and then, in your cloned repository, switch to this revision
+using::
+
+    hg up -r XXXXX
+
+where XXXXX is the revision id.
+
+.. _our nightly tests: http://buildbot.pypy.org/summary?branch=
+
+
+Install build-time dependencies
+-------------------------------
+
+To build PyPy on Unix using the C translation backend, you need at least a C
+compiler and ``make`` installed. Further, some optional modules have additional
+dependencies:
+
+cffi, ctypes
+    libffi, pkg-config
+
+zlib
+    libz
+
+bz2
+    libbz2
+
+lzma (PyPy3 only)
+    liblzma
+
+sqlite3
+    libsqlite3
+
+curses
+    libncurses + cffi dependencies from above
+
+pyexpat
+    libexpat1
+
+_ssl
+    libssl
+
+Make sure to have these libraries (with development headers) installed before
+building PyPy, otherwise the resulting binary will not contain these modules.
+
+On Debian, this is the command to install all build-time dependencies::
+
+    apt-get install gcc make libffi-dev pkg-config libz-dev libbz2-dev \
+    libsqlite3-dev libncurses-dev libexpat1-dev libssl-dev
+
+For the optional lzma module on PyPy3 you will also need ``liblzma-dev``.
+
+On Fedora::
+
+    yum install gcc make libffi-devel pkgconfig zlib-devel bzip2-devel \
+    lib-sqlite3-devel ncurses-devel expat-devel openssl-devel
+
+For the optional lzma module on PyPy3 you will also need ``xz-devel``.
+
+On SLES11::
+
+    zypper install gcc make python-devel pkg-config \
+    zlib-devel libopenssl-devel libbz2-devel sqlite3-devel \
+    libexpat-devel libffi-devel python-curses
+
+For the optional lzma module on PyPy3 you will also need ``xz-devel``.
+
+On Mac OS X, most of these build-time dependencies are installed alongside
+the Developer Tools. However, note that in order for the installation to
+find them you may need to run::
+
+    xcode-select --install
+
+
+Run the translation
+-------------------
+
+Translate with JIT::
+
+    pypy rpython/bin/rpython --opt=jit pypy/goal/targetpypystandalone.py
+
+Translate without JIT::
+
+    pypy rpython/bin/rpython --opt=2 pypy/goal/targetpypystandalone.py
+
+(You can use ``python`` instead of ``pypy`` here, which will take longer
+but works too.)
+
+If everything works correctly this will create an executable ``pypy-c`` in the
+current directory. The executable behaves mostly like a normal Python
+interpreter (see :doc:`cpython_differences`).
+
+
+.. _translate-pypy:
+
+Translating with non-standard options
+-------------------------------------
+
+It is possible to have non-standard features enabled for translation,
+but they are not really tested any more.  Look, for example, at the
+:doc:`objspace proxies ` document.
+
+
+
+Installation
+------------
+
+PyPy dynamically finds the location of its libraries depending on the location
+of the executable. The directory hierarchy of a typical PyPy installation
+looks like this::
+
+    ./bin/pypy
+    ./include/
+    ./lib_pypy/
+    ./lib-python/2.7
+    ./site-packages/
+
+The hierarchy shown above is relative to a PREFIX directory. PREFIX is
+computed by starting from the directory where the executable resides, and
+"walking up" the filesystem until we find a directory containing ``lib_pypy``
+and ``lib-python/2.7``.
+
+To install PyPy system wide on unix-like systems, it is recommended to put the
+whole hierarchy alone (e.g. in ``/opt/pypy``) and put a symlink to the
+``pypy`` executable into ``/usr/bin`` or ``/usr/local/bin``.
+
+If the executable fails to find suitable libraries, it will report ``debug:
+WARNING: library path not found, using compiled-in sys.path`` and then attempt
+to continue normally. If the default path is usable, most code will be fine.
+However, the ``sys.prefix`` will be unset and some existing libraries assume
+that this is never the case.
+
+
+.. TODO windows
diff --git a/pypy/doc/cleanup.rst b/pypy/doc/cleanup.rst
deleted file mode 100644
--- a/pypy/doc/cleanup.rst
+++ /dev/null
@@ -1,11 +0,0 @@
-Old documentation that needs review
------------------------------------
-
-.. The following stuff is old (and crufty?), and needs further investigation:
-
-.. doc-index: This needs merging somehow
-
-.. toctree::
-
-   distribution.rst
-
diff --git a/pypy/doc/coding-guide.rst b/pypy/doc/coding-guide.rst
--- a/pypy/doc/coding-guide.rst
+++ b/pypy/doc/coding-guide.rst
@@ -1,21 +1,17 @@
-====================================
 Coding Guide
-====================================
+============
 
 .. contents::
 
 This document describes coding requirements and conventions for
 working with the PyPy code base.  Please read it carefully and
 ask back any questions you might have. The document does not talk
-very much about coding style issues. We mostly follow `PEP 8`_ though.
+very much about coding style issues. We mostly follow :pep:`8` though.
 If in doubt, follow the style that is already present in the code base.
 
-.. _`PEP 8`: http://www.python.org/dev/peps/pep-0008/
-
-.. _`RPython`:
 
 Overview and motivation
-========================
+------------------------
 
 We are writing a Python interpreter in Python, using Python's well known
 ability to step behind the algorithmic problems as a language. At first glance,
@@ -25,7 +21,7 @@
 
 
 CPython vs. PyPy
--------------------
+~~~~~~~~~~~~~~~~
 
 Compared to the CPython implementation, Python takes the role of the C
 Code. We rewrite the CPython interpreter in Python itself.  We could
@@ -44,20 +40,20 @@
 but let's stick with this somewhat canonical approach.
 
 
-.. _`application-level`:
-.. _`interpreter-level`:
+.. _application-level:
+.. _interpreter-level:
 
 Application-level and interpreter-level execution and objects
--------------------------------------------------------------
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 Since Python is used for implementing all of our code base, there is a
-crucial distinction to be aware of: that between *interpreter-level* objects and 
+crucial distinction to be aware of: that between *interpreter-level* objects and
 *application-level* objects.  The latter are the ones that you deal with
 when you write normal python programs.  Interpreter-level code, however,
 cannot invoke operations nor access attributes from application-level
 objects.  You will immediately recognize any interpreter level code in
 PyPy, because half the variable and object names start with a ``w_``, which
-indicates that they are `wrapped`_ application-level values. 
+indicates that they are `wrapped`_ application-level values.
 
 Let's show the difference with a simple example.  To sum the contents of
 two variables ``a`` and ``b``, one would write the simple application-level
@@ -80,10 +76,10 @@
 interpreting.
 
 
-.. _`app-preferable`: 
+.. _app-preferable:
 
-Application level is often preferable 
--------------------------------------
+Application level is often preferable
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 Application-level code is substantially higher-level, and therefore
 correspondingly easier to write and debug.  For example, suppose we want
@@ -113,11 +109,11 @@
             space.setitem(w_self, w_key, w_value)
 
 This interpreter-level implementation looks much more similar to the C
-source code.  It is still more readable than its C counterpart because 
-it doesn't contain memory management details and can use Python's native 
-exception mechanism. 
+source code.  It is still more readable than its C counterpart because
+it doesn't contain memory management details and can use Python's native
+exception mechanism.
 
-In any case, it should be obvious that the application-level implementation 
+In any case, it should be obvious that the application-level implementation
 is definitely more readable, more elegant and more maintainable than the
 interpreter-level one (and indeed, dict.update is really implemented at
 applevel in PyPy).
@@ -129,10 +125,11 @@
 level code is usually preferable.  We have an abstraction (called the
 'Gateway') which allows the caller of a function to remain ignorant of
 whether a particular function is implemented at application or
-interpreter level. 
+interpreter level.
+
 
 Our runtime interpreter is "RPython"
-----------------------------------------------
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 In order to make a C code generator feasible all code on interpreter level has
 to restrict itself to a subset of the Python language, and we adhere to some
@@ -159,7 +156,7 @@
 elegant: For the definition of all the opcodes of the Python
 interpreter, the module ``dis`` is imported and used to initialize our
 bytecode interpreter.  (See ``__initclass__`` in
-`pypy/interpreter/pyopcode.py`_).  This
+:source:`pypy/interpreter/pyopcode.py`).  This
 saves us from adding extra modules to PyPy. The import code is run at
 startup time, and we are allowed to use the CPython builtin import
 function.
@@ -172,308 +169,13 @@
 enables the code generator to emit efficient machine level replacements
 for pure integer objects, for instance.
 
-RPython
-=================
-
-RPython Definition
-------------------
-
-RPython is a restricted subset of Python that is amenable to static analysis.
-Although there are additions to the language and some things might surprisingly
-work, this is a rough list of restrictions that should be considered. Note
-that there are tons of special cased restrictions that you'll encounter
-as you go. The exact definition is "RPython is everything that our translation
-toolchain can accept" :)
-
-.. _`wrapped object`: coding-guide.html#wrapping-rules
-
-Flow restrictions
--------------------------
-
-**variables**
-
-  variables should contain values of at most one type as described in
-  `Object restrictions`_ at each control flow point, that means for
-  example that joining control paths using the same variable to
-  contain both a string and a int must be avoided.  It is allowed to
-  mix None (basically with the role of a null pointer) with many other
-  types: `wrapped objects`, class instances, lists, dicts, strings, etc.
-  but *not* with int, floats or tuples.
-
-**constants**
-
-  all module globals are considered constants.  Their binding must not
-  be changed at run-time.  Moreover, global (i.e. prebuilt) lists and
-  dictionaries are supposed to be immutable: modifying e.g. a global
-  list will give inconsistent results.  However, global instances don't
-  have this restriction, so if you need mutable global state, store it
-  in the attributes of some prebuilt singleton instance.
-
-
-
-**control structures**
-
-  all allowed, ``for`` loops restricted to builtin types, generators
-  very restricted.
-
-**range**
-
-  ``range`` and ``xrange`` are identical. ``range`` does not necessarily create an array,
-  only if the result is modified. It is allowed everywhere and completely
-  implemented. The only visible difference to CPython is the inaccessibility
-  of the ``xrange`` fields start, stop and step.
-
-**definitions**
-
-  run-time definition of classes or functions is not allowed.
-
-**generators**
-
-  generators are supported, but their exact scope is very limited. you can't
-  merge two different generator in one control point.
-
-**exceptions**
-
-+ fully supported
-+ see below `Exception rules`_ for restrictions on exceptions raised by built-in operations
-
-
-Object restrictions
--------------------------
-
-We are using
-
-**integer, float, boolean**
-
-  works.
-
-**strings**
-
-  a lot of, but not all string methods are supported and those that are
-  supported, not necesarilly accept all arguments.  Indexes can be
-  negative.  In case they are not, then you get slightly more efficient
-  code if the translator can prove that they are non-negative.  When
-  slicing a string it is necessary to prove that the slice start and
-  stop indexes are non-negative. There is no implicit str-to-unicode cast
-  anywhere. Simple string formatting using the ``%`` operator works, as long
-  as the format string is known at translation time; the only supported
-  formatting specifiers are ``%s``, ``%d``, ``%x``, ``%o``, ``%f``, plus
-  ``%r`` but only for user-defined instances. Modifiers such as conversion
-  flags, precision, length etc. are not supported. Moreover, it is forbidden
-  to mix unicode and strings when formatting.
-
-**tuples**
-
-  no variable-length tuples; use them to store or return pairs or n-tuples of
-  values. Each combination of types for elements and length constitute
-  a separate and not mixable type.
-
-**lists**
-
-  lists are used as an allocated array.  Lists are over-allocated, so list.append()
-  is reasonably fast. However, if you use a fixed-size list, the code
-  is more efficient. Annotator can figure out most of the time that your
-  list is fixed-size, even when you use list comprehension.
-  Negative or out-of-bound indexes are only allowed for the
-  most common operations, as follows:
-
-  - *indexing*:
-    positive and negative indexes are allowed. Indexes are checked when requested
-    by an IndexError exception clause.
-  
-  - *slicing*:
-    the slice start must be within bounds. The stop doesn't need to, but it must
-    not be smaller than the start.  All negative indexes are disallowed, except for
-    the [:-1] special case.  No step.  Slice deletion follows the same rules.
-    
-  - *slice assignment*:
-    only supports ``lst[x:y] = sublist``, if ``len(sublist) == y - x``.
-    In other words, slice assignment cannot change the total length of the list,
-    but just replace items.
-
-  - *other operators*:
-    ``+``, ``+=``, ``in``, ``*``, ``*=``, ``==``, ``!=`` work as expected.
-
-  - *methods*:
-    append, index, insert, extend, reverse, pop.  The index used in pop() follows
-    the same rules as for *indexing* above.  The index used in insert() must be within
-    bounds and not negative.
-
-**dicts**
-
-  dicts with a unique key type only, provided it is hashable. Custom
-  hash functions and custom equality will not be honored.
-  Use ``rpython.rlib.objectmodel.r_dict`` for custom hash functions.
-
-
-**list comprehensions**
-
-  May be used to create allocated, initialized arrays.
-
-**functions**
-
-+ statically called functions may use defaults and a variable number of
-  arguments (which may be passed as a list instead of a tuple, so write code
-  that does not depend on it being a tuple).
-
-+ dynamic dispatch enforces the use of signatures that are equal for all
-  possible called function, or at least "compatible enough".  This
-  concerns mainly method calls, when the method is overridden or in any
-  way given different definitions in different classes.  It also concerns
-  the less common case of explicitly manipulated function objects.
-  Describing the exact compatibility rules is rather involved (but if you
-  break them, you should get explicit errors from the rtyper and not
-  obscure crashes.)
-
-**builtin functions**
-
-  A number of builtin functions can be used.  The precise set can be
-  found in `rpython/annotator/builtin.py`_ (see ``def builtin_xxx()``).
-  Some builtin functions may be limited in what they support, though.
-
-  ``int, float, str, ord, chr``... are available as simple conversion
-  functions.  Note that ``int, float, str``... have a special meaning as
-  a type inside of isinstance only.
-
-**classes**
-
-+ methods and other class attributes do not change after startup
-+ single inheritance is fully supported
-+ use `rpython.rlib.objectmodel.import_from_mixin(M)` in a class
-  body to copy the whole content of a class `M`.  This can be used
-  to implement mixins: functions and staticmethods are duplicated
-  (the other class attributes are just copied unmodified).
-
-+ classes are first-class objects too
-
-**objects**
-
-  Normal rules apply. The only special methods that are honoured are
-  ``__init__``, ``__del__``, ``__len__``, ``__getitem__``, ``__setitem__``,
-  ``__getslice__``, ``__setslice__``, and ``__iter__``. To handle slicing,
-  ``__getslice__`` and ``__setslice__`` must be used; using ``__getitem__`` and
-   ``__setitem__`` for slicing isn't supported. Additionally, using negative
-   indices for slicing is still not support, even when using ``__getslice__``.
-
-This layout makes the number of types to take care about quite limited.
-
-
-Integer Types
--------------------------
-
-While implementing the integer type, we stumbled over the problem that
-integers are quite in flux in CPython right now. Starting with Python 2.4,
-integers mutate into longs on overflow.  In contrast, we need
-a way to perform wrap-around machine-sized arithmetic by default, while still
-being able to check for overflow when we need it explicitly.  Moreover, we need
-a consistent behavior before and after translation.
-
-We use normal integers for signed arithmetic.  It means that before
-translation we get longs in case of overflow, and after translation we get a
-silent wrap-around.  Whenever we need more control, we use the following
-helpers (which live in `rpython/rlib/rarithmetic.py`_):
-
-**ovfcheck()**
-
-  This special function should only be used with a single arithmetic operation
-  as its argument, e.g. ``z = ovfcheck(x+y)``.  Its intended meaning is to
-  perform the given operation in overflow-checking mode.
-
-  At run-time, in Python, the ovfcheck() function itself checks the result
-  and raises OverflowError if it is a ``long``.  But the code generators use
-  ovfcheck() as a hint: they replace the whole ``ovfcheck(x+y)`` expression
-  with a single overflow-checking addition in C.
-
-**intmask()**
-
-  This function is used for wrap-around arithmetic.  It returns the lower bits
-  of its argument, masking away anything that doesn't fit in a C "signed long int".
-  Its purpose is, in Python, to convert from a Python ``long`` that resulted from a
-  previous operation back to a Python ``int``.  The code generators ignore
-  intmask() entirely, as they are doing wrap-around signed arithmetic all the time
-  by default anyway.  (We have no equivalent of the "int" versus "long int"
-  distinction of C at the moment and assume "long ints" everywhere.)
-
-**r_uint**
-
-  In a few cases (e.g. hash table manipulation), we need machine-sized unsigned
-  arithmetic.  For these cases there is the r_uint class, which is a pure
-  Python implementation of word-sized unsigned integers that silently wrap
-  around.  ("word-sized" and "machine-sized" are used equivalently and mean
-  the native size, which you get using "unsigned long" in C.)
-  The purpose of this class (as opposed to helper functions as above)
-  is consistent typing: both Python and the annotator will propagate r_uint
-  instances in the program and interpret all the operations between them as
-  unsigned.  Instances of r_uint are special-cased by the code generators to
-  use the appropriate low-level type and operations.
-  Mixing of (signed) integers and r_uint in operations produces r_uint that
-  means unsigned results.  To convert back from r_uint to signed integers, use
-  intmask().
-
-
-Exception rules
----------------------
-
-Exceptions are by default not generated for simple cases.::
-
-    #!/usr/bin/python
-
-        lst = [1,2,3,4,5]
-        item = lst[i]    # this code is not checked for out-of-bound access
-
-        try:
-            item = lst[i]
-        except IndexError:
-            # complain
-
-Code with no exception handlers does not raise exceptions (after it has been
-translated, that is.  When you run it on top of CPython, it may raise
-exceptions, of course). By supplying an exception handler, you ask for error
-checking. Without, you assure the system that the operation cannot fail.
-This rule does not apply to *function calls*: any called function is
-assumed to be allowed to raise any exception.
-
-For example::
-
-    x = 5.1
-    x = x + 1.2       # not checked for float overflow
-    try:
-        x = x + 1.2
-    except OverflowError:
-        # float result too big
-
-But::
-
-    z = some_function(x, y)    # can raise any exception
-    try:
-        z = some_other_function(x, y)
-    except IndexError:
-        # only catches explicitly-raised IndexErrors in some_other_function()
-        # other exceptions can be raised, too, and will not be caught here.
-
-The ovfcheck() function described above follows the same rule: in case of
-overflow, it explicitly raise OverflowError, which can be caught anywhere.
-
-Exceptions explicitly raised or re-raised will always be generated.
-
-PyPy is debuggable on top of CPython
-------------------------------------
-
-PyPy has the advantage that it is runnable on standard
-CPython.  That means, we can run all of PyPy with all exception
-handling enabled, so we might catch cases where we failed to
-adhere to our implicit assertions.
-
-.. _`wrapping rules`:
-.. _`wrapped`:
-
-
+.. _wrapped:
 
 Wrapping rules
-==============
+--------------
 
 Wrapping
---------- 
+~~~~~~~~
 
 PyPy is made of Python source code at two levels: there is on the one hand
 *application-level code* that looks like normal Python code, and that
@@ -500,7 +202,7 @@
 structure.
 
 For example, an application-level Python ``list``
-is implemented by the `standard object space`_ as an
+is implemented by the :ref:`standard object space ` as an
 instance of ``W_ListObject``, which has an instance attribute
 ``wrappeditems`` (an interpreter-level list which contains the
 application-level list's items as wrapped objects).
@@ -509,7 +211,7 @@
 
 
 Naming conventions
-------------------
+~~~~~~~~~~~~~~~~~~
 
 * ``space``: the object space is only visible at
   interpreter-level code, where it is by convention passed around by the name
@@ -529,14 +231,14 @@
 
 
 Operations on ``w_xxx``
------------------------
+~~~~~~~~~~~~~~~~~~~~~~~
 
 The core bytecode interpreter considers wrapped objects as black boxes.
 It is not allowed to inspect them directly.  The allowed
 operations are all implemented on the object space: they are
 called ``space.xxx()``, where ``xxx`` is a standard operation
 name (``add``, ``getattr``, ``call``, ``eq``...). They are documented in the
-`object space document`_.
+:ref:`object space document `.
 
 A short warning: **don't do** ``w_x == w_y`` or ``w_x is w_y``!
 rationale for this rule is that there is no reason that two
@@ -548,12 +250,11 @@
 use ``space.is_true(space.is_(w_x, w_y))`` or better
 ``space.is_w(w_x, w_y)``.
 
-.. _`object space document`: objspace.html#interface
 
-.. _`applevel-exceptions`: 
+.. _applevel-exceptions:
 
 Application-level exceptions
-----------------------------
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 Interpreter-level code can use exceptions freely.  However,
 all application-level exceptions are represented as an
@@ -585,10 +286,10 @@
 instances of subclasses.
 
 
-.. _`modules`:
+.. _modules:
 
 Modules in PyPy
-===============
+---------------
 
 Modules visible from application programs are imported from
 interpreter or application level files.  PyPy reuses almost all python
@@ -597,18 +298,19 @@
 because they rely on implementation details of CPython.
 
 If we don't just modify an original CPython module but need to rewrite
-it from scratch we put it into `lib_pypy/`_ as a pure application level
+it from scratch we put it into :source:`lib_pypy/` as a pure application level
 module.
 
 When we need access to interpreter-level objects we put the module into
-`pypy/module`_.  Such modules use a `mixed module mechanism`_
+:source:`pypy/module`.  Such modules use a `mixed module mechanism`_
 which makes it convenient to use both interpreter- and application-level parts
 for the implementation.  Note that there is no extra facility for
 pure-interpreter level modules, you just write a mixed module and leave the
 application-level part empty.
 
+
 Determining the location of a module implementation
----------------------------------------------------
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 You can interactively find out where a module comes from, when running py.py.
 here are examples for the possible locations::
@@ -626,8 +328,9 @@
     '/home/hpk/pypy-dist/lib-python/2.7/os.py'
     >>>>
 
+
 Module directories / Import order
----------------------------------
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 Here is the order in which PyPy looks up Python modules:
 
@@ -650,10 +353,11 @@
 
     The modified CPython library.
 
-.. _`modify modules`:
+
+.. _modify modules:
 
 Modifying a CPython library module or regression test
--------------------------------------------------------
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 Although PyPy is very compatible with CPython we sometimes need
 to change modules contained in our copy of the standard library,
@@ -665,34 +369,31 @@
 to see what is changed we have a branch called `vendor/stdlib`
 wich contains the unmodified cpython stdlib
 
-.. _`mixed module mechanism`:
-.. _`mixed modules`:
+
+.. _mixed module mechanism:
+.. _mixed-modules:
 
 Implementing a mixed interpreter/application level Module
----------------------------------------------------------
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 If a module needs to access PyPy's interpreter level
 then it is implemented as a mixed module.
 
-Mixed modules are directories in `pypy/module`_ with an  `__init__.py`
+Mixed modules are directories in :source:`pypy/module` with an  `__init__.py`
 file containing specifications where each name in a module comes from.
 Only specified names will be exported to a Mixed Module's applevel
 namespace.
 
-Sometimes it is necessary to really write some functions in C (or
-whatever target language). See `rffi`_ and `external functions
-documentation`_ for details. The latter approach is cumbersome and
-being phased out and former has currently quite a few rough edges.
+Sometimes it is necessary to really write some functions in C (or whatever
+target language). See :ref:`rffi ` details.
 
-.. _`rffi`: rffi.html
-.. _`external functions documentation`: translation.html#extfunccalls
 
 application level definitions
-.............................
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 Application level specifications are found in the `appleveldefs`
 dictionary found in ``__init__.py`` files of directories in ``pypy/module``.
-For example, in `pypy/module/__builtin__/__init__.py`_ you find the following
+For example, in :source:`pypy/module/__builtin__/__init__.py` you find the following
 entry specifying where ``__builtin__.locals`` comes from::
 
      ...
@@ -703,12 +404,13 @@
 interpreted at application level and the wrapped function value for ``locals``
 will be extracted accordingly.
 
+
 interpreter level definitions
-.............................
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 Interpreter level specifications are found in the ``interpleveldefs``
 dictionary found in ``__init__.py`` files of directories in ``pypy/module``.
-For example, in `pypy/module/__builtin__/__init__.py`_ the following
+For example, in :source:`pypy/module/__builtin__/__init__.py` the following
 entry specifies where ``__builtin__.len`` comes from::
 
      ...
@@ -724,7 +426,7 @@
         return space.len(w_obj)
 
 Exposed interpreter level functions usually take a ``space`` argument
-and some wrapped values (see `wrapping rules`_) .
+and some wrapped values (see `Wrapping rules`_) .
 
 You can also use a convenient shortcut in ``interpleveldefs`` dictionaries:
 namely an expression in parentheses to specify an interpreter level
@@ -743,37 +445,40 @@
 --withoutmod-mymodule (the latter being the default)) for py.py and
 translate.py.
 
+
 Testing modules in ``lib_pypy/``
---------------------------------
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-You can go to the `pypy/module/test_lib_pypy/`_ directory and invoke the testing tool
+You can go to the :source:`pypy/module/test_lib_pypy/` directory and invoke the testing tool
 ("py.test" or "python ../../pypy/test_all.py") to run tests against the
-lib_pypy hierarchy.  Note, that tests in `pypy/module/test_lib_pypy/`_ are allowed
+lib_pypy hierarchy.  Note, that tests in :source:`pypy/module/test_lib_pypy/` are allowed
 and encouraged to let their tests run at interpreter level although
-`lib_pypy/`_ modules eventually live at PyPy's application level.
+:source:`lib_pypy/` modules eventually live at PyPy's application level.
 This allows us to quickly test our python-coded reimplementations
 against CPython.
 
+
 Testing modules in ``pypy/module``
-----------------------------------
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 Simply change to ``pypy/module`` or to a subdirectory and `run the
 tests as usual`_.
 
 
 Testing modules in ``lib-python``
------------------------------------
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 In order to let CPython's regression tests run against PyPy
-you can switch to the `lib-python/`_ directory and run
+you can switch to the :source:`lib-python/` directory and run
 the testing tool in order to start compliance tests.
 (XXX check windows compatibility for producing test reports).
 
+
 Naming conventions and directory layout
-===========================================
+---------------------------------------
 
 Directory and File Naming
--------------------------
+~~~~~~~~~~~~~~~~~~~~~~~~~
 
 - directories/modules/namespaces are always **lowercase**
 
@@ -786,8 +491,9 @@
 
 - keep filenames concise and completion-friendly.
 
+
 Naming of python objects
-------------------------
+~~~~~~~~~~~~~~~~~~~~~~~~
 
 - class names are **CamelCase**
 
@@ -803,8 +509,9 @@
   includes w_self.  Don't use ``w_`` in application level
   python only code.
 
+
 Committing & Branching to the repository
------------------------------------------------------
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 - write good log messages because several people
   are reading the diffs.
@@ -815,22 +522,23 @@
   ``try1`` doesn't already exists) you should do::
 
     hg branch try1
-    
+
   The branch will be recorded in the repository only after a commit. To switch
   back to the default branch::
-  
+
     hg update default
-    
+
   For further details use the help or refer to the `official wiki`_::
-  
+
     hg help branch
 
-.. _`official wiki`: http://mercurial.selenic.com/wiki/Branch
+.. _official wiki: http://mercurial.selenic.com/wiki/Branch
 
-.. _`using development tracker`:
+
+.. _using-development-tracker:
 
 Using the development bug/feature tracker
-=========================================
+-----------------------------------------
 
 We have a `development tracker`_, based on Richard Jones'
 `roundup`_ application.  You can file bugs,
@@ -838,15 +546,14 @@
 for the next milestone, both from an E-Mail and from a
 web interface.
 
-.. _`development tracker`: https://bugs.pypy.org/
-.. _`roundup`: http://roundup.sourceforge.net/
+.. _development tracker: https://bugs.pypy.org/
+.. _roundup: http://roundup.sourceforge.net/
 
 
-.. _`testing in PyPy`:
-.. _`test-design`: 
+.. _testing:
 
 Testing in PyPy
-===============
+---------------
 
 Our tests are based on the `py.test`_ tool which lets you write
 unittests without boilerplate.  All tests of modules
@@ -859,12 +566,11 @@
 - **Application Level tests**. They run at application level which means
   that they look like straight python code but they are interpreted by PyPy.
 
-.. _`standard object space`: objspace.html#standard-object-space
-.. _`objectspace`: objspace.html
-.. _`py.test`: http://pytest.org/
+.. _py.test: http://pytest.org/
+
 
 Interpreter level tests
------------------------
+~~~~~~~~~~~~~~~~~~~~~~~
 
 You can write test functions and methods like this::
 
@@ -880,8 +586,9 @@
 module global level and use plain 'assert' statements thanks to the
 usage of the `py.test`_ tool.
 
+
 Application Level tests
------------------------
+~~~~~~~~~~~~~~~~~~~~~~~
 
 For testing the conformance and well-behavedness of PyPy it
 is often sufficient to write "normal" application-level
@@ -916,7 +623,8 @@
             assert self.d["a"] == 1
             assert self.d["b"] == 2
 
-.. _`run the tests as usual`:
+
+.. _run the tests as usual:
 
 Another possibility is to use cls.space.appexec, for example::
 
@@ -959,7 +667,7 @@
 
 
 Command line tool test_all
---------------------------
+~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 You can run almost all of PyPy's tests by invoking::
 
@@ -969,8 +677,9 @@
 located in the ``py/bin/`` directory.  For switches to
 modify test execution pass the ``-h`` option.
 
+
 Coverage reports
-----------------
+~~~~~~~~~~~~~~~~
 
 In order to get coverage reports the `pytest-cov`_ plugin is included.
 it adds some extra requirements ( coverage_ and `cov-core`_ )
@@ -978,12 +687,13 @@
 
   python test_all.py --cov file_or_direcory_to_cover file_or_directory
 
-.. _`pytest-cov`: http://pypi.python.org/pypi/pytest-cov
-.. _`coverage`: http://pypi.python.org/pypi/coverage
-.. _`cov-core`: http://pypi.python.org/pypi/cov-core
+.. _pytest-cov: http://pypi.python.org/pypi/pytest-cov
+.. _coverage: http://pypi.python.org/pypi/coverage
+.. _cov-core: http://pypi.python.org/pypi/cov-core
+
 
 Test conventions
-----------------
+~~~~~~~~~~~~~~~~
 
 - adding features requires adding appropriate tests.  (It often even
   makes sense to first write the tests so that you are sure that they
@@ -993,29 +703,28 @@
   which contain unit tests.  Such scripts can usually be executed
   directly or are collectively run by pypy/test_all.py
 
-.. _`change documentation and website`:
+
+.. _change documentation and website:
 
 Changing documentation and website
-==================================
+----------------------------------
 
 documentation/website files in your local checkout
----------------------------------------------------
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 Most of the PyPy's documentation is kept in `pypy/doc`.
 You can simply edit or add '.rst' files which contain ReST-markuped
 files.  Here is a `ReST quickstart`_ but you can also just look
 at the existing documentation and see how things work.
 
-.. _`ReST quickstart`: http://docutils.sourceforge.net/docs/user/rst/quickref.html
-
 Note that the web site of http://pypy.org/ is maintained separately.
 For now it is in the repository https://bitbucket.org/pypy/pypy.org
 
+.. _ReST quickstart: http://docutils.sourceforge.net/docs/user/rst/quickref.html
+
+
 Automatically test documentation/website changes
-------------------------------------------------
-
-.. _`sphinx home page`:
-.. _`sphinx`: http://sphinx.pocoo.org/
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 We automatically check referential integrity and ReST-conformance.  In order to
 run the tests you need sphinx_ installed.  Then go to the local checkout
@@ -1035,5 +744,4 @@
 
 which will check that remote URLs are reachable.
 
-
-.. include:: _ref.txt
+.. _sphinx: http://sphinx.pocoo.org/
diff --git a/pypy/doc/conf.py b/pypy/doc/conf.py
--- a/pypy/doc/conf.py
+++ b/pypy/doc/conf.py
@@ -40,9 +40,9 @@
 
 # Add any Sphinx extension module names here, as strings. They can be extensions
 # coming with Sphinx (named 'sphinx.ext.*') or your custom ones.
-extensions = ['sphinx.ext.autodoc', 'sphinx.ext.doctest', 'sphinx.ext.intersphinx',
-              'sphinx.ext.todo', 'sphinx.ext.ifconfig', 'sphinx.ext.graphviz',
-              'pypyconfig']
+extensions = ['sphinx.ext.autodoc', 'sphinx.ext.doctest', 'sphinx.ext.extlinks',
+              'sphinx.ext.intersphinx', 'sphinx.ext.todo', 'sphinx.ext.ifconfig',
+              'sphinx.ext.graphviz', 'pypyconfig']
 
 # Add any paths that contain templates here, relative to this directory.
 templates_path = ['_templates']
@@ -215,7 +215,8 @@
 
 
 # Example configuration for intersphinx: refer to the Python standard library.
-intersphinx_mapping = {'http://docs.python.org/': None}
+intersphinx_mapping = {'python': ('http://docs.python.org/2.7', None),
+                       'rpython': ('http://rpython.readthedocs.org/en/latest/', None)}
 
 # -- Options for manpage output-------------------------------------------------
 
@@ -225,3 +226,4 @@
    u'The PyPy Project', 1)
 ]
 
+extlinks = {'source': ('https://bitbucket.org/pypy/pypy/src/default/%s', '')}
diff --git a/pypy/doc/config/confrest.py b/pypy/doc/config/confrest.py
deleted file mode 100644
--- a/pypy/doc/config/confrest.py
+++ /dev/null
@@ -1,62 +0,0 @@
-from pypy.doc.confrest import *
-from pypy.config.makerestdoc import make_cmdline_overview
-from pypy.config import pypyoption
-from rpython.config.config import Config
-from rpython.config import translationoption
-
-
-all_optiondescrs = [pypyoption.pypy_optiondescription,
-                    translationoption.translation_optiondescription,
-                   ]
-start_to_descr = dict([(descr._name, descr) for descr in all_optiondescrs])
-
-class PyPyPage(PyPyPage): 
-    def fill(self):
-        super(PyPyPage, self).fill()
-        self.menubar[:] = html.div(
-            html.a("general documentation", href="../index.html",
-                   class_="menu"), " ",
-            html.a("config index", href="index.html",
-                   class_="menu"), " ",
-            html.a("command-line overview", href="commandline.html",
-                   class_="menu"), " ",
-            " ", id="menubar")
-
-class Project(Project): 
-    stylesheet = "../style.css"
-    title = "PyPy Configuration"
-    prefix_title = "PyPy Configuration"
-    Page = PyPyPage 
-
-    def get_content(self, txtpath, encoding):
-        if txtpath.basename == "commandline.rst":
-            result = []
-            for line in txtpath.read().splitlines():
-                if line.startswith('.. GENERATE:'):
-                    start = line[len('.. GENERATE:'):].strip()
-                    descr = start_to_descr[start]
-                    line = make_cmdline_overview(descr, title=False).text()
-                result.append(line)
-            return "\n".join(result)
-        fullpath = txtpath.purebasename
-        start = fullpath.split(".")[0]
-        path = fullpath.rsplit(".", 1)[0]
-        basedescr = start_to_descr.get(start)
-        if basedescr is None:
-            return txtpath.read()
-        if fullpath.count(".") == 0:
-            descr = basedescr
-            path = ""
-        else:
-            conf = Config(basedescr)
-            subconf, step = conf._cfgimpl_get_home_by_path(
-                    fullpath.split(".", 1)[1])
-            descr = getattr(subconf._cfgimpl_descr, step)
-        text = unicode(descr.make_rest_doc(path).text())
-        if txtpath.check(file=True):
-            content = txtpath.read()
-            if content:
-                text += "\nDescription\n==========="
-                return u"%s\n\n%s" % (text, unicode(txtpath.read(), encoding))
-        return text
-
diff --git a/pypy/doc/config/generate.py b/pypy/doc/config/generate.py
--- a/pypy/doc/config/generate.py
+++ b/pypy/doc/config/generate.py
@@ -1,6 +1,5 @@
 import py
 from pypy.config import pypyoption, makerestdoc
-from pypy.doc.config.confrest import all_optiondescrs
 from rpython.config import translationoption, config
 
 all_optiondescrs = [pypyoption.pypy_optiondescription,
diff --git a/pypy/doc/config/objspace.std.builtinshortcut.txt b/pypy/doc/config/objspace.std.builtinshortcut.txt
deleted file mode 100644
--- a/pypy/doc/config/objspace.std.builtinshortcut.txt
+++ /dev/null
@@ -1,5 +0,0 @@
-A shortcut speeding up primitive operations between built-in types.
-
-This is a space-time trade-off: at the moment, this option makes a
-translated pypy-c executable bigger by about 1.7 MB.  (This can probably
-be improved with careful analysis.)
diff --git a/pypy/doc/config/objspace.std.multimethods.txt b/pypy/doc/config/objspace.std.multimethods.txt
deleted file mode 100644
--- a/pypy/doc/config/objspace.std.multimethods.txt
+++ /dev/null
@@ -1,8 +0,0 @@
-Choose the multimethod implementation.
-
-* ``doubledispatch`` turns
-  a multimethod call into a sequence of normal method calls.
-
-* ``mrd`` uses a technique known as Multiple Row Displacement
-  which precomputes a few compact tables of numbers and

From noreply at buildbot.pypy.org  Wed Nov 19 00:22:12 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Wed, 19 Nov 2014 00:22:12 +0100 (CET)
Subject: [pypy-commit] pypy stmgc-c7: Fix (and reduce the diff to default)
Message-ID: <20141118232212.1E84C1D2487@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: stmgc-c7
Changeset: r74580:dd13b4ea897b
Date: 2014-11-19 00:19 +0100
http://bitbucket.org/pypy/pypy/changeset/dd13b4ea897b/

Log:	Fix (and reduce the diff to default)

diff --git a/rpython/translator/c/src/g_prerequisite.h b/rpython/translator/c/src/g_prerequisite.h
--- a/rpython/translator/c/src/g_prerequisite.h
+++ b/rpython/translator/c/src/g_prerequisite.h
@@ -12,11 +12,13 @@
 #include 
 
 
-#ifdef __GNUC__
+#ifdef __GNUC__       /* other platforms too, probably */
+typedef _Bool bool_t;
 # define RPY_VARLENGTH   /* nothing: [RPY_VARLENGTH] => [] */
 # define RPY_LENGTH0     0       /* array decl [0] are ok  */
 # define RPY_DUMMY_VARLENGTH     char _dummy[0];
 #else
+typedef unsigned char bool_t;
 # define RPY_VARLENGTH   1       /* [RPY_VARLENGTH] => [1] */
 # define RPY_LENGTH0     1       /* array decl [0] are bad */
 # define RPY_DUMMY_VARLENGTH     /* nothing */

From noreply at buildbot.pypy.org  Wed Nov 19 00:22:13 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Wed, 19 Nov 2014 00:22:13 +0100 (CET)
Subject: [pypy-commit] pypy stmgc-c7: merge heads
Message-ID: <20141118232213.7287E1D2487@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: stmgc-c7
Changeset: r74581:1187868fcd79
Date: 2014-11-19 00:20 +0100
http://bitbucket.org/pypy/pypy/changeset/1187868fcd79/

Log:	merge heads

diff --git a/pypy/stm/plot_stm_log.py b/pypy/stm/plot_stm_log.py
new file mode 100755
--- /dev/null
+++ b/pypy/stm/plot_stm_log.py
@@ -0,0 +1,194 @@
+#!/usr/bin/env python
+import matplotlib
+import sys
+matplotlib.use('gtkagg')
+
+args = None
+import matplotlib.pyplot as plt
+
+import print_stm_log as psl
+
+
+########## DRAWING STUFF ##########
+
+BOX_HEIGHT = 0.8
+HALF_HEIGHT = 0.1 + BOX_HEIGHT / 2
+QUARTER_HEIGHT = 0.1 + BOX_HEIGHT / 4
+
+
+def plot_boxes(boxes, y, ax):
+    coords = [(x, w) for x, w, c, i in boxes]
+    colors = [c for x, w, c, i in boxes]
+    bars = ax.broken_barh(coords, (y+0.1, BOX_HEIGHT),
+                          facecolors=colors, lw=1, edgecolor=(0, 0, 0),
+                          picker=True, antialiased=False, rasterized=True)
+
+    bars.boxes = boxes
+
+
+def plot_hlines(hlines, y, ax):
+    args = [[[x1, x2], [y+HALF_HEIGHT, y+HALF_HEIGHT], color] for
+            x1, x2, color in hlines]
+    # flatten:
+    args = [item for sublist in args for item in sublist]
+    ax.plot(*args, linewidth=5, antialiased=False, rasterized=True)
+
+
+####################################
+
+
+def add_box(boxes, x1, x2, color, info):
+    boxes.append((x1, x2-x1, color, info))
+
+
+def add_hline(hlines, x1, x2, color):
+    hlines.append((x1, x2, color))
+
+
+def add_transaction(boxes, hlines, inited, inevitabled,
+                    ended, aborted, pauses, info=""):
+    assert inited is not None
+
+    if inevitabled is not None:
+        add_box(boxes, inited, inevitabled, 'b', info)
+        add_box(boxes, inevitabled, ended, 'orange', info)
+    elif not aborted:
+        add_box(boxes, inited, ended, 'g', info)
+    else:
+        add_box(boxes, inited, ended, 'r', info)
+
+    for start, end in pauses:
+        add_hline(hlines, start, end, 'magenta')
+
+
+class Transaction(object):
+    def __init__(self, thread_num, start_time):
+        self.thread_num = thread_num
+        self.start_time = start_time
+        self.stop_time = 0
+        self.aborted = False
+        self.pauses = []
+
+
+def plot_log(logentries, ax):
+    curr_trs = {}
+    finished_trs = {}
+    for entry in logentries:
+        th_num = entry.threadnum
+
+        if entry.event == psl.STM_TRANSACTION_START:
+            if th_num in curr_trs:
+                print "WARNING: Start of transaction while there is one already running"
+            curr_trs[th_num] = Transaction(th_num, entry.timestamp)
+        elif entry.event == psl.STM_TRANSACTION_COMMIT:
+            tr = curr_trs.get(th_num)
+            if tr is not None:
+                tr.stop_time = entry.timestamp
+                xs = finished_trs.setdefault(th_num, [])
+                xs.append(curr_trs[th_num])
+                del curr_trs[th_num]
+        elif entry.event == psl.STM_TRANSACTION_ABORT:
+            tr = curr_trs.get(th_num)
+            if tr is not None:
+                tr.stop_time = entry.timestamp
+                tr.aborted = True
+                xs = finished_trs.setdefault(th_num, [])
+                xs.append(curr_trs[th_num])
+                del curr_trs[th_num]
+        elif entry.event in (psl.STM_WAIT_SYNC_PAUSE, psl.STM_WAIT_CONTENTION,
+                             psl.STM_WAIT_FREE_SEGMENT):
+            tr = curr_trs.get(th_num)
+            if tr is not None:
+                tr.pauses.append((entry.timestamp, entry.timestamp))
+        elif entry.event == psl.STM_WAIT_DONE:
+            tr = curr_trs.get(th_num)
+            if tr is not None:
+                tr.pauses[-1] = (tr.pauses[-1][0], entry.timestamp)
+
+
+    # plt.ion()
+    for th_num, trs in finished_trs.items():
+        # plt.draw()
+        # plt.show()
+        print "Thread", th_num
+        print "> Transactions:", len(trs)
+
+        boxes = []
+        hlines = []
+        for tr in trs:
+            add_transaction(boxes, hlines,
+                            tr.start_time, None, tr.stop_time,
+                            tr.aborted, tr.pauses)
+        plot_boxes(boxes, th_num, ax)
+        plot_hlines(hlines, th_num, ax)
+        print "> Pauses:", len(hlines)
+
+    # plt.ioff()
+
+    return finished_trs
+
+
+def onpick(event):
+    if hasattr(event.artist, "info"):
+        print "== pick ==\n", event.artist.info
+    if hasattr(event.artist, "boxes"):
+        x = event.mouseevent.xdata
+        for x1, w, c, i in event.artist.boxes:
+            if x >= x1 and x <= x1+w:
+                print "== pick ==\n", i
+                break
+        else:
+            print "== pick ==\nCould not find info"
+
+
+
+
+def plot(logentries):
+    global fig
+
+
+    print "Draw..."
+    fig = plt.figure()
+    grid_spec = matplotlib.gridspec.GridSpec(1, 1)
+    axs = [fig.add_subplot(grid_spec[0])]
+
+    trs = plot_log(logentries, axs[0])
+
+    thread_count = len(trs)
+
+    axs[0].set_ylabel("Thread")
+    axs[0].set_ylim(0, thread_count)
+    axs[0].set_yticks([r+0.5 for r in range(thread_count)])
+    axs[0].set_yticklabels(range(1, thread_count+1))
+    #axs[0].set_xticks([])
+    print "Drawn."
+
+    axs[0].set_xlabel("Runtime [s]")
+
+    # adjust labels:
+    first_trs = [th[0].start_time for th in trs.values()]
+    last_trs = [th[-1].stop_time for th in trs.values()]
+    start_time, stop_time = min(first_trs), max(last_trs)
+    offset = (stop_time - start_time) * 0.1
+    axs[0].set_xlim((start_time - offset, stop_time + offset))
+
+    def label_format(x, pos):
+        return "%.2f" % (x - start_time)
+    major_formatter = matplotlib.ticker.FuncFormatter(label_format)
+    axs[0].xaxis.set_major_formatter(major_formatter)
+
+    # event connect
+    fig.canvas.mpl_connect('pick_event', onpick)
+
+    plt.draw()
+    plt.show()
+
+# ____________________________________________________________
+
+def main(argv):
+    assert len(argv) == 1, "expected a filename argument"
+    plot(psl.parse_log(argv[0]))
+    return 0
+
+if __name__ == '__main__':
+    sys.exit(main(sys.argv[1:]))
diff --git a/pypy/stm/print_stm_log.py b/pypy/stm/print_stm_log.py
--- a/pypy/stm/print_stm_log.py
+++ b/pypy/stm/print_stm_log.py
@@ -126,6 +126,7 @@
         self.aborted_time = 0.0
         self.paused_time = 0.0
         self.num_events = 0
+        self.timestamps = []
 
     def sortkey(self):
         return self.aborted_time + self.paused_time
@@ -149,7 +150,8 @@
     return r + '%'
 
 def dump(logentries):
-    total_time = logentries[-1].timestamp - logentries[0].timestamp
+    start_time, stop_time = logentries[0].timestamp, logentries[-1].timestamp
+    total_time = stop_time - start_time
     print 'Total real time:             %.3fs' % (total_time,)
     #
     threads = {}
@@ -173,6 +175,7 @@
             if c is None:
                 c = conflicts[summary] = ConflictSummary(*summary)
             c.num_events += 1
+            c.timestamps.append(entry.timestamp)
             t = threads.get(entry.threadnum)
             if t is not None and t.in_transaction():
                 t._conflict = ("local", c, entry)
@@ -184,7 +187,8 @@
                 t2 = threads.get(entry.otherthreadnum)
                 if t2 is not None and t2.in_transaction():
                     t2._conflict = ("remote", c, entry)
-        elif entry.event in (STM_WAIT_SYNC_PAUSE, STM_WAIT_CONTENTION):
+        elif entry.event in (STM_WAIT_SYNC_PAUSE, STM_WAIT_CONTENTION,
+                             STM_WAIT_FREE_SEGMENT):
             t = threads.get(entry.threadnum)
             if t is not None and t.in_transaction():
                 t.transaction_pause(entry)
@@ -193,18 +197,25 @@
             if t is not None and t.in_transaction():
                 t.transaction_unpause(entry)
     #
-    total_cpu_time = sum([t.cpu_time for t in threads.values()])
+    total_cpu_time = sum([v.cpu_time for v in threads.values()])
     print 'Total CPU time in STM mode:  %.3fs (%s)' % (
         total_cpu_time, percent(total_cpu_time, total_time))
     print
     #
     values = sorted(conflicts.values(), key=ConflictSummary.sortkey)
     for c in values[-1:-15:-1]:
+        intervals = 48
+        timeline = [0] * intervals
+        for t in c.timestamps:
+            idx = int((t - start_time) / total_time * intervals)
+            timeline[idx] += 1
+
         print '%.3fs lost in aborts, %.3fs paused (%dx %s)' % (
             c.aborted_time, c.paused_time, c.num_events, event_name[c.event])
         print_marker(c.marker1)
         if c.marker2:
             print_marker(c.marker2)
+        print "time line:", "".join(['x' if i else '.' for i in timeline])
         print
 
 

From noreply at buildbot.pypy.org  Wed Nov 19 01:00:41 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Wed, 19 Nov 2014 01:00:41 +0100 (CET)
Subject: [pypy-commit] pypy stmgc-c7: fixes
Message-ID: <20141119000041.E76B41C1147@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: stmgc-c7
Changeset: r74582:9a95c6696552
Date: 2014-11-19 01:00 +0100
http://bitbucket.org/pypy/pypy/changeset/9a95c6696552/

Log:	fixes

diff --git a/pypy/module/_stm/time.py b/pypy/module/_stm/time.py
--- a/pypy/module/_stm/time.py
+++ b/pypy/module/_stm/time.py
@@ -12,8 +12,8 @@
     includes=["time.h"],
     libraries=["rt"],
     post_include_bits = ["""
-extern double pypy_clock_get_time(void);
-extern double pypy_clock_get_clock(void);
+RPY_EXTERN double pypy_clock_get_time(void);
+RPY_EXTERN double pypy_clock_get_clock(void);
 """],
     separate_module_sources = ["""
 double pypy_clock_get_time(void) {
diff --git a/rpython/translator/c/src/signals.h b/rpython/translator/c/src/signals.h
--- a/rpython/translator/c/src/signals.h
+++ b/rpython/translator/c/src/signals.h
@@ -35,7 +35,9 @@
    export a function with the correct name for testing */
 RPY_EXTERN
 void *pypysig_getaddr_occurred(void);
+RPY_EXTERN
 long pypysig_get_occurred(void);
+RPY_EXTERN
 void pypysig_set_occurred(long);
 #define pypysig_getaddr_occurred()   ((void *)(&pypysig_counter))
 #define pypysig_get_occurred()       (pypysig_counter.value)

From noreply at buildbot.pypy.org  Wed Nov 19 01:01:36 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Wed, 19 Nov 2014 01:01:36 +0100 (CET)
Subject: [pypy-commit] pypy stmgc-c7: fix
Message-ID: <20141119000136.78FE41C1147@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: stmgc-c7
Changeset: r74583:c9e013e285d1
Date: 2014-11-19 01:01 +0100
http://bitbucket.org/pypy/pypy/changeset/c9e013e285d1/

Log:	fix

diff --git a/rpython/rlib/rstm.py b/rpython/rlib/rstm.py
--- a/rpython/rlib/rstm.py
+++ b/rpython/rlib/rstm.py
@@ -157,6 +157,7 @@
 def pop_marker():
     llop.stm_pop_marker(lltype.Void)
 
+ at dont_look_inside
 def stm_count():
     return llop.stm_count(lltype.Signed)
 

From noreply at buildbot.pypy.org  Wed Nov 19 01:13:59 2014
From: noreply at buildbot.pypy.org (rlamy)
Date: Wed, 19 Nov 2014 01:13:59 +0100 (CET)
Subject: [pypy-commit] pypy default: remove obsolete special case
Message-ID: <20141119001359.F29ED1C104D@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy 
Branch: 
Changeset: r74584:d86dde387f83
Date: 2014-11-19 00:13 +0000
http://bitbucket.org/pypy/pypy/changeset/d86dde387f83/

Log:	remove obsolete special case

diff --git a/rpython/flowspace/flowcontext.py b/rpython/flowspace/flowcontext.py
--- a/rpython/flowspace/flowcontext.py
+++ b/rpython/flowspace/flowcontext.py
@@ -856,14 +856,9 @@
     def WITH_CLEANUP(self, oparg):
         # Note: RPython context managers receive None in lieu of tracebacks
         # and cannot suppress the exception.
-        # This opcode changed a lot between CPython versions
-        if sys.version_info >= (2, 6):
-            unroller = self.popvalue()
-            w_exitfunc = self.popvalue()
-            self.pushvalue(unroller)
-        else:
-            w_exitfunc = self.popvalue()
-            unroller = self.peekvalue(0)
+        unroller = self.popvalue()
+        w_exitfunc = self.popvalue()
+        self.pushvalue(unroller)
 
         if isinstance(unroller, Raise):
             w_exc = unroller.w_exc

From noreply at buildbot.pypy.org  Wed Nov 19 01:30:52 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Wed, 19 Nov 2014 01:30:52 +0100 (CET)
Subject: [pypy-commit] pypy stmgc-c7: more dont_look_insides
Message-ID: <20141119003052.1C5AC1C104D@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: stmgc-c7
Changeset: r74585:28d9f49a5bf2
Date: 2014-11-19 01:30 +0100
http://bitbucket.org/pypy/pypy/changeset/28d9f49a5bf2/

Log:	more dont_look_insides

diff --git a/rpython/rlib/rstm.py b/rpython/rlib/rstm.py
--- a/rpython/rlib/rstm.py
+++ b/rpython/rlib/rstm.py
@@ -182,10 +182,12 @@
                                        ('index', lltype.Unsigned),
                                        ('object', llmemory.GCREF))
 
+ at dont_look_inside
 def ll_hashtable_get(h, key):
     # 'key' must be a plain integer.  Returns a GCREF.
     return llop.stm_hashtable_read(llmemory.GCREF, h, h.ll_raw_hashtable, key)
 
+ at dont_look_inside
 def ll_hashtable_set(h, key, value):
     llop.stm_hashtable_write(lltype.Void, h, h.ll_raw_hashtable, key, value)
 
@@ -203,6 +205,7 @@
 
 _false = CDefinedIntSymbolic('0', default=0)    # remains in the C code
 
+ at dont_look_inside
 def create_hashtable():
     if not we_are_translated():
         return HashtableForTest()      # for tests

From noreply at buildbot.pypy.org  Wed Nov 19 07:59:35 2014
From: noreply at buildbot.pypy.org (fijal)
Date: Wed, 19 Nov 2014 07:59:35 +0100 (CET)
Subject: [pypy-commit] pypy optresult: fix llgraph backend to run the first
	test
Message-ID: <20141119065935.41B7F1C100F@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: optresult
Changeset: r74586:2902163d7469
Date: 2014-11-18 15:31 +0200
http://bitbucket.org/pypy/pypy/changeset/2902163d7469/

Log:	fix llgraph backend to run the first test

diff --git a/rpython/jit/backend/llgraph/runner.py b/rpython/jit/backend/llgraph/runner.py
--- a/rpython/jit/backend/llgraph/runner.py
+++ b/rpython/jit/backend/llgraph/runner.py
@@ -25,7 +25,9 @@
         # We need to clone the list of operations because the
         # front-end will mutate them under our feet again.  We also
         # need to make sure things get freed.
-        def mapping(box, _cache={}):
+        _cache={}
+        
+        def mapping(box):
             if isinstance(box, Const) or box is None:
                 return box
             try:
@@ -44,10 +46,10 @@
                     newdescr = WeakrefDescr(op.getdescr())
             else:
                 newdescr = None
-            newop = op.copy_and_change(op.getopnum(),
+            newop = op._copy_and_change(op.getopnum(),
                                        map(mapping, op.getarglist()),
-                                       mapping(op.result),
                                        newdescr)
+            _cache[op] = newop
             if op.getfailargs() is not None:
                 newop.setfailargs(map(mapping, op.getfailargs()))
             self.operations.append(newop)
@@ -749,8 +751,8 @@
                     i = 0
                 self.do_renaming(targetargs, j.args)
                 continue
-            if op.result is not None:
-                self.setenv(op.result, resval)
+            if op.type != 'v':
+                self.setenv(op, resval)
             else:
                 assert resval is None
             i += 1
diff --git a/rpython/jit/backend/test/runner_test.py b/rpython/jit/backend/test/runner_test.py
--- a/rpython/jit/backend/test/runner_test.py
+++ b/rpython/jit/backend/test/runner_test.py
@@ -7,7 +7,7 @@
                                          JitCellToken, TargetToken,
                                          ConstInt, ConstPtr,
                                          BoxFloat, ConstFloat)
-from rpython.jit.metainterp.resoperation import ResOperation, rop
+from rpython.jit.metainterp.resoperation import ResOperation, rop, InputArgInt
 from rpython.jit.metainterp.typesystem import deref
 from rpython.jit.codewriter.effectinfo import EffectInfo
 from rpython.jit.tool.oparser import parse
@@ -123,12 +123,10 @@
         self.cpu.done_with_this_frame_descr_void = None
 
     def test_compile_linear_loop(self):
-        i0 = BoxInt()
-        i1 = BoxInt()
-        operations = [
-            ResOperation(rop.INT_ADD, [i0, ConstInt(1)], i1),
-            ResOperation(rop.FINISH, [i1], None, descr=BasicFinalDescr(1))
-            ]
+        i0 = InputArgInt(0)
+        op0 = ResOperation(rop.INT_ADD, [i0, ConstInt(1)])
+        op1 = ResOperation(rop.FINISH, [op0], descr=BasicFinalDescr(1))
+        operations = [op0, op1]
         inputargs = [i0]
         looptoken = JitCellToken()
         self.cpu.compile_loop(inputargs, operations, looptoken)
diff --git a/rpython/jit/metainterp/resoperation.py b/rpython/jit/metainterp/resoperation.py
--- a/rpython/jit/metainterp/resoperation.py
+++ b/rpython/jit/metainterp/resoperation.py
@@ -86,7 +86,7 @@
     # common methods
     # --------------
 
-    def _copy_and_change(self, opnum, args=None, result=None, descr=None):
+    def _copy_and_change(self, opnum, args=None, descr=None):
         "shallow copy: the returned operation is meant to be used in place of self"
         if args is None:
             args = self.getarglist()
@@ -270,8 +270,8 @@
     def setfailargs(self, fail_args):
         self._fail_args = fail_args
 
-    def _copy_and_change(self, opnum, args=None, result=None, descr=None):
-        newop = AbstractResOp._copy_and_change(self, opnum, args, result, descr)
+    def _copy_and_change(self, opnum, args=None, descr=None):
+        newop = AbstractResOp._copy_and_change(self, opnum, args, descr)
         newop.setfailargs(self.getfailargs())
         return newop
 
@@ -345,15 +345,15 @@
         return None
         
 class InputArgInt(IntOp, AbstractInputArg):
-    def __init__(self, intval):
+    def __init__(self, intval=0):
         self.setint(intval)            
 
 class InputArgFloat(FloatOp, AbstractInputArg):
-    def __init__(self, f):
+    def __init__(self, f=0.0):
         self.setfloatstorage(f)
 
 class InputArgRef(RefOp, AbstractInputArg):
-    def __init__(self, r):
+    def __init__(self, r=lltype.nullptr(llmemory.GCREF.TO)):
         self.setref_base(r)
 
 # ============

From noreply at buildbot.pypy.org  Wed Nov 19 07:59:36 2014
From: noreply at buildbot.pypy.org (fijal)
Date: Wed, 19 Nov 2014 07:59:36 +0100 (CET)
Subject: [pypy-commit] pypy optresult: start whacking at runner test
Message-ID: <20141119065936.8579B1C100F@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: optresult
Changeset: r74587:88dc5db70d06
Date: 2014-11-18 15:36 +0200
http://bitbucket.org/pypy/pypy/changeset/88dc5db70d06/

Log:	start whacking at runner test

diff --git a/rpython/jit/backend/test/runner_test.py b/rpython/jit/backend/test/runner_test.py
--- a/rpython/jit/backend/test/runner_test.py
+++ b/rpython/jit/backend/test/runner_test.py
@@ -3,11 +3,11 @@
                                          AbstractDescr,
                                          BasicFailDescr,
                                          BasicFinalDescr,
-                                         BoxInt, Box, BoxPtr,
                                          JitCellToken, TargetToken,
                                          ConstInt, ConstPtr,
-                                         BoxFloat, ConstFloat)
-from rpython.jit.metainterp.resoperation import ResOperation, rop, InputArgInt
+                                         ConstFloat)
+from rpython.jit.metainterp.resoperation import ResOperation, rop, InputArgInt,\
+     InputArgFloat
 from rpython.jit.metainterp.typesystem import deref
 from rpython.jit.codewriter.effectinfo import EffectInfo
 from rpython.jit.tool.oparser import parse
@@ -25,9 +25,6 @@
 IS_32_BIT = sys.maxint < 2**32
 IS_64_BIT = sys.maxint > 2**32
 
-def boxfloat(x):
-    return BoxFloat(longlong.getfloatstorage(x))
-
 def constfloat(x):
     return ConstFloat(longlong.getfloatstorage(x))
 
@@ -139,12 +136,10 @@
     def test_compile_linear_float_loop(self):
         if not self.cpu.supports_floats:
             py.test.skip("requires floats")
-        i0 = BoxFloat()
-        i1 = BoxFloat()
-        operations = [
-            ResOperation(rop.FLOAT_ADD, [i0, constfloat(2.3)], i1),
-            ResOperation(rop.FINISH, [i1], None, descr=BasicFinalDescr(1))
-            ]
+        i0 = InputArgFloat()
+        op0 = ResOperation(rop.FLOAT_ADD, [i0, constfloat(2.3)])
+        op1 = ResOperation(rop.FINISH, [op0], descr=BasicFinalDescr(1))
+        operations = [op0, op1]
         inputargs = [i0]
         looptoken = JitCellToken()
         self.cpu.compile_loop(inputargs, operations, looptoken)
@@ -157,20 +152,17 @@
         assert fail.identifier == 1
 
     def test_compile_loop(self):
-        i0 = BoxInt()
-        i1 = BoxInt()
-        i2 = BoxInt()
         looptoken = JitCellToken()
         targettoken = TargetToken()
-        operations = [
-            ResOperation(rop.LABEL, [i0], None, descr=targettoken),
-            ResOperation(rop.INT_ADD, [i0, ConstInt(1)], i1),
-            ResOperation(rop.INT_LE, [i1, ConstInt(9)], i2),
-            ResOperation(rop.GUARD_TRUE, [i2], None, descr=BasicFailDescr(2)),
-            ResOperation(rop.JUMP, [i1], None, descr=targettoken),
-            ]
+        i0 = InputArgInt()
+        op0 = ResOperation(rop.LABEL, [i0], descr=targettoken)
+        op1 = ResOperation(rop.INT_ADD, [i0, ConstInt(1)])
+        op2 = ResOperation(rop.INT_LE, [op1, ConstInt(9)])
+        op3 = ResOperation(rop.GUARD_TRUE, [op2], descr=BasicFailDescr(2))
+        op4 = ResOperation(rop.JUMP, [op1], descr=targettoken)
+        operations = [op0, op1, op2, op3, op4]
         inputargs = [i0]
-        operations[3].setfailargs([i1])
+        operations[3].setfailargs([op1])
 
         self.cpu.compile_loop(inputargs, operations, looptoken)
         deadframe = self.cpu.execute_token(looptoken, 2)
@@ -1702,12 +1694,12 @@
         from rpython.rlib.rfloat import INFINITY, NAN, isinf, isnan
         from rpython.jit.metainterp.resoperation import opname
 
-        fzer = boxfloat(0.0)
-        fone = boxfloat(1.0)
-        fmqr = boxfloat(-0.25)
-        finf = boxfloat(INFINITY)
-        fmnf = boxfloat(-INFINITY)
-        fnan = boxfloat(NAN)
+        fzer = 0.0
+        fone = 1.0
+        fmqr = -0.25
+        finf = INFINITY
+        fmnf = -INFINITY
+        fnan = NAN
 
         all_cases_unary =  [(a,)   for a in [fzer,fone,fmqr,finf,fmnf,fnan]]
         all_cases_binary = [(a, b) for a in [fzer,fone,fmqr,finf,fmnf,fnan]

From noreply at buildbot.pypy.org  Wed Nov 19 07:59:38 2014
From: noreply at buildbot.pypy.org (fijal)
Date: Wed, 19 Nov 2014 07:59:38 +0100 (CET)
Subject: [pypy-commit] pypy optresult: port some of test_runner
Message-ID: <20141119065938.249FC1C100F@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: optresult
Changeset: r74588:8edc0cc212e8
Date: 2014-11-19 08:59 +0200
http://bitbucket.org/pypy/pypy/changeset/8edc0cc212e8/

Log:	port some of test_runner

diff --git a/rpython/jit/backend/test/runner_test.py b/rpython/jit/backend/test/runner_test.py
--- a/rpython/jit/backend/test/runner_test.py
+++ b/rpython/jit/backend/test/runner_test.py
@@ -120,13 +120,13 @@
         self.cpu.done_with_this_frame_descr_void = None
 
     def test_compile_linear_loop(self):
-        i0 = InputArgInt(0)
-        op0 = ResOperation(rop.INT_ADD, [i0, ConstInt(1)])
-        op1 = ResOperation(rop.FINISH, [op0], descr=BasicFinalDescr(1))
-        operations = [op0, op1]
-        inputargs = [i0]
+        loop = parse("""
+        [i0]
+        i1 = int_add(i0, 1)
+        finish(i1, descr=faildescr)
+        """, namespace={"faildescr": BasicFinalDescr(1)})
         looptoken = JitCellToken()
-        self.cpu.compile_loop(inputargs, operations, looptoken)
+        self.cpu.compile_loop(loop.inputargs, loop.operations, looptoken)
         deadframe = self.cpu.execute_token(looptoken, 2)
         fail = self.cpu.get_latest_descr(deadframe)
         res = self.cpu.get_int_value(deadframe, 0)
@@ -136,13 +136,13 @@
     def test_compile_linear_float_loop(self):
         if not self.cpu.supports_floats:
             py.test.skip("requires floats")
-        i0 = InputArgFloat()
-        op0 = ResOperation(rop.FLOAT_ADD, [i0, constfloat(2.3)])
-        op1 = ResOperation(rop.FINISH, [op0], descr=BasicFinalDescr(1))
-        operations = [op0, op1]
-        inputargs = [i0]
+        loop = parse("""
+        [f0]
+        f1 = float_add(f0, 2.3)
+        finish(f1, descr=fdescr)
+        """, namespace={'fdescr': BasicFinalDescr(1)})
         looptoken = JitCellToken()
-        self.cpu.compile_loop(inputargs, operations, looptoken)
+        self.cpu.compile_loop(loop.inputargs, loop.operations, looptoken)
         deadframe = self.cpu.execute_token(looptoken,
                                            longlong.getfloatstorage(2.8))
         fail = self.cpu.get_latest_descr(deadframe)
@@ -154,17 +154,16 @@
     def test_compile_loop(self):
         looptoken = JitCellToken()
         targettoken = TargetToken()
-        i0 = InputArgInt()
-        op0 = ResOperation(rop.LABEL, [i0], descr=targettoken)
-        op1 = ResOperation(rop.INT_ADD, [i0, ConstInt(1)])
-        op2 = ResOperation(rop.INT_LE, [op1, ConstInt(9)])
-        op3 = ResOperation(rop.GUARD_TRUE, [op2], descr=BasicFailDescr(2))
-        op4 = ResOperation(rop.JUMP, [op1], descr=targettoken)
-        operations = [op0, op1, op2, op3, op4]
-        inputargs = [i0]
-        operations[3].setfailargs([op1])
-
-        self.cpu.compile_loop(inputargs, operations, looptoken)
+        loop = parse("""
+        [i0]
+        label(i0, descr=targettoken)
+        i1 = int_add(i0, 1)
+        i2 = int_le(i1, 9)
+        guard_true(i2, descr=fdescr) [i1]
+        jump(i1, descr=targettoken)
+        """, namespace={'targettoken': targettoken,
+                        'fdescr': BasicFailDescr(2)})
+        self.cpu.compile_loop(loop.inputargs, loop.operations, looptoken)
         deadframe = self.cpu.execute_token(looptoken, 2)
         fail = self.cpu.get_latest_descr(deadframe)
         assert fail.identifier == 2
@@ -172,24 +171,21 @@
         assert res == 10
 
     def test_compile_with_holes_in_fail_args(self):
-        i0 = BoxInt()
-        i1 = BoxInt()
-        i2 = BoxInt()
-        i3 = BoxInt()
+        targettoken = TargetToken()
+        loop = parse("""
+        [i3]
+        i0 = int_sub(i3, 42)
+        label(i0, descr=targettoken)
+        i1 = int_add(i0, 1)
+        i2 = int_le(i1, 9)
+        guard_true(i2, descr=fdescr) []
+        jump(i1, descr=targettoken)
+        """, namespace={'targettoken': targettoken,
+                        'fdescr': BasicFailDescr(2)})
         looptoken = JitCellToken()
-        targettoken = TargetToken()
-        operations = [
-            ResOperation(rop.INT_SUB, [i3, ConstInt(42)], i0),
-            ResOperation(rop.LABEL, [i0], None, descr=targettoken),
-            ResOperation(rop.INT_ADD, [i0, ConstInt(1)], i1),
-            ResOperation(rop.INT_LE, [i1, ConstInt(9)], i2),
-            ResOperation(rop.GUARD_TRUE, [i2], None, descr=BasicFailDescr(2)),
-            ResOperation(rop.JUMP, [i1], None, descr=targettoken),
-            ]
-        inputargs = [i3]
-        operations[4].setfailargs([None, None, i1, None])
-
-        self.cpu.compile_loop(inputargs, operations, looptoken)
+        loop.operations[4].setfailargs([None, None, loop.operations[2], None])
+
+        self.cpu.compile_loop(loop.inputargs, loop.operations, looptoken)
         deadframe = self.cpu.execute_token(looptoken, 44)
         fail = self.cpu.get_latest_descr(deadframe)
         assert fail.identifier == 2
@@ -199,62 +195,53 @@
     def test_backends_dont_keep_loops_alive(self):
         import weakref, gc
         self.cpu.dont_keepalive_stuff = True
-        i0 = BoxInt()
-        i1 = BoxInt()
-        i2 = BoxInt()
+        targettoken = TargetToken()
+        loop = parse("""
+        [i0]
+        label(i0, descr=targettoken)
+        i1 = int_add(i0, 1)
+        i2 = int_le(i1, 9)
+        guard_true(i2, descr=fdescr) [i1]
+        jump(i1, descr=targettoken)
+        """, namespace={'targettoken': targettoken, 'fdescr': BasicFailDescr()})
         looptoken = JitCellToken()
-        targettoken = TargetToken()
-        operations = [
-            ResOperation(rop.LABEL, [i0], None, descr=targettoken),
-            ResOperation(rop.INT_ADD, [i0, ConstInt(1)], i1),
-            ResOperation(rop.INT_LE, [i1, ConstInt(9)], i2),
-            ResOperation(rop.GUARD_TRUE, [i2], None, descr=BasicFailDescr()),
-            ResOperation(rop.JUMP, [i1], None, descr=targettoken),
-            ]
-        inputargs = [i0]
-        operations[3].setfailargs([i1])
-        wr_i1 = weakref.ref(i1)
-        wr_guard = weakref.ref(operations[2])
-        self.cpu.compile_loop(inputargs, operations, looptoken)
+        wr_i1 = weakref.ref(loop.operations[1])
+        wr_guard = weakref.ref(loop.operations[3])
+        self.cpu.compile_loop(loop.inputargs, loop.operations, looptoken)
         if hasattr(looptoken, '_x86_ops_offset'):
             del looptoken._x86_ops_offset # else it's kept alive
-        del i0, i1, i2
-        del inputargs
-        del operations
+        del loop
         gc.collect()
         assert not wr_i1() and not wr_guard()
 
     def test_compile_bridge(self):
         self.cpu.tracker.total_compiled_loops = 0
         self.cpu.tracker.total_compiled_bridges = 0
-        i0 = BoxInt()
-        i1 = BoxInt()
-        i2 = BoxInt()
+        targettoken = TargetToken()
         faildescr1 = BasicFailDescr(1)
         faildescr2 = BasicFailDescr(2)
+        loop = parse("""
+        [i0]
+        label(i0, descr=targettoken)
+        i1 = int_add(i0, 1)
+        i2 = int_le(i1, 9)
+        guard_true(i2, descr=faildescr1) [i1]
+        jump(i1, descr=targettoken)
+        """, namespace={'targettoken': targettoken,
+                        'faildescr1': faildescr1})
         looptoken = JitCellToken()
-        targettoken = TargetToken()
-        operations = [
-            ResOperation(rop.LABEL, [i0], None, descr=targettoken),
-            ResOperation(rop.INT_ADD, [i0, ConstInt(1)], i1),
-            ResOperation(rop.INT_LE, [i1, ConstInt(9)], i2),
-            ResOperation(rop.GUARD_TRUE, [i2], None, descr=faildescr1),
-            ResOperation(rop.JUMP, [i1], None, descr=targettoken),
-            ]
-        inputargs = [i0]
-        operations[3].setfailargs([i1])
-        self.cpu.compile_loop(inputargs, operations, looptoken)
-
-        i1b = BoxInt()
-        i3 = BoxInt()
-        bridge = [
-            ResOperation(rop.INT_LE, [i1b, ConstInt(19)], i3),
-            ResOperation(rop.GUARD_TRUE, [i3], None, descr=faildescr2),
-            ResOperation(rop.JUMP, [i1b], None, descr=targettoken),
-        ]
-        bridge[1].setfailargs([i1b])
-
-        self.cpu.compile_bridge(faildescr1, [i1b], bridge, looptoken)
+        self.cpu.compile_loop(loop.inputargs, loop.operations, looptoken)
+
+        bridge = parse("""
+        [i1]
+        i3 = int_le(i1, 19)
+        guard_true(i3, descr=faildescr2) [i1]
+        jump(i1, descr=targettoken)
+        """, namespace={"targettoken": targettoken,
+                        'faildescr2': faildescr2})
+
+        self.cpu.compile_bridge(faildescr1, bridge.inputargs,
+                                bridge.operations, looptoken)
 
         deadframe = self.cpu.execute_token(looptoken, 2)
         fail = self.cpu.get_latest_descr(deadframe)
@@ -267,36 +254,32 @@
         return looptoken
 
     def test_compile_bridge_with_holes(self):
-        i0 = BoxInt()
-        i1 = BoxInt()
-        i2 = BoxInt()
-        i3 = BoxInt()
         faildescr1 = BasicFailDescr(1)
         faildescr2 = BasicFailDescr(2)
         looptoken = JitCellToken()
         targettoken = TargetToken()
-        operations = [
-            ResOperation(rop.INT_SUB, [i3, ConstInt(42)], i0),
-            ResOperation(rop.LABEL, [i0], None, descr=targettoken),
-            ResOperation(rop.INT_ADD, [i0, ConstInt(1)], i1),
-            ResOperation(rop.INT_LE, [i1, ConstInt(9)], i2),
-            ResOperation(rop.GUARD_TRUE, [i2], None, descr=faildescr1),
-            ResOperation(rop.JUMP, [i1], None, descr=targettoken),
-            ]
-        inputargs = [i3]
-        operations[4].setfailargs([None, i1, None])
-        self.cpu.compile_loop(inputargs, operations, looptoken)
-
-        i1b = BoxInt()
-        i3 = BoxInt()
-        bridge = [
-            ResOperation(rop.INT_LE, [i1b, ConstInt(19)], i3),
-            ResOperation(rop.GUARD_TRUE, [i3], None, descr=faildescr2),
-            ResOperation(rop.JUMP, [i1b], None, descr=targettoken),
-        ]
-        bridge[1].setfailargs([i1b])
-
-        self.cpu.compile_bridge(faildescr1, [i1b], bridge, looptoken)
+        loop = parse("""
+        [i3]
+        i0 = int_sub(i3, 42)
+        label(i0, descr=targettoken)
+        i1 = int_add(i0, 1)
+        i2 = int_le(i1, 9)
+        guard_true(i2, descr=faildescr1) []
+        jump(i1, descr=targettoken)
+        """, namespace={'targettoken': targettoken,
+                        'faildescr1': faildescr1})
+        loop.operations[-2].setfailargs([None, loop.operations[2], None])
+        self.cpu.compile_loop(loop.inputargs, loop.operations, looptoken)
+
+        bridge = parse("""
+        [i1]
+        i3 = int_le(i1, 19)
+        guard_true(i3, descr=faildescr2) [i1]
+        jump(i1, descr=targettoken)
+        """, namespace={'targettoken': targettoken,
+                        'faildescr2': faildescr2})
+        self.cpu.compile_bridge(faildescr1, bridge.inputargs,
+                                bridge.operations, looptoken)
 
         deadframe = self.cpu.execute_token(looptoken, 2)
         fail = self.cpu.get_latest_descr(deadframe)
@@ -305,42 +288,40 @@
         assert res == 20
 
     def test_compile_big_bridge_out_of_small_loop(self):
-        i0 = BoxInt()
         faildescr1 = BasicFailDescr(1)
+        loop = parse("""
+        [i0]
+        guard_false(i0, descr=faildescr1) [i0]
+        finish(descr=finaldescr)
+        """, namespace={'faildescr1': faildescr1,
+                        'finaldescr': BasicFinalDescr(2)})
         looptoken = JitCellToken()
-        operations = [
-            ResOperation(rop.GUARD_FALSE, [i0], None, descr=faildescr1),
-            ResOperation(rop.FINISH, [], None, descr=BasicFinalDescr(2)),
-            ]
-        inputargs = [i0]
-        operations[0].setfailargs([i0])
-        self.cpu.compile_loop(inputargs, operations, looptoken)
-
-        i1list = [BoxInt() for i in range(150)]
+        self.cpu.compile_loop(loop.inputargs, loop.operations, looptoken)
+
         bridge = []
+        i0 = InputArgInt()
         iprev = i0
-        for i1 in i1list:
-            bridge.append(ResOperation(rop.INT_ADD, [iprev, ConstInt(1)], i1))
-            iprev = i1
-        bridge.append(ResOperation(rop.GUARD_FALSE, [i0], None,
+        for i in range(150):
+            iprev = ResOperation(rop.INT_ADD, [iprev, ConstInt(1)])
+            bridge.append(iprev)
+        bridge.append(ResOperation(rop.GUARD_FALSE, [i0],
                                    descr=BasicFailDescr(3)))
-        bridge.append(ResOperation(rop.FINISH, [], None,
+        bridge.append(ResOperation(rop.FINISH, [],
                                    descr=BasicFinalDescr(4)))
-        bridge[-2].setfailargs(i1list)
+        bridge[-2].setfailargs(bridge[:-2])
 
         self.cpu.compile_bridge(faildescr1, [i0], bridge, looptoken)
 
         deadframe = self.cpu.execute_token(looptoken, 1)
         fail = self.cpu.get_latest_descr(deadframe)
         assert fail.identifier == 3
-        for i in range(len(i1list)):
+        for i in range(150):
             res = self.cpu.get_int_value(deadframe, i)
             assert res == 2 + i
 
     def test_finish(self):
         from rpython.jit.backend.llsupport.llmodel import final_descr_rd_locs
 
-        i0 = BoxInt()
         class UntouchableFailDescr(AbstractFailDescr):
             final_descr = True
             rd_locs = final_descr_rd_locs
@@ -352,10 +333,11 @@
                 py.test.fail("finish descrs should not be touched")
         faildescr = UntouchableFailDescr() # to check that is not touched
         looptoken = JitCellToken()
-        operations = [
-            ResOperation(rop.FINISH, [i0], None, descr=faildescr)
-            ]
-        self.cpu.compile_loop([i0], operations, looptoken)
+        loop = parse("""
+        [i0]
+        finish(i0, descr=faildescr)
+        """, namespace={'faildescr': faildescr})
+        self.cpu.compile_loop(loop.inputargs, loop.operations, looptoken)
         deadframe = self.cpu.execute_token(looptoken, 99)
         fail = self.cpu.get_latest_descr(deadframe)
         assert fail is faildescr
@@ -364,7 +346,7 @@
 
         looptoken = JitCellToken()
         operations = [
-            ResOperation(rop.FINISH, [ConstInt(42)], None, descr=faildescr)
+            ResOperation(rop.FINISH, [ConstInt(42)], descr=faildescr)
             ]
         self.cpu.compile_loop([], operations, looptoken)
         deadframe = self.cpu.execute_token(looptoken)
@@ -375,7 +357,7 @@
 
         looptoken = JitCellToken()
         operations = [
-            ResOperation(rop.FINISH, [], None, descr=faildescr)
+            ResOperation(rop.FINISH, [], descr=faildescr)
             ]
         self.cpu.compile_loop([], operations, looptoken)
         deadframe = self.cpu.execute_token(looptoken)
@@ -384,9 +366,9 @@
 
         if self.cpu.supports_floats:
             looptoken = JitCellToken()
-            f0 = BoxFloat()
+            f0 = InputArgFloat()
             operations = [
-                ResOperation(rop.FINISH, [f0], None, descr=faildescr)
+                ResOperation(rop.FINISH, [f0], descr=faildescr)
                 ]
             self.cpu.compile_loop([f0], operations, looptoken)
             value = longlong.getfloatstorage(-61.25)
@@ -398,7 +380,7 @@
 
             looptoken = JitCellToken()
             operations = [
-                ResOperation(rop.FINISH, [constfloat(42.5)], None, descr=faildescr)
+                ResOperation(rop.FINISH, [constfloat(42.5)], descr=faildescr)
                 ]
             self.cpu.compile_loop([], operations, looptoken)
             deadframe = self.cpu.execute_token(looptoken)
@@ -409,24 +391,19 @@
 
     def test_execute_operations_in_env(self):
         cpu = self.cpu
-        x = BoxInt(123)
-        y = BoxInt(456)
-        z = BoxInt(579)
-        t = BoxInt(455)
-        u = BoxInt(0)    # False
+        targettoken = TargetToken()
+        loop = parse("""
+        [i1, i0]
+        label(i0, i1, descr=targettoken)
+        i2 = int_add(i1, i0)
+        i3 = int_sub(i0, 1)
+        i4 = int_eq(i3, 0)
+        guard_false(i4, descr=fdescr) [i3, i2]
+        jump(i3, i2, descr=targettoken)
+        """, namespace={'targettoken': targettoken,
+                        'fdescr': BasicFailDescr()})
         looptoken = JitCellToken()
-        targettoken = TargetToken()
-        operations = [
-            ResOperation(rop.LABEL, [y, x], None, descr=targettoken),
-            ResOperation(rop.INT_ADD, [x, y], z),
-            ResOperation(rop.INT_SUB, [y, ConstInt(1)], t),
-            ResOperation(rop.INT_EQ, [t, ConstInt(0)], u),
-            ResOperation(rop.GUARD_FALSE, [u], None,
-                         descr=BasicFailDescr()),
-            ResOperation(rop.JUMP, [t, z], None, descr=targettoken),
-            ]
-        operations[-2].setfailargs([t, z])
-        cpu.compile_loop([x, y], operations, looptoken)
+        cpu.compile_loop(loop.inputargs, loop.operations, looptoken)
         deadframe = self.cpu.execute_token(looptoken, 0, 10)
         assert self.cpu.get_int_value(deadframe, 0) == 0
         assert self.cpu.get_int_value(deadframe, 1) == 55

From noreply at buildbot.pypy.org  Wed Nov 19 08:09:32 2014
From: noreply at buildbot.pypy.org (fijal)
Date: Wed, 19 Nov 2014 08:09:32 +0100 (CET)
Subject: [pypy-commit] pypy optresult: merged default
Message-ID: <20141119070932.053C81C104D@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: optresult
Changeset: r74589:daec44fd7fcf
Date: 2014-11-19 09:09 +0200
http://bitbucket.org/pypy/pypy/changeset/daec44fd7fcf/

Log:	merged default

diff too long, truncating to 2000 out of 4280 lines

diff --git a/lib-python/conftest.py b/lib-python/conftest.py
--- a/lib-python/conftest.py
+++ b/lib-python/conftest.py
@@ -59,7 +59,7 @@
     def __init__(self, basename, core=False, compiler=None, usemodules='',
                  skip=None):
         self.basename = basename
-        self._usemodules = usemodules.split() + ['signal', 'rctime', 'itertools', '_socket']
+        self._usemodules = usemodules.split() + ['signal', 'time', 'itertools', '_socket']
         self._compiler = compiler
         self.core = core
         self.skip = skip
diff --git a/pypy/config/pypyoption.py b/pypy/config/pypyoption.py
--- a/pypy/config/pypyoption.py
+++ b/pypy/config/pypyoption.py
@@ -29,7 +29,7 @@
 # --allworkingmodules
 working_modules = default_modules.copy()
 working_modules.update([
-    "_socket", "unicodedata", "mmap", "fcntl", "_locale", "pwd", "rctime" ,
+    "_socket", "unicodedata", "mmap", "fcntl", "_locale", "pwd", "time" ,
     "select", "zipimport", "_lsprof", "crypt", "signal", "_rawffi", "termios",
     "zlib", "bz2", "struct", "_hashlib", "_md5", "_sha", "_minimal_curses",
     "cStringIO", "thread", "itertools", "pyexpat", "_ssl", "cpyext", "array",
@@ -40,7 +40,7 @@
 
 translation_modules = default_modules.copy()
 translation_modules.update([
-    "fcntl", "rctime", "select", "signal", "_rawffi", "zlib", "struct", "_md5",
+    "fcntl", "time", "select", "signal", "_rawffi", "zlib", "struct", "_md5",
     "cStringIO", "array", "binascii",
     # the following are needed for pyrepl (and hence for the
     # interactive prompt/pdb)
@@ -64,19 +64,15 @@
     default_modules.add("_locale")
 
 if sys.platform == "sunos5":
-    working_modules.remove('mmap')   # depend on ctypes, can't get at c-level 'errono'
-    working_modules.remove('rctime') # depend on ctypes, missing tm_zone/tm_gmtoff
-    working_modules.remove('signal') # depend on ctypes, can't get at c-level 'errono'
     working_modules.remove('fcntl')  # LOCK_NB not defined
     working_modules.remove("_minimal_curses")
     working_modules.remove("termios")
-    working_modules.remove("_multiprocessing")   # depends on rctime
     if "cppyy" in working_modules:
         working_modules.remove("cppyy")  # depends on ctypes
 
 
 module_dependencies = {
-    '_multiprocessing': [('objspace.usemodules.rctime', True),
+    '_multiprocessing': [('objspace.usemodules.time', True),
                          ('objspace.usemodules.thread', True)],
     'cpyext': [('objspace.usemodules.array', True)],
     'cppyy': [('objspace.usemodules.cpyext', True)],
diff --git a/pypy/doc/config/objspace.usemodules.rctime.txt b/pypy/doc/config/objspace.usemodules.rctime.txt
deleted file mode 100644
--- a/pypy/doc/config/objspace.usemodules.rctime.txt
+++ /dev/null
@@ -1,7 +0,0 @@
-Use the 'rctime' module. 
-
-'rctime' is our `rffi`_ based implementation of the builtin 'time' module.
-It supersedes the less complete :config:`objspace.usemodules.time`,
-at least for C-like targets (the C and LLVM backends).
-
-.. _`rffi`: ../rffi.html
diff --git a/pypy/doc/config/objspace.usemodules.time.txt b/pypy/doc/config/objspace.usemodules.time.txt
--- a/pypy/doc/config/objspace.usemodules.time.txt
+++ b/pypy/doc/config/objspace.usemodules.time.txt
@@ -1,5 +1,1 @@
 Use the 'time' module. 
-
-Obsolete; use :config:`objspace.usemodules.rctime` for our up-to-date version
-of the application-level 'time' module, at least for C-like targets (the C
-and LLVM backends).
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -43,3 +43,11 @@
 .. branch nditer-external_loop
 
 Implement `external_loop` arguement to numpy's nditer
+
+.. branch kill-rctime
+
+Rename pypy/module/rctime to pypy/module/time, since it contains the implementation of the 'time' module.
+
+.. branch: ssa-flow
+
+Use SSA form for flow graphs inside build_flow() and part of simplify_graph()
diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -522,11 +522,6 @@
                 if name not in modules:
                     modules.append(name)
 
-        # a bit of custom logic: rctime take precedence over time
-        # XXX this could probably be done as a "requires" in the config
-        if 'rctime' in modules and 'time' in modules:
-            modules.remove('time')
-
         self._builtinmodule_list = modules
         return self._builtinmodule_list
 
diff --git a/pypy/module/_file/test/test_file_extra.py b/pypy/module/_file/test/test_file_extra.py
--- a/pypy/module/_file/test/test_file_extra.py
+++ b/pypy/module/_file/test/test_file_extra.py
@@ -221,7 +221,7 @@
     expected_filename = str(udir.join('sample'))
     expected_mode = 'rb'
     extra_args = ()
-    spaceconfig = {'usemodules': ['binascii', 'rctime', 'struct']}
+    spaceconfig = {'usemodules': ['binascii', 'time', 'struct']}
 
     def setup_method(self, method):
         space = self.space
@@ -281,7 +281,7 @@
     expected_filename = ''
     expected_mode = 'rb'
     extra_args = ()
-    spaceconfig = {'usemodules': ['binascii', 'rctime', 'struct']}
+    spaceconfig = {'usemodules': ['binascii', 'time', 'struct']}
 
     def setup_method(self, method):
         space = self.space
@@ -359,7 +359,7 @@
 #  A few extra tests
 
 class AppTestAFewExtra:
-    spaceconfig = {'usemodules': ['_socket', 'array', 'binascii', 'rctime',
+    spaceconfig = {'usemodules': ['_socket', 'array', 'binascii', 'time',
                                   'struct']}
 
     def setup_method(self, method):
diff --git a/pypy/module/_lsprof/test/test_cprofile.py b/pypy/module/_lsprof/test/test_cprofile.py
--- a/pypy/module/_lsprof/test/test_cprofile.py
+++ b/pypy/module/_lsprof/test/test_cprofile.py
@@ -1,6 +1,6 @@
 class AppTestCProfile(object):
     spaceconfig = {
-        "usemodules": ['_lsprof', 'rctime'],
+        "usemodules": ['_lsprof', 'time'],
     }
 
     def setup_class(cls):
diff --git a/pypy/module/_md5/test/test_md5.py b/pypy/module/_md5/test/test_md5.py
--- a/pypy/module/_md5/test/test_md5.py
+++ b/pypy/module/_md5/test/test_md5.py
@@ -5,7 +5,7 @@
 
 class AppTestMD5(object):
     spaceconfig = {
-        'usemodules': ['_md5', 'binascii', 'rctime', 'struct'],
+        'usemodules': ['_md5', 'binascii', 'time', 'struct'],
     }
 
     def setup_class(cls):
diff --git a/pypy/module/_multiprocessing/interp_semaphore.py b/pypy/module/_multiprocessing/interp_semaphore.py
--- a/pypy/module/_multiprocessing/interp_semaphore.py
+++ b/pypy/module/_multiprocessing/interp_semaphore.py
@@ -254,7 +254,7 @@
         start = _GetTickCount()
 
         while True:
-            from pypy.module.rctime.interp_time import State
+            from pypy.module.time.interp_time import State
             interrupt_event = space.fromcache(State).get_interrupt_event()
             handles = [self.handle, interrupt_event]
 
diff --git a/pypy/module/_random/test/test_random.py b/pypy/module/_random/test/test_random.py
--- a/pypy/module/_random/test/test_random.py
+++ b/pypy/module/_random/test/test_random.py
@@ -1,6 +1,6 @@
 class AppTestRandom:
     spaceconfig = {
-        "usemodules": ['_random', 'rctime'],
+        "usemodules": ['_random', 'time'],
     }
 
     def test_dict(self):
diff --git a/pypy/module/_sha/test/test_sha.py b/pypy/module/_sha/test/test_sha.py
--- a/pypy/module/_sha/test/test_sha.py
+++ b/pypy/module/_sha/test/test_sha.py
@@ -5,7 +5,7 @@
 
 class AppTestSHA(object):
     spaceconfig = {
-        'usemodules': ['_sha', 'binascii', 'rctime', 'struct'],
+        'usemodules': ['_sha', 'binascii', 'time', 'struct'],
     }
 
     def setup_class(cls):
diff --git a/pypy/module/bz2/test/test_bz2_file.py b/pypy/module/bz2/test/test_bz2_file.py
--- a/pypy/module/bz2/test/test_bz2_file.py
+++ b/pypy/module/bz2/test/test_bz2_file.py
@@ -53,7 +53,7 @@
 
 class AppTestBZ2File(CheckAllocation):
     spaceconfig = {
-        'usemodules': ['bz2', 'binascii', 'rctime', 'struct']
+        'usemodules': ['bz2', 'binascii', 'time', 'struct']
     }
 
     def setup_class(cls):
diff --git a/pypy/module/cppyy/include/capi.h b/pypy/module/cppyy/include/capi.h
--- a/pypy/module/cppyy/include/capi.h
+++ b/pypy/module/cppyy/include/capi.h
@@ -2,6 +2,7 @@
 #define CPPYY_CAPI
 
 #include 
+#include "src/precommondefs.h"
 
 #ifdef __cplusplus
 extern "C" {
@@ -15,102 +16,167 @@
     typedef void* (*cppyy_methptrgetter_t)(cppyy_object_t);
 
     /* name to opaque C++ scope representation -------------------------------- */
+    RPY_EXTERN
     int cppyy_num_scopes(cppyy_scope_t parent);
+    RPY_EXTERN
     char* cppyy_scope_name(cppyy_scope_t parent, int iscope);
 
+    RPY_EXTERN
     char* cppyy_resolve_name(const char* cppitem_name);
+    RPY_EXTERN
     cppyy_scope_t cppyy_get_scope(const char* scope_name);
+    RPY_EXTERN
     cppyy_type_t cppyy_get_template(const char* template_name);
+    RPY_EXTERN
     cppyy_type_t cppyy_actual_class(cppyy_type_t klass, cppyy_object_t obj);
 
     /* memory management ------------------------------------------------------ */
+    RPY_EXTERN
     cppyy_object_t cppyy_allocate(cppyy_type_t type);
+    RPY_EXTERN
     void cppyy_deallocate(cppyy_type_t type, cppyy_object_t self);
+    RPY_EXTERN
     void cppyy_destruct(cppyy_type_t type, cppyy_object_t self);
 
     /* method/function dispatching -------------------------------------------- */
+    RPY_EXTERN
     void   cppyy_call_v(cppyy_method_t method, cppyy_object_t self, int nargs, void* args);
+    RPY_EXTERN
     unsigned char cppyy_call_b(cppyy_method_t method, cppyy_object_t self, int nargs, void* args);
+    RPY_EXTERN
     char   cppyy_call_c(cppyy_method_t method, cppyy_object_t self, int nargs, void* args);
+    RPY_EXTERN
     short  cppyy_call_h(cppyy_method_t method, cppyy_object_t self, int nargs, void* args);
+    RPY_EXTERN
     int    cppyy_call_i(cppyy_method_t method, cppyy_object_t self, int nargs, void* args);
+    RPY_EXTERN
     long   cppyy_call_l(cppyy_method_t method, cppyy_object_t self, int nargs, void* args);
+    RPY_EXTERN
     long long cppyy_call_ll(cppyy_method_t method, cppyy_object_t self, int nargs, void* args);
+    RPY_EXTERN
     float  cppyy_call_f(cppyy_method_t method, cppyy_object_t self, int nargs, void* args);
+    RPY_EXTERN
     double cppyy_call_d(cppyy_method_t method, cppyy_object_t self, int nargs, void* args);
 
+    RPY_EXTERN
     void*  cppyy_call_r(cppyy_method_t method, cppyy_object_t self, int nargs, void* args);
+    RPY_EXTERN
     char*  cppyy_call_s(cppyy_method_t method, cppyy_object_t self, int nargs, void* args);
 
+    RPY_EXTERN
     cppyy_object_t cppyy_constructor(cppyy_method_t method, cppyy_type_t klass, int nargs, void* args);
+    RPY_EXTERN
     cppyy_object_t cppyy_call_o(cppyy_method_t method, cppyy_object_t self, int nargs, void* args, cppyy_type_t result_type);
 
+    RPY_EXTERN
     cppyy_methptrgetter_t cppyy_get_methptr_getter(cppyy_scope_t scope, cppyy_index_t idx);
 
     /* handling of function argument buffer ----------------------------------- */
+    RPY_EXTERN
     void*  cppyy_allocate_function_args(int nargs);
+    RPY_EXTERN
     void   cppyy_deallocate_function_args(void* args);
+    RPY_EXTERN
     size_t cppyy_function_arg_sizeof();
+    RPY_EXTERN
     size_t cppyy_function_arg_typeoffset();
 
     /* scope reflection information ------------------------------------------- */
+    RPY_EXTERN
     int cppyy_is_namespace(cppyy_scope_t scope);
+    RPY_EXTERN
     int cppyy_is_enum(const char* type_name);
 
     /* class reflection information ------------------------------------------- */
+    RPY_EXTERN
     char* cppyy_final_name(cppyy_type_t type);
+    RPY_EXTERN
     char* cppyy_scoped_final_name(cppyy_type_t type);
+    RPY_EXTERN
     int cppyy_has_complex_hierarchy(cppyy_type_t type);
+    RPY_EXTERN
     int cppyy_num_bases(cppyy_type_t type);
+    RPY_EXTERN
     char* cppyy_base_name(cppyy_type_t type, int base_index);
+    RPY_EXTERN
     int cppyy_is_subtype(cppyy_type_t derived, cppyy_type_t base);
 
     /* calculate offsets between declared and actual type, up-cast: direction > 0; down-cast: direction < 0 */
+    RPY_EXTERN
     ptrdiff_t cppyy_base_offset(cppyy_type_t derived, cppyy_type_t base, cppyy_object_t address, int direction);
 
     /* method/function reflection information --------------------------------- */
+    RPY_EXTERN
     int cppyy_num_methods(cppyy_scope_t scope);
+    RPY_EXTERN
     cppyy_index_t cppyy_method_index_at(cppyy_scope_t scope, int imeth);
+    RPY_EXTERN
     cppyy_index_t* cppyy_method_indices_from_name(cppyy_scope_t scope, const char* name);
 
+    RPY_EXTERN
     char* cppyy_method_name(cppyy_scope_t scope, cppyy_index_t idx);
+    RPY_EXTERN
     char* cppyy_method_result_type(cppyy_scope_t scope, cppyy_index_t idx);
+    RPY_EXTERN
     int cppyy_method_num_args(cppyy_scope_t scope, cppyy_index_t idx);
+    RPY_EXTERN
     int cppyy_method_req_args(cppyy_scope_t scope, cppyy_index_t idx);
+    RPY_EXTERN
     char* cppyy_method_arg_type(cppyy_scope_t scope, cppyy_index_t idx, int arg_index);
+    RPY_EXTERN
     char* cppyy_method_arg_default(cppyy_scope_t scope, cppyy_index_t idx, int arg_index);
+    RPY_EXTERN
     char* cppyy_method_signature(cppyy_scope_t scope, cppyy_index_t idx);
 
+    RPY_EXTERN
     int cppyy_method_is_template(cppyy_scope_t scope, cppyy_index_t idx);
+    RPY_EXTERN
     int cppyy_method_num_template_args(cppyy_scope_t scope, cppyy_index_t idx);
+    RPY_EXTERN
     char* cppyy_method_template_arg_name(cppyy_scope_t scope, cppyy_index_t idx, cppyy_index_t iarg);
 
+    RPY_EXTERN
     cppyy_method_t cppyy_get_method(cppyy_scope_t scope, cppyy_index_t idx);
+    RPY_EXTERN
     cppyy_index_t cppyy_get_global_operator(
         cppyy_scope_t scope, cppyy_scope_t lc, cppyy_scope_t rc, const char* op);
 
     /* method properties ------------------------------------------------------ */
+    RPY_EXTERN
     int cppyy_is_constructor(cppyy_type_t type, cppyy_index_t idx);
+    RPY_EXTERN
     int cppyy_is_staticmethod(cppyy_type_t type, cppyy_index_t idx);
 
     /* data member reflection information ------------------------------------- */
+    RPY_EXTERN
     int cppyy_num_datamembers(cppyy_scope_t scope);
+    RPY_EXTERN
     char* cppyy_datamember_name(cppyy_scope_t scope, int datamember_index);
+    RPY_EXTERN
     char* cppyy_datamember_type(cppyy_scope_t scope, int datamember_index);
+    RPY_EXTERN
     ptrdiff_t cppyy_datamember_offset(cppyy_scope_t scope, int datamember_index);
 
+    RPY_EXTERN
     int cppyy_datamember_index(cppyy_scope_t scope, const char* name);
 
     /* data member properties ------------------------------------------------- */
+    RPY_EXTERN
     int cppyy_is_publicdata(cppyy_type_t type, int datamember_index);
+    RPY_EXTERN
     int cppyy_is_staticdata(cppyy_type_t type, int datamember_index);
 
     /* misc helpers ----------------------------------------------------------- */
+    RPY_EXTERN
     long long cppyy_strtoll(const char* str);
+    RPY_EXTERN
     unsigned long long cppyy_strtoull(const char* str);
+    RPY_EXTERN
     void cppyy_free(void* ptr);
 
+    RPY_EXTERN
     cppyy_object_t cppyy_charp2stdstring(const char* str);
+    RPY_EXTERN
     cppyy_object_t cppyy_stdstring2stdstring(cppyy_object_t ptr);
 
 #ifdef __cplusplus
diff --git a/pypy/module/cppyy/src/dummy_backend.cxx b/pypy/module/cppyy/src/dummy_backend.cxx
--- a/pypy/module/cppyy/src/dummy_backend.cxx
+++ b/pypy/module/cppyy/src/dummy_backend.cxx
@@ -1,4 +1,3 @@
-#include "src/precommondefs.h"
 #include "cppyy.h"
 #include "capi.h"
 
@@ -349,29 +348,24 @@
 
 
 /* name to opaque C++ scope representation -------------------------------- */
-RPY_EXTERN
 int cppyy_num_scopes(cppyy_scope_t handle) {
     return 0;
 }
 
-RPY_EXTERN
 char* cppyy_resolve_name(const char* cppitem_name) {
     return cppstring_to_cstring(cppitem_name);
 }
 
-RPY_EXTERN
 cppyy_scope_t cppyy_get_scope(const char* scope_name) {
     return s_handles[scope_name];  // lookup failure will return 0 (== error)
 }
 
-RPY_EXTERN
 cppyy_type_t cppyy_actual_class(cppyy_type_t klass, cppyy_object_t /* obj */) {
     return klass;
 }
 
 
 /* memory management ------------------------------------------------------ */
-RPY_EXTERN
 void cppyy_destruct(cppyy_type_t handle, cppyy_object_t self) {
     if (handle == s_handles["example01"])
        delete (dummy::example01*)self;
@@ -379,7 +373,6 @@
 
 
 /* method/function dispatching -------------------------------------------- */
-RPY_EXTERN
 void cppyy_call_v(cppyy_method_t method, cppyy_object_t self, int nargs, void* args) {
     long idx = (long)method;
     if (idx == s_methods["static_example01::staticSetPayload_payload*_double"]) {
@@ -469,7 +462,6 @@
     }
 }
 
-RPY_EXTERN
 unsigned char cppyy_call_b(cppyy_method_t method, cppyy_object_t self, int nargs, void* args) {
     unsigned char result = 0;
     const long idx = (long)method;
@@ -482,7 +474,6 @@
     return result;
 }
 
-RPY_EXTERN
 char cppyy_call_c(cppyy_method_t method, cppyy_object_t self, int nargs, void* args) {
     char result = 0;
     const long idx = (long)method;
@@ -498,7 +489,6 @@
     return result;
 }
 
-RPY_EXTERN
 short cppyy_call_h(cppyy_method_t method, cppyy_object_t self, int nargs, void* args) {
     short result = 0;
     const long idx = (long)method; 
@@ -514,7 +504,6 @@
     return result;
 }
 
-RPY_EXTERN
 int cppyy_call_i(cppyy_method_t method, cppyy_object_t self, int nargs, void* args) {
     int result = 0;
     const long idx = (long)method;
@@ -547,7 +536,6 @@
     return result;
 }
 
-RPY_EXTERN
 long cppyy_call_l(cppyy_method_t method, cppyy_object_t self, int nargs, void* args) {
     long result = 0;
     const long idx = (long)method;
@@ -689,7 +677,6 @@
     return result;
 }
 
-RPY_EXTERN
 long long cppyy_call_ll(cppyy_method_t method, cppyy_object_t self, int nargs, void* args) {
     long long result = 0;
     const long idx = (long)method;
@@ -705,7 +692,6 @@
     return result;
 }   
 
-RPY_EXTERN
 float cppyy_call_f(cppyy_method_t method, cppyy_object_t self, int nargs, void* args) {
     float result = 0;
     const long idx = (long)method;
@@ -718,7 +704,6 @@
     return result;
 }   
 
-RPY_EXTERN
 double cppyy_call_d(cppyy_method_t method, cppyy_object_t self, int nargs, void* args) {
     double result = 0.;
     const long idx = (long)method;
@@ -740,7 +725,6 @@
     return result;
 }
 
-RPY_EXTERN
 char* cppyy_call_s(cppyy_method_t method, cppyy_object_t self, int nargs, void* args) {
     char* result = 0;
     const long idx = (long)method;
@@ -753,7 +737,6 @@
     return result;
 }
 
-RPY_EXTERN
 cppyy_object_t cppyy_constructor(cppyy_method_t method, cppyy_type_t handle, int nargs, void* args) {
     void* result = 0;
     const long idx = (long)method;
@@ -776,14 +759,12 @@
     return (cppyy_object_t)result;
 }
 
-RPY_EXTERN
 cppyy_methptrgetter_t cppyy_get_methptr_getter(cppyy_type_t /* handle */, cppyy_index_t /* method_index */) {
     return (cppyy_methptrgetter_t)0;
 }
 
 
 /* handling of function argument buffer ----------------------------------- */
-RPY_EXTERN
 void* cppyy_allocate_function_args(int nargs) {
     CPPYY_G__value* args = (CPPYY_G__value*)malloc(nargs*sizeof(CPPYY_G__value));
     for (int i = 0; i < nargs; ++i)
@@ -793,36 +774,30 @@
 
 
 /* handling of function argument buffer ----------------------------------- */
-RPY_EXTERN
 void cppyy_deallocate_function_args(void* args) {
     free(args);
 }
 
-RPY_EXTERN
 size_t cppyy_function_arg_sizeof() {
     return sizeof(CPPYY_G__value);
 }
 
-RPY_EXTERN
 size_t cppyy_function_arg_typeoffset() {
     return offsetof(CPPYY_G__value, type);
 }
 
 
 /* scope reflection information ------------------------------------------- */
-RPY_EXTERN
 int cppyy_is_namespace(cppyy_scope_t /* handle */) {
     return 0;
 }   
 
-RPY_EXTERN
 int cppyy_is_enum(const char* /* type_name */) {
     return 0;
 }
     
     
 /* class reflection information ------------------------------------------- */
-RPY_EXTERN
 char* cppyy_final_name(cppyy_type_t handle) {
     for (Handles_t::iterator isp = s_handles.begin(); isp != s_handles.end(); ++isp) {
         if (isp->second == handle)
@@ -831,75 +806,61 @@
     return cppstring_to_cstring("");
 }
 
-RPY_EXTERN
 char* cppyy_scoped_final_name(cppyy_type_t handle) {
     return cppyy_final_name(handle);
 }   
 
-RPY_EXTERN
 int cppyy_has_complex_hierarchy(cppyy_type_t /* handle */) {
     return 0;
 }
 
-RPY_EXTERN
 int cppyy_num_bases(cppyy_type_t /*handle*/) {
    return 0;
 }
 
 
 /* method/function reflection information --------------------------------- */
-RPY_EXTERN
 int cppyy_num_methods(cppyy_scope_t handle) {
     return s_scopes[handle].m_methods.size();
 }
 
-RPY_EXTERN
 cppyy_index_t cppyy_method_index_at(cppyy_scope_t /* scope */, int imeth) {
     return (cppyy_index_t)imeth;
 }
 
-RPY_EXTERN
 char* cppyy_method_name(cppyy_scope_t handle, cppyy_index_t method_index) {
     return cppstring_to_cstring(s_scopes[handle].m_methods[(int)method_index].m_name);
 }
 
-RPY_EXTERN
 char* cppyy_method_result_type(cppyy_scope_t handle, cppyy_index_t method_index) {
     return cppstring_to_cstring(s_scopes[handle].m_methods[method_index].m_returntype);
 }
     
-RPY_EXTERN
 int cppyy_method_num_args(cppyy_scope_t handle, cppyy_index_t method_index) {
     return s_scopes[handle].m_methods[method_index].m_argtypes.size();
 }
 
-RPY_EXTERN
 int cppyy_method_req_args(cppyy_scope_t handle, cppyy_index_t method_index) {
     return cppyy_method_num_args(handle, method_index);
 }
 
-RPY_EXTERN
 char* cppyy_method_arg_type(cppyy_scope_t handle, cppyy_index_t method_index, int arg_index) {
     return cppstring_to_cstring(s_scopes[handle].m_methods[method_index].m_argtypes[arg_index]);
 }
 
-RPY_EXTERN
 char* cppyy_method_arg_default(
         cppyy_scope_t /* handle */, cppyy_index_t /* method_index */, int /* arg_index */) {
     return cppstring_to_cstring("");
 }
 
-RPY_EXTERN
 char* cppyy_method_signature(cppyy_scope_t /* handle */, cppyy_index_t /* method_index */) {
     return cppstring_to_cstring("");
 }
 
-RPY_EXTERN
 int cppyy_method_is_template(cppyy_scope_t /* handle */, cppyy_index_t /* method_index */) {
     return 0;
 }
     
-RPY_EXTERN
 cppyy_method_t cppyy_get_method(cppyy_scope_t handle, cppyy_index_t method_index) {
     if (s_scopes.find(handle) != s_scopes.end()) {
         long id = s_scopes[handle].m_method_offset + (long)method_index;
@@ -911,7 +872,6 @@
 
 
 /* method properties -----------------------------------------------------  */
-RPY_EXTERN
 int cppyy_is_constructor(cppyy_type_t handle, cppyy_index_t method_index) {
     if (s_scopes.find(handle) != s_scopes.end())
         return s_scopes[handle].m_methods[method_index].m_type == kConstructor;
@@ -919,7 +879,6 @@
     return 0;
 }
 
-RPY_EXTERN
 int cppyy_is_staticmethod(cppyy_type_t handle, cppyy_index_t method_index) {
     if (s_scopes.find(handle) != s_scopes.end())
         return s_scopes[handle].m_methods[method_index].m_type == kStatic;
@@ -929,34 +888,28 @@
 
 
 /* data member reflection information ------------------------------------- */
-RPY_EXTERN
 int cppyy_num_datamembers(cppyy_scope_t handle) {
     return s_scopes[handle].m_datambrs.size();
 }
 
-RPY_EXTERN
 char* cppyy_datamember_name(cppyy_scope_t handle, int idatambr) {
     return cppstring_to_cstring(s_scopes[handle].m_datambrs[idatambr].m_name);
 }
 
-RPY_EXTERN
 char* cppyy_datamember_type(cppyy_scope_t handle, int idatambr) {
     return cppstring_to_cstring(s_scopes[handle].m_datambrs[idatambr].m_type);
 }
 
-RPY_EXTERN
 ptrdiff_t cppyy_datamember_offset(cppyy_scope_t handle, int idatambr) {
     return s_scopes[handle].m_datambrs[idatambr].m_offset;
 }
 
 
 /* data member properties ------------------------------------------------  */
-RPY_EXTERN
 int cppyy_is_publicdata(cppyy_scope_t handle, int idatambr) {
     return 1;
 }
 
-RPY_EXTERN
 int cppyy_is_staticdata(cppyy_scope_t handle, int idatambr) {
     return s_scopes[handle].m_datambrs[idatambr].m_isstatic;
 }
@@ -964,44 +917,37 @@
 
 /* misc helpers ----------------------------------------------------------- */
 #if defined(_MSC_VER)
-RPY_EXTERN
 long long cppyy_strtoll(const char* str) {
     return _strtoi64(str, NULL, 0);
 }
 
 extern "C" {
-RPY_EXTERN
 unsigned long long cppyy_strtoull(const char* str) {
     return _strtoui64(str, NULL, 0);
 }
 }
 #else
-RPY_EXTERN
 long long cppyy_strtoll(const char* str) {
     return strtoll(str, NULL, 0);
 }
 
 extern "C" {
-RPY_EXTERN
 unsigned long long cppyy_strtoull(const char* str) {
     return strtoull(str, NULL, 0);
 }
 }
 #endif
 
-RPY_EXTERN
 void cppyy_free(void* ptr) {
     free(ptr);
 }
 
-RPY_EXTERN
 cppyy_object_t cppyy_charp2stdstring(const char* str) {
     void* arena = new char[sizeof(std::string)];
     new (arena) std::string(str);
     return (cppyy_object_t)arena;
 }
 
-RPY_EXTERN
 cppyy_object_t cppyy_stdstring2stdstring(cppyy_object_t ptr) {
     void* arena = new char[sizeof(std::string)];
     new (arena) std::string(*(std::string*)ptr);
diff --git a/pypy/module/cpyext/test/conftest.py b/pypy/module/cpyext/test/conftest.py
--- a/pypy/module/cpyext/test/conftest.py
+++ b/pypy/module/cpyext/test/conftest.py
@@ -7,7 +7,7 @@
     # it's necessary to run "import time" at least once before any
     # other cpyext test, otherwise the same statement will fail in
     # test_datetime.py.
-    space = gettestobjspace(usemodules=['rctime'])
+    space = gettestobjspace(usemodules=['time'])
     space.getbuiltinmodule("time")
 
 def pytest_ignore_collect(path, config):
diff --git a/pypy/module/cpyext/test/test_cpyext.py b/pypy/module/cpyext/test/test_cpyext.py
--- a/pypy/module/cpyext/test/test_cpyext.py
+++ b/pypy/module/cpyext/test/test_cpyext.py
@@ -102,7 +102,7 @@
 class LeakCheckingTest(object):
     """Base class for all cpyext tests."""
     spaceconfig = dict(usemodules=['cpyext', 'thread', '_rawffi', 'array',
-                                   'itertools', 'rctime', 'binascii', 'micronumpy'])
+                                   'itertools', 'time', 'binascii', 'micronumpy'])
     spaceconfig['std.withmethodcache'] = True
 
     enable_leak_checking = True
diff --git a/pypy/module/fcntl/test/test_fcntl.py b/pypy/module/fcntl/test/test_fcntl.py
--- a/pypy/module/fcntl/test/test_fcntl.py
+++ b/pypy/module/fcntl/test/test_fcntl.py
@@ -12,7 +12,7 @@
 
 class AppTestFcntl:
     spaceconfig = dict(usemodules=('fcntl', 'array', 'struct', 'termios',
-                                   'select', 'rctime'))
+                                   'select', 'time'))
 
     def setup_class(cls):
         tmpprefix = str(udir.ensure('test_fcntl', dir=1).join('tmp_'))
diff --git a/pypy/module/imp/test/test_app.py b/pypy/module/imp/test/test_app.py
--- a/pypy/module/imp/test/test_app.py
+++ b/pypy/module/imp/test/test_app.py
@@ -4,7 +4,7 @@
 
 class AppTestImpModule:
     spaceconfig = {
-        'usemodules': ['binascii', 'imp', 'itertools', 'rctime', 'struct'],
+        'usemodules': ['binascii', 'imp', 'itertools', 'time', 'struct'],
     }
 
     def setup_class(cls):
diff --git a/pypy/module/imp/test/test_import.py b/pypy/module/imp/test/test_import.py
--- a/pypy/module/imp/test/test_import.py
+++ b/pypy/module/imp/test/test_import.py
@@ -149,7 +149,7 @@
 
 class AppTestImport:
     spaceconfig = {
-        "usemodules": ['_md5', 'rctime'],
+        "usemodules": ['_md5', 'time'],
     }
 
     def setup_class(cls):
@@ -1044,7 +1044,7 @@
 
 class AppTestImportHooks(object):
     spaceconfig = {
-        "usemodules": ['struct', 'itertools', 'rctime'],
+        "usemodules": ['struct', 'itertools', 'time'],
     }
 
     def setup_class(cls):
@@ -1304,7 +1304,7 @@
 
 
 class AppTestMultithreadedImp(object):
-    spaceconfig = dict(usemodules=['thread', 'rctime'])
+    spaceconfig = dict(usemodules=['thread', 'time'])
 
     def setup_class(cls):
         #if not conftest.option.runappdirect:
diff --git a/pypy/module/math/test/test_math.py b/pypy/module/math/test/test_math.py
--- a/pypy/module/math/test/test_math.py
+++ b/pypy/module/math/test/test_math.py
@@ -7,7 +7,7 @@
 
 class AppTestMath:
     spaceconfig = {
-        "usemodules": ['math', 'struct', 'itertools', 'rctime', 'binascii'],
+        "usemodules": ['math', 'struct', 'itertools', 'time', 'binascii'],
     }
 
     def setup_class(cls):
diff --git a/pypy/module/micronumpy/nditer.py b/pypy/module/micronumpy/nditer.py
--- a/pypy/module/micronumpy/nditer.py
+++ b/pypy/module/micronumpy/nditer.py
@@ -89,10 +89,9 @@
     for w_item in lst:
         if not space.isinstance_w(w_item, space.w_str) and not \
                 space.isinstance_w(w_item, space.w_unicode):
-            typename = space.type(w_item).getname(space)
             raise oefmt(space.w_TypeError,
-                        'expected string or Unicode object, %s found',
-                        typename)
+                        "expected string or Unicode object, %T found",
+                        w_item)
         item = space.str_w(w_item)
         if item == 'external_loop':
             nditer.external_loop = True
diff --git a/pypy/module/mmap/interp_mmap.py b/pypy/module/mmap/interp_mmap.py
--- a/pypy/module/mmap/interp_mmap.py
+++ b/pypy/module/mmap/interp_mmap.py
@@ -5,6 +5,7 @@
 from rpython.rlib import rmmap, rarithmetic
 from rpython.rlib.buffer import Buffer
 from rpython.rlib.rmmap import RValueError, RTypeError, RMMapError
+from rpython.rlib.rstring import StringBuilder
 
 if rmmap.HAVE_LARGEFILE_SUPPORT:
     OFF_T = rarithmetic.r_longlong
@@ -163,17 +164,18 @@
         self.check_valid()
 
         space = self.space
-        start, stop, step = space.decode_index(w_index, self.mmap.size)
+        start, stop, step, length = space.decode_index4(w_index, self.mmap.size)
         if step == 0:  # index only
             return space.wrap(self.mmap.getitem(start))
         elif step == 1:
             if stop - start < 0:
                 return space.wrap("")
-            return space.wrap(self.mmap.getslice(start, stop - start))
+            return space.wrap(self.mmap.getslice(start, length))
         else:
-            res = "".join([self.mmap.getitem(i)
-                           for i in range(start, stop, step)])
-            return space.wrap(res)
+            b = StringBuilder(length)
+            for i in range(start, stop, step):
+                b.append(self.mmap.getitem(i))
+            return space.wrap(b.build())
 
     def descr_setitem(self, w_index, w_value):
         space = self.space
diff --git a/pypy/module/posix/test/test_posix2.py b/pypy/module/posix/test/test_posix2.py
--- a/pypy/module/posix/test/test_posix2.py
+++ b/pypy/module/posix/test/test_posix2.py
@@ -14,7 +14,7 @@
 import signal
 
 def setup_module(mod):
-    usemodules = ['binascii', 'posix', 'struct', 'rctime']
+    usemodules = ['binascii', 'posix', 'struct', 'time']
     if os.name != 'nt':
         usemodules += ['fcntl']
     else:
diff --git a/pypy/module/posix/test/test_posix_libfile.py b/pypy/module/posix/test/test_posix_libfile.py
--- a/pypy/module/posix/test/test_posix_libfile.py
+++ b/pypy/module/posix/test/test_posix_libfile.py
@@ -10,7 +10,7 @@
 
 class AppTestPosix:
     spaceconfig = {
-        "usemodules": ['posix', 'rctime'],
+        "usemodules": ['posix', 'time'],
     }
 
     def setup_class(cls):
diff --git a/pypy/module/pyexpat/test/test_parser.py b/pypy/module/pyexpat/test/test_parser.py
--- a/pypy/module/pyexpat/test/test_parser.py
+++ b/pypy/module/pyexpat/test/test_parser.py
@@ -177,7 +177,7 @@
 
 class AppTestPyexpat2:
     spaceconfig = dict(usemodules=['pyexpat', 'itertools', '_socket',
-                                   'rctime', 'struct', 'binascii'])
+                                   'time', 'struct', 'binascii'])
 
     def test_django_bug(self):
         xml_str = ''
diff --git a/pypy/module/pypyjit/test/test_policy.py b/pypy/module/pypyjit/test/test_policy.py
--- a/pypy/module/pypyjit/test/test_policy.py
+++ b/pypy/module/pypyjit/test/test_policy.py
@@ -29,7 +29,7 @@
     assert pypypolicy.look_inside_function(get_ident)
 
 def test_time():
-    from pypy.module.rctime.interp_time import time
+    from pypy.module.time.interp_time import time
     assert pypypolicy.look_inside_function(time)
 
 def test_io():
diff --git a/pypy/module/rctime/__init__.py b/pypy/module/rctime/__init__.py
deleted file mode 100644
--- a/pypy/module/rctime/__init__.py
+++ /dev/null
@@ -1,42 +0,0 @@
-
-from pypy.interpreter.mixedmodule import MixedModule
-import os
-
-_WIN = os.name == "nt"
-
-class Module(MixedModule):
-    applevel_name = 'time'
-
-    interpleveldefs = {
-        'time': 'interp_time.time',
-        'clock': 'interp_time.clock',
-        'ctime': 'interp_time.ctime',
-        'asctime': 'interp_time.asctime',
-        'gmtime': 'interp_time.gmtime',
-        'localtime': 'interp_time.localtime',
-        'mktime': 'interp_time.mktime',
-        'strftime': 'interp_time.strftime',
-        'sleep' : 'interp_time.sleep',
-    }
-
-    if os.name == "posix":
-        interpleveldefs['tzset'] = 'interp_time.tzset'
-
-    appleveldefs = {
-        'struct_time': 'app_time.struct_time',
-        '__doc__': 'app_time.__doc__',
-        'strptime': 'app_time.strptime',
-    }
-
-    def startup(self, space):
-        if _WIN:
-            from pypy.module.rctime.interp_time import State
-            space.fromcache(State).startup(space)
-
-        # this machinery is needed to expose constants
-        # that have to be initialized one time only
-        from pypy.module.rctime import interp_time
-
-        interp_time._init_timezone(space)
-        interp_time._init_accept2dyear(space)
-
diff --git a/pypy/module/rctime/interp_time.py b/pypy/module/rctime/interp_time.py
deleted file mode 100644
--- a/pypy/module/rctime/interp_time.py
+++ /dev/null
@@ -1,670 +0,0 @@
-from rpython.rtyper.tool import rffi_platform as platform
-from rpython.rtyper.lltypesystem import rffi
-from pypy.interpreter.error import OperationError, oefmt
-from pypy.interpreter.gateway import unwrap_spec
-from rpython.rtyper.lltypesystem import lltype
-from rpython.rlib.rarithmetic import intmask
-from rpython.rlib import rposix
-from rpython.translator.tool.cbuild import ExternalCompilationInfo
-import os
-import sys
-import time as pytime
-
-_POSIX = os.name == "posix"
-_WIN = os.name == "nt"
-_CYGWIN = sys.platform == "cygwin"
-
-_time_zones = []
-if _CYGWIN:
-    _time_zones = ["GMT-12", "GMT-11", "GMT-10", "GMT-9", "GMT-8", "GMT-7",
-                   "GMT-6", "GMT-5", "GMT-4", "GMT-3", "GMT-2", "GMT-1",
-                   "GMT",  "GMT+1", "GMT+2", "GMT+3", "GMT+4", "GMT+5",
-                   "GMT+6",  "GMT+7", "GMT+8", "GMT+9", "GMT+10", "GMT+11",
-                   "GMT+12",  "GMT+13", "GMT+14"]
-
-if _WIN:
-    # Interruptible sleeps on Windows:
-    # We install a specific Console Ctrl Handler which sets an 'event'.
-    # time.sleep() will actually call WaitForSingleObject with the desired
-    # timeout.  On Ctrl-C, the signal handler is called, the event is set,
-    # and the wait function exits.
-    from rpython.rlib import rwin32
-    from pypy.interpreter.error import wrap_windowserror, wrap_oserror
-    from rpython.rlib import rthread as thread
-
-    eci = ExternalCompilationInfo(
-        includes = ['windows.h'],
-        post_include_bits = [
-            "RPY_EXTERN\n"
-            "BOOL pypy_timemodule_setCtrlHandler(HANDLE event);"],
-        separate_module_sources=['''
-            static HANDLE interrupt_event;
-
-            static BOOL WINAPI CtrlHandlerRoutine(
-              DWORD dwCtrlType)
-            {
-                SetEvent(interrupt_event);
-                /* allow other default handlers to be called.
-                 * Default Python handler will setup the
-                 * KeyboardInterrupt exception.
-                 */
-                return 0;
-            }
-
-            BOOL pypy_timemodule_setCtrlHandler(HANDLE event)
-            {
-                interrupt_event = event;
-                return SetConsoleCtrlHandler(CtrlHandlerRoutine, TRUE);
-            }
-
-        '''],
-        )
-    _setCtrlHandlerRoutine = rffi.llexternal(
-        'pypy_timemodule_setCtrlHandler',
-        [rwin32.HANDLE], rwin32.BOOL,
-        compilation_info=eci)
-
-    class GlobalState:
-        def __init__(self):
-            self.init()
-
-        def init(self):
-            self.interrupt_event = rwin32.NULL_HANDLE
-
-        def startup(self, space):
-            # Initialize the event handle used to signal Ctrl-C
-            try:
-                globalState.interrupt_event = rwin32.CreateEvent(
-                    rffi.NULL, True, False, rffi.NULL)
-            except WindowsError, e:
-                raise wrap_windowserror(space, e)
-            if not _setCtrlHandlerRoutine(globalState.interrupt_event):
-                raise wrap_windowserror(
-                    space, rwin32.lastWindowsError("SetConsoleCtrlHandler"))
-
-    globalState = GlobalState()
-
-    class State:
-        def __init__(self, space):
-            self.main_thread = 0
-
-        def _cleanup_(self):
-            self.main_thread = 0
-            globalState.init()
-
-        def startup(self, space):
-            self.main_thread = thread.get_ident()
-            globalState.startup(space)
-
-        def get_interrupt_event(self):
-            return globalState.interrupt_event
-
-
-_includes = ["time.h"]
-if _POSIX:
-    _includes.append('sys/time.h')
-
-class CConfig:
-    _compilation_info_ = ExternalCompilationInfo(
-        includes = _includes
-    )
-    CLOCKS_PER_SEC = platform.ConstantInteger("CLOCKS_PER_SEC")
-    clock_t = platform.SimpleType("clock_t", rffi.ULONG)
-    has_gettimeofday = platform.Has('gettimeofday')
-
-if _POSIX:
-    calling_conv = 'c'
-    CConfig.timeval = platform.Struct("struct timeval",
-                                      [("tv_sec", rffi.INT),
-                                       ("tv_usec", rffi.INT)])
-    if _CYGWIN:
-        CConfig.tm = platform.Struct("struct tm", [("tm_sec", rffi.INT),
-            ("tm_min", rffi.INT), ("tm_hour", rffi.INT), ("tm_mday", rffi.INT),
-            ("tm_mon", rffi.INT), ("tm_year", rffi.INT), ("tm_wday", rffi.INT),
-            ("tm_yday", rffi.INT), ("tm_isdst", rffi.INT)])
-    else:
-        CConfig.tm = platform.Struct("struct tm", [("tm_sec", rffi.INT),
-            ("tm_min", rffi.INT), ("tm_hour", rffi.INT), ("tm_mday", rffi.INT),
-            ("tm_mon", rffi.INT), ("tm_year", rffi.INT), ("tm_wday", rffi.INT),
-            ("tm_yday", rffi.INT), ("tm_isdst", rffi.INT), ("tm_gmtoff", rffi.LONG),
-            ("tm_zone", rffi.CCHARP)])
-elif _WIN:
-    calling_conv = 'win'
-    CConfig.tm = platform.Struct("struct tm", [("tm_sec", rffi.INT),
-        ("tm_min", rffi.INT), ("tm_hour", rffi.INT), ("tm_mday", rffi.INT),
-        ("tm_mon", rffi.INT), ("tm_year", rffi.INT), ("tm_wday", rffi.INT),
-        ("tm_yday", rffi.INT), ("tm_isdst", rffi.INT)])
-
-class cConfig:
-    pass
-
-for k, v in platform.configure(CConfig).items():
-    setattr(cConfig, k, v)
-cConfig.tm.__name__ = "_tm"
-
-def external(name, args, result, eci=CConfig._compilation_info_):
-    if _WIN and rffi.sizeof(rffi.TIME_T) == 8:
-        # Recent Microsoft compilers use 64bit time_t and
-        # the corresponding functions are named differently
-        if (rffi.TIME_T in args or rffi.TIME_TP in args
-            or result in (rffi.TIME_T, rffi.TIME_TP)):
-            name = '_' + name + '64'
-    return rffi.llexternal(name, args, result,
-                           compilation_info=eci,
-                           calling_conv=calling_conv,
-                           releasegil=False)
-
-if _POSIX:
-    cConfig.timeval.__name__ = "_timeval"
-    timeval = cConfig.timeval
-
-CLOCKS_PER_SEC = cConfig.CLOCKS_PER_SEC
-clock_t = cConfig.clock_t
-tm = cConfig.tm
-glob_buf = lltype.malloc(tm, flavor='raw', zero=True, immortal=True)
-
-if cConfig.has_gettimeofday:
-    c_gettimeofday = external('gettimeofday', [rffi.VOIDP, rffi.VOIDP], rffi.INT)
-TM_P = lltype.Ptr(tm)
-c_clock = external('clock', [rffi.TIME_TP], clock_t)
-c_time = external('time', [rffi.TIME_TP], rffi.TIME_T)
-c_ctime = external('ctime', [rffi.TIME_TP], rffi.CCHARP)
-c_gmtime = external('gmtime', [rffi.TIME_TP], TM_P)
-c_mktime = external('mktime', [TM_P], rffi.TIME_T)
-c_asctime = external('asctime', [TM_P], rffi.CCHARP)
-c_localtime = external('localtime', [rffi.TIME_TP], TM_P)
-if _POSIX:
-    c_tzset = external('tzset', [], lltype.Void)
-if _WIN:
-    win_eci = ExternalCompilationInfo(
-        includes = ["time.h"],
-        post_include_bits = ["RPY_EXTERN "
-                             "long pypy_get_timezone();\n"
-                             "RPY_EXTERN "
-                             "int pypy_get_daylight();\n"
-                             "RPY_EXTERN "
-                             "char** pypy_get_tzname();\n"
-                             "RPY_EXTERN "
-                             "void pypy__tzset();"],
-        separate_module_sources = ["""
-        long pypy_get_timezone() { return timezone; }
-        int pypy_get_daylight() { return daylight; }
-        char** pypy_get_tzname() { return tzname; }
-        void pypy__tzset() { _tzset(); }
-        """])
-    # Ensure sure that we use _tzset() and timezone from the same C Runtime.
-    c_tzset = external('pypy__tzset', [], lltype.Void, win_eci)
-    c_get_timezone = external('pypy_get_timezone', [], rffi.LONG, win_eci)
-    c_get_daylight = external('pypy_get_daylight', [], rffi.INT, win_eci)
-    c_get_tzname = external('pypy_get_tzname', [], rffi.CCHARPP, win_eci)
-
-c_strftime = external('strftime', [rffi.CCHARP, rffi.SIZE_T, rffi.CCHARP, TM_P],
-                      rffi.SIZE_T)
-
-def _init_accept2dyear(space):
-    if os.environ.get("PYTHONY2K"):
-        accept2dyear = 0
-    else:
-        accept2dyear = 1
-    _set_module_object(space, "accept2dyear", space.wrap(accept2dyear))
-
-def _init_timezone(space):
-    timezone = daylight = altzone = 0
-    tzname = ["", ""]
-
-    if _WIN:
-        c_tzset()
-        timezone = c_get_timezone()
-        altzone = timezone - 3600
-        daylight = c_get_daylight()
-        tzname_ptr = c_get_tzname()
-        tzname = rffi.charp2str(tzname_ptr[0]), rffi.charp2str(tzname_ptr[1])
-
-    if _POSIX:
-        if _CYGWIN:
-            YEAR = (365 * 24 + 6) * 3600
-
-            # about January 11th
-            t = (((c_time(lltype.nullptr(rffi.TIME_TP.TO))) / YEAR) * YEAR + 10 * 24 * 3600)
-            # we cannot have reference to stack variable, put it on the heap
-            t_ref = lltype.malloc(rffi.TIME_TP.TO, 1, flavor='raw')
-            t_ref[0] = rffi.cast(rffi.TIME_T, t)
-            p = c_localtime(t_ref)
-            q = c_gmtime(t_ref)
-            janzone = (p.c_tm_hour + 24 * p.c_tm_mday) - (q.c_tm_hour + 24 * q.c_tm_mday)
-            if janzone < -12:
-                janname = "   "
-            elif janzone > 14:
-                janname = "   "
-            else:
-                janname = _time_zones[janzone - 12]
-            janzone = janzone * 3600
-            # about July 11th
-            tt = t + YEAR / 2
-            t_ref[0] = rffi.cast(rffi.TIME_T, tt)
-            p = c_localtime(t_ref)
-            q = c_gmtime(t_ref)
-            julyzone = (p.c_tm_hour + 24 * p.c_tm_mday) - (q.c_tm_hour + 24 * q.c_tm_mday)
-            if julyzone < -12:
-                julyname = "   "
-            elif julyzone > 14:
-                julyname = "   "
-            else:
-                julyname = _time_zones[julyzone - 12]
-            julyzone = julyzone * 3600
-            lltype.free(t_ref, flavor='raw')
-
-            if janzone < julyzone:
-                # DST is reversed in the southern hemisphere
-                timezone = julyzone
-                altzone = janzone
-                daylight = int(janzone != julyzone)
-                tzname = [julyname, janname]
-            else:
-                timezone = janzone
-                altzone = julyzone
-                daylight = int(janzone != julyzone)
-                tzname = [janname, julyname]
-
-        else:
-            YEAR = (365 * 24 + 6) * 3600
-
-            t = (((c_time(lltype.nullptr(rffi.TIME_TP.TO))) / YEAR) * YEAR)
-            # we cannot have reference to stack variable, put it on the heap
-            t_ref = lltype.malloc(rffi.TIME_TP.TO, 1, flavor='raw')
-            t_ref[0] = rffi.cast(rffi.TIME_T, t)
-            p = c_localtime(t_ref)
-            janzone = -p.c_tm_gmtoff
-            tm_zone = rffi.charp2str(p.c_tm_zone)
-            janname = ["   ", tm_zone][bool(tm_zone)]
-            tt = t + YEAR / 2
-            t_ref[0] = rffi.cast(rffi.TIME_T, tt)
-            p = c_localtime(t_ref)
-            lltype.free(t_ref, flavor='raw')
-            tm_zone = rffi.charp2str(p.c_tm_zone)
-            julyzone = -p.c_tm_gmtoff
-            julyname = ["   ", tm_zone][bool(tm_zone)]
-
-            if janzone < julyzone:
-                # DST is reversed in the southern hemisphere
-                timezone = julyzone
-                altzone = janzone
-                daylight = int(janzone != julyzone)
-                tzname = [julyname, janname]
-            else:
-                timezone = janzone
-                altzone = julyzone
-                daylight = int(janzone != julyzone)
-                tzname = [janname, julyname]
-
-    _set_module_object(space, "timezone", space.wrap(timezone))
-    _set_module_object(space, 'daylight', space.wrap(daylight))
-    tzname_w = [space.wrap(tzname[0]), space.wrap(tzname[1])]
-    _set_module_object(space, 'tzname', space.newtuple(tzname_w))
-    _set_module_object(space, 'altzone', space.wrap(altzone))
-
-def _get_error_msg():
-    errno = rposix.get_errno()
-    return os.strerror(errno)
-
-if sys.platform != 'win32':
-    @unwrap_spec(secs=float)
-    def sleep(space, secs):
-        if secs < 0:
-            raise OperationError(space.w_IOError,
-                                 space.wrap("Invalid argument: negative time in sleep"))
-        pytime.sleep(secs)
-else:
-    from rpython.rlib import rwin32
-    from errno import EINTR
-    def _simple_sleep(space, secs, interruptible):
-        if secs == 0.0 or not interruptible:
-            pytime.sleep(secs)
-        else:
-            millisecs = int(secs * 1000)
-            interrupt_event = space.fromcache(State).get_interrupt_event()
-            rwin32.ResetEvent(interrupt_event)
-            rc = rwin32.WaitForSingleObject(interrupt_event, millisecs)
-            if rc == rwin32.WAIT_OBJECT_0:
-                # Yield to make sure real Python signal handler
-                # called.
-                pytime.sleep(0.001)
-                raise wrap_oserror(space,
-                                   OSError(EINTR, "sleep() interrupted"))
-    @unwrap_spec(secs=float)
-    def sleep(space, secs):
-        if secs < 0:
-            raise OperationError(space.w_IOError,
-                                 space.wrap("Invalid argument: negative time in sleep"))
-        # as decreed by Guido, only the main thread can be
-        # interrupted.
-        main_thread = space.fromcache(State).main_thread
-        interruptible = (main_thread == thread.get_ident())
-        MAX = sys.maxint / 1000.0 # > 24 days
-        while secs > MAX:
-            _simple_sleep(space, MAX, interruptible)
-            secs -= MAX
-        _simple_sleep(space, secs, interruptible)
-
-def _get_module_object(space, obj_name):
-    w_module = space.getbuiltinmodule('time')
-    w_obj = space.getattr(w_module, space.wrap(obj_name))
-    return w_obj
-
-def _set_module_object(space, obj_name, w_obj_value):
-    w_module = space.getbuiltinmodule('time')
-    space.setattr(w_module, space.wrap(obj_name), w_obj_value)
-
-def _get_inttime(space, w_seconds):
-    # w_seconds can be a wrapped None (it will be automatically wrapped
-    # in the callers, so we never get a real None here).
-    if space.is_none(w_seconds):
-        seconds = pytime.time()
-    else:
-        seconds = space.float_w(w_seconds)
-    #
-    t = rffi.cast(rffi.TIME_T, seconds)
-    #
-    # Logic from CPython: How much info did we lose?  We assume that
-    # time_t is an integral type.  If we lost a second or more, the
-    # input doesn't fit in a time_t; call it an error.
-    diff = seconds - rffi.cast(lltype.Float, t)
-    if diff <= -1.0 or diff >= 1.0:
-        raise OperationError(space.w_ValueError,
-                      space.wrap("timestamp out of range for platform time_t"))
-    return t
-
-def _tm_to_tuple(space, t):
-    time_tuple = [
-        space.wrap(rffi.getintfield(t, 'c_tm_year') + 1900),
-        space.wrap(rffi.getintfield(t, 'c_tm_mon') + 1), # want january == 1
-        space.wrap(rffi.getintfield(t, 'c_tm_mday')),
-        space.wrap(rffi.getintfield(t, 'c_tm_hour')),
-        space.wrap(rffi.getintfield(t, 'c_tm_min')),
-        space.wrap(rffi.getintfield(t, 'c_tm_sec')),
-        space.wrap((rffi.getintfield(t, 'c_tm_wday') + 6) % 7), # want monday == 0
-        space.wrap(rffi.getintfield(t, 'c_tm_yday') + 1), # want january, 1 == 1
-        space.wrap(rffi.getintfield(t, 'c_tm_isdst'))]
-
-    w_struct_time = _get_module_object(space, 'struct_time')
-    w_time_tuple = space.newtuple(time_tuple)
-    return space.call_function(w_struct_time, w_time_tuple)
-
-def _gettmarg(space, w_tup, allowNone=True):
-    if space.is_none(w_tup):
-        if not allowNone:
-            raise OperationError(space.w_TypeError,
-                                 space.wrap("tuple expected"))
-        # default to the current local time
-        tt = rffi.r_time_t(int(pytime.time()))
-        t_ref = lltype.malloc(rffi.TIME_TP.TO, 1, flavor='raw')
-        t_ref[0] = tt
-        pbuf = c_localtime(t_ref)
-        lltype.free(t_ref, flavor='raw')
-        if not pbuf:
-            raise OperationError(space.w_ValueError,
-                space.wrap(_get_error_msg()))
-        return pbuf
-
-    tup_w = space.fixedview(w_tup)
-    if len(tup_w) != 9:
-        raise oefmt(space.w_TypeError,
-                    "argument must be sequence of length 9, not %d",
-                    len(tup_w))
-
-    y = space.int_w(tup_w[0])
-    tm_mon = space.int_w(tup_w[1])
-    if tm_mon == 0:
-        tm_mon = 1
-    tm_mday = space.int_w(tup_w[2])
-    if tm_mday == 0:
-        tm_mday = 1
-    tm_yday = space.int_w(tup_w[7])
-    if tm_yday == 0:
-        tm_yday = 1
-    rffi.setintfield(glob_buf, 'c_tm_mon', tm_mon)
-    rffi.setintfield(glob_buf, 'c_tm_mday', tm_mday)
-    rffi.setintfield(glob_buf, 'c_tm_hour', space.int_w(tup_w[3]))
-    rffi.setintfield(glob_buf, 'c_tm_min', space.int_w(tup_w[4]))
-    rffi.setintfield(glob_buf, 'c_tm_sec', space.int_w(tup_w[5]))
-    rffi.setintfield(glob_buf, 'c_tm_wday', space.int_w(tup_w[6]))
-    rffi.setintfield(glob_buf, 'c_tm_yday', tm_yday)
-    rffi.setintfield(glob_buf, 'c_tm_isdst', space.int_w(tup_w[8]))
-    if _POSIX:
-        if _CYGWIN:
-            pass
-        else:
-            # actually never happens, but makes annotator happy
-            glob_buf.c_tm_zone = lltype.nullptr(rffi.CCHARP.TO)
-            rffi.setintfield(glob_buf, 'c_tm_gmtoff', 0)
-
-    if y < 1900:
-        w_accept2dyear = _get_module_object(space, "accept2dyear")
-        accept2dyear = space.int_w(w_accept2dyear)
-
-        if not accept2dyear:
-            raise OperationError(space.w_ValueError,
-                space.wrap("year >= 1900 required"))
-
-        if 69 <= y <= 99:
-            y += 1900
-        elif 0 <= y <= 68:
-            y += 2000
-        else:
-            raise OperationError(space.w_ValueError,
-                space.wrap("year out of range"))
-
-    # tm_wday does not need checking of its upper-bound since taking "%
-    #  7" in gettmarg() automatically restricts the range.
-    if rffi.getintfield(glob_buf, 'c_tm_wday') < -1:
-        raise OperationError(space.w_ValueError,
-                             space.wrap("day of week out of range"))
-
-    rffi.setintfield(glob_buf, 'c_tm_year', y - 1900)
-    rffi.setintfield(glob_buf, 'c_tm_mon',
-                     rffi.getintfield(glob_buf, 'c_tm_mon') - 1)
-    rffi.setintfield(glob_buf, 'c_tm_wday',
-                     (rffi.getintfield(glob_buf, 'c_tm_wday') + 1) % 7)
-    rffi.setintfield(glob_buf, 'c_tm_yday',
-                     rffi.getintfield(glob_buf, 'c_tm_yday') - 1)
-
-    return glob_buf
-
-def time(space):
-    """time() -> floating point number
-
-    Return the current time in seconds since the Epoch.
-    Fractions of a second may be present if the system clock provides them."""
-
-    secs = pytime.time()
-    return space.wrap(secs)
-
-if _WIN:
-    class PCCache:
-        pass
-    pccache = PCCache()
-    pccache.divisor = 0.0
-    pccache.ctrStart = 0
-
-def clock(space):
-    """clock() -> floating point number
-
-    Return the CPU time or real time since the start of the process or since
-    the first call to clock().  This has as much precision as the system
-    records."""
-
-    return space.wrap(pytime.clock())
-
-def ctime(space, w_seconds=None):
-    """ctime([seconds]) -> string
-
-    Convert a time in seconds since the Epoch to a string in local time.
-    This is equivalent to asctime(localtime(seconds)). When the time tuple is
-    not present, current time as returned by localtime() is used."""
-
-    seconds = _get_inttime(space, w_seconds)
-
-    t_ref = lltype.malloc(rffi.TIME_TP.TO, 1, flavor='raw')
-    t_ref[0] = seconds
-    p = c_ctime(t_ref)
-    lltype.free(t_ref, flavor='raw')
-    if not p:
-        raise OperationError(space.w_ValueError,
-            space.wrap("unconvertible time"))
-
-    return space.wrap(rffi.charp2str(p)[:-1]) # get rid of new line
-
-# by now w_tup is an optional argument (and not *args)
-# because of the ext. compiler bugs in handling such arguments (*args, **kwds)
-def asctime(space, w_tup=None):
-    """asctime([tuple]) -> string
-
-    Convert a time tuple to a string, e.g. 'Sat Jun 06 16:26:11 1998'.
-    When the time tuple is not present, current time as returned by localtime()
-    is used."""
-    buf_value = _gettmarg(space, w_tup)
-    p = c_asctime(buf_value)
-    if not p:
-        raise OperationError(space.w_ValueError,
-            space.wrap("unconvertible time"))
-
-    return space.wrap(rffi.charp2str(p)[:-1]) # get rid of new line
-
-def gmtime(space, w_seconds=None):
-    """gmtime([seconds]) -> (tm_year, tm_mon, tm_day, tm_hour, tm_min,
-                          tm_sec, tm_wday, tm_yday, tm_isdst)
-
-    Convert seconds since the Epoch to a time tuple expressing UTC (a.k.a.
-    GMT).  When 'seconds' is not passed in, convert the current time instead.
-    """
-
-    # rpython does not support that a variable has two incompatible builtins
-    # as value so we have to duplicate the code. NOT GOOD! see localtime() too
-    seconds = _get_inttime(space, w_seconds)
-    t_ref = lltype.malloc(rffi.TIME_TP.TO, 1, flavor='raw')
-    t_ref[0] = seconds
-    p = c_gmtime(t_ref)
-    lltype.free(t_ref, flavor='raw')
-
-    if not p:
-        raise OperationError(space.w_ValueError, space.wrap(_get_error_msg()))
-    return _tm_to_tuple(space, p)
-
-def localtime(space, w_seconds=None):
-    """localtime([seconds]) -> (tm_year, tm_mon, tm_day, tm_hour, tm_min,
-                             tm_sec, tm_wday, tm_yday, tm_isdst)
-
-    Convert seconds since the Epoch to a time tuple expressing local time.
-    When 'seconds' is not passed in, convert the current time instead."""
-
-    seconds = _get_inttime(space, w_seconds)
-    t_ref = lltype.malloc(rffi.TIME_TP.TO, 1, flavor='raw')
-    t_ref[0] = seconds
-    p = c_localtime(t_ref)
-    lltype.free(t_ref, flavor='raw')
-
-    if not p:
-        raise OperationError(space.w_ValueError, space.wrap(_get_error_msg()))
-    return _tm_to_tuple(space, p)
-
-def mktime(space, w_tup):
-    """mktime(tuple) -> floating point number
-
-    Convert a time tuple in local time to seconds since the Epoch."""
-
-    buf = _gettmarg(space, w_tup, allowNone=False)
-    rffi.setintfield(buf, "c_tm_wday", -1)
-    tt = c_mktime(buf)
-    # A return value of -1 does not necessarily mean an error, but tm_wday
-    # cannot remain set to -1 if mktime succeeds.
-    if tt == -1 and rffi.getintfield(buf, "c_tm_wday") == -1:
-        raise OperationError(space.w_OverflowError,
-            space.wrap("mktime argument out of range"))
-
-    return space.wrap(float(tt))
-
-if _POSIX:
-    def tzset(space):
-        """tzset()
-
-        Initialize, or reinitialize, the local timezone to the value stored in
-        os.environ['TZ']. The TZ environment variable should be specified in
-        standard Unix timezone format as documented in the tzset man page
-        (eg. 'US/Eastern', 'Europe/Amsterdam'). Unknown timezones will silently
-        fall back to UTC. If the TZ environment variable is not set, the local
-        timezone is set to the systems best guess of wallclock time.
-        Changing the TZ environment variable without calling tzset *may* change
-        the local timezone used by methods such as localtime, but this behaviour
-        should not be relied on"""
-
-        c_tzset()
-
-        # reset timezone, altzone, daylight and tzname
-        _init_timezone(space)
-
- at unwrap_spec(format=str)
-def strftime(space, format, w_tup=None):
-    """strftime(format[, tuple]) -> string
-
-    Convert a time tuple to a string according to a format specification.
-    See the library reference manual for formatting codes. When the time tuple
-    is not present, current time as returned by localtime() is used."""
-    buf_value = _gettmarg(space, w_tup)
-
-    # Checks added to make sure strftime() does not crash Python by
-    # indexing blindly into some array for a textual representation
-    # by some bad index (fixes bug #897625).
-    # No check for year since handled in gettmarg().
-    if rffi.getintfield(buf_value, 'c_tm_mon') < 0 or rffi.getintfield(buf_value, 'c_tm_mon') > 11:
-        raise OperationError(space.w_ValueError,
-                             space.wrap("month out of range"))
-    if rffi.getintfield(buf_value, 'c_tm_mday') < 1 or rffi.getintfield(buf_value, 'c_tm_mday') > 31:
-        raise OperationError(space.w_ValueError,
-                             space.wrap("day of month out of range"))
-    if rffi.getintfield(buf_value, 'c_tm_hour') < 0 or rffi.getintfield(buf_value, 'c_tm_hour') > 23:
-        raise OperationError(space.w_ValueError,
-                             space.wrap("hour out of range"))
-    if rffi.getintfield(buf_value, 'c_tm_min') < 0 or rffi.getintfield(buf_value, 'c_tm_min') > 59:
-        raise OperationError(space.w_ValueError,
-                             space.wrap("minute out of range"))
-    if rffi.getintfield(buf_value, 'c_tm_sec') < 0 or rffi.getintfield(buf_value, 'c_tm_sec') > 61:
-        raise OperationError(space.w_ValueError,
-                             space.wrap("seconds out of range"))
-    if rffi.getintfield(buf_value, 'c_tm_yday') < 0 or rffi.getintfield(buf_value, 'c_tm_yday') > 365:
-        raise OperationError(space.w_ValueError,
-                             space.wrap("day of year out of range"))
-    if rffi.getintfield(buf_value, 'c_tm_isdst') < -1 or rffi.getintfield(buf_value, 'c_tm_isdst') > 1:
-        raise OperationError(space.w_ValueError,
-                             space.wrap("daylight savings flag out of range"))
-
-    if _WIN:
-        # check that the format string contains only valid directives
-        length = len(format)
-        i = 0
-        while i < length:
-            if format[i] == '%':
-                i += 1
-                if i < length and format[i] == '#':
-                    # not documented by python
-                    i += 1
-                if i >= length or format[i] not in "aAbBcdHIjmMpSUwWxXyYzZ%":
-                    raise OperationError(space.w_ValueError,
-                                         space.wrap("invalid format string"))
-            i += 1
-
-    i = 1024
-    while True:
-        outbuf = lltype.malloc(rffi.CCHARP.TO, i, flavor='raw')
-        try:
-            buflen = c_strftime(outbuf, i, format, buf_value)
-            if buflen > 0 or i >= 256 * len(format):
-                # if the buffer is 256 times as long as the format,
-                # it's probably not failing for lack of room!
-                # More likely, the format yields an empty result,
-                # e.g. an empty format, or %Z when the timezone
-                # is unknown.
-                result = rffi.charp2strn(outbuf, intmask(buflen))
-                return space.wrap(result)
-        finally:
-            lltype.free(outbuf, flavor='raw')
-        i += i
diff --git a/pypy/module/rctime/test/test_rctime.py b/pypy/module/rctime/test/test_rctime.py
deleted file mode 100644
--- a/pypy/module/rctime/test/test_rctime.py
+++ /dev/null
@@ -1,333 +0,0 @@
-class AppTestRCTime:
-    spaceconfig = {
-        "usemodules": ['rctime', 'struct', 'binascii'],
-    }
-
-    def test_attributes(self):
-        import time as rctime
-        assert isinstance(rctime.accept2dyear, int)
-        assert isinstance(rctime.altzone, int)
-        assert isinstance(rctime.daylight, int)
-        assert isinstance(rctime.timezone, int)
-        assert isinstance(rctime.tzname, tuple)
-        assert isinstance(rctime.__doc__, str)
-
-    def test_sleep(self):
-        import time as rctime
-        import sys
-        import os
-        raises(TypeError, rctime.sleep, "foo")
-        rctime.sleep(0.12345)
-        raises(IOError, rctime.sleep, -1.0)
-
-    def test_clock(self):
-        import time as rctime
-        rctime.clock()
-        assert isinstance(rctime.clock(), float)
-
-    def test_time(self):
-        import time as rctime
-        t1 = rctime.time()
-        assert isinstance(rctime.time(), float)
-        assert rctime.time() != 0.0 # 0.0 means failure
-        rctime.sleep(0.02)
-        t2 = rctime.time()
-        assert t1 != t2       # the resolution should be at least 0.01 secs
-
-    def test_ctime(self):
-        import time as rctime
-        raises(TypeError, rctime.ctime, "foo")
-        rctime.ctime(None)
-        rctime.ctime()
-        res = rctime.ctime(0)
-        assert isinstance(res, str)
-        rctime.ctime(rctime.time())
-        raises(ValueError, rctime.ctime, 1E200)
-        raises(OverflowError, rctime.ctime, 10**900)
-
-    def test_gmtime(self):
-        import time as rctime
-        raises(TypeError, rctime.gmtime, "foo")
-        rctime.gmtime()
-        rctime.gmtime(None)
-        rctime.gmtime(0)
-        res = rctime.gmtime(rctime.time())
-        assert isinstance(res, rctime.struct_time)
-        assert res[-1] == 0 # DST is always zero in gmtime()
-        t0 = rctime.mktime(rctime.gmtime())
-        t1 = rctime.mktime(rctime.gmtime(None))
-        assert 0 <= (t1 - t0) < 1.2
-        t = rctime.time()
-        assert rctime.gmtime(t) == rctime.gmtime(t)
-        raises(ValueError, rctime.gmtime, 2**64)
-        raises(ValueError, rctime.gmtime, -2**64)
-
-    def test_localtime(self):
-        import time as rctime
-        import os
-        raises(TypeError, rctime.localtime, "foo")
-        rctime.localtime()
-        rctime.localtime(None)
-        rctime.localtime(0)
-        res = rctime.localtime(rctime.time())
-        assert isinstance(res, rctime.struct_time)
-        t0 = rctime.mktime(rctime.localtime())
-        t1 = rctime.mktime(rctime.localtime(None))
-        assert 0 <= (t1 - t0) < 1.2
-        t = rctime.time()
-        assert rctime.localtime(t) == rctime.localtime(t)
-        if os.name == 'nt':
-            raises(ValueError, rctime.localtime, -1)
-        else:
-            rctime.localtime(-1)
-
-    def test_mktime(self):
-        import time as rctime
-        import os, sys
-        raises(TypeError, rctime.mktime, "foo")
-        raises(TypeError, rctime.mktime, None)
-        raises(TypeError, rctime.mktime, (1, 2))
-        raises(TypeError, rctime.mktime, (1, 2, 3, 4, 5, 6, 'f', 8, 9))
-        res = rctime.mktime(rctime.localtime())
-        assert isinstance(res, float)
-
-        ltime = rctime.localtime()
-        rctime.accept2dyear == 0
-        ltime = list(ltime)
-        ltime[0] = 1899
-        raises(ValueError, rctime.mktime, tuple(ltime))
-        rctime.accept2dyear == 1
-
-        ltime = list(ltime)
-        ltime[0] = 67
-        ltime = tuple(ltime)
-        if os.name != "nt" and sys.maxint < 1<<32:   # time_t may be 64bit
-            raises(OverflowError, rctime.mktime, ltime)
-
-        ltime = list(ltime)
-        ltime[0] = 100
-        raises(ValueError, rctime.mktime, tuple(ltime))
-
-        t = rctime.time()
-        assert long(rctime.mktime(rctime.localtime(t))) == long(t)
-        assert long(rctime.mktime(rctime.gmtime(t))) - rctime.timezone == long(t)
-        ltime = rctime.localtime()
-        assert rctime.mktime(tuple(ltime)) == rctime.mktime(ltime)
-        if os.name != 'nt':
-            assert rctime.mktime(rctime.localtime(-1)) == -1
-
-        res = rctime.mktime((2000, 1, 1, 0, 0, 0, -1, -1, -1))
-        if os.name == 'nt':
-            assert rctime.ctime(res) == 'Sat Jan 01 00:00:00 2000'
-        else:
-            assert rctime.ctime(res) == 'Sat Jan  1 00:00:00 2000'
-
-    def test_asctime(self):
-        import time as rctime
-        rctime.asctime()
-        # raises(TypeError, rctime.asctime, None)
-        raises(TypeError, rctime.asctime, ())
-        raises(TypeError, rctime.asctime, (1,))
-        raises(TypeError, rctime.asctime, range(8))
-        raises(TypeError, rctime.asctime, (1, 2))
-        raises(TypeError, rctime.asctime, (1, 2, 3, 4, 5, 6, 'f', 8, 9))
-        raises(TypeError, rctime.asctime, "foo")
-        res = rctime.asctime()
-        assert isinstance(res, str)
-        rctime.asctime(rctime.localtime())
-        t = rctime.time()
-        assert rctime.ctime(t) == rctime.asctime(rctime.localtime(t))
-        if rctime.timezone:
-            assert rctime.ctime(t) != rctime.asctime(rctime.gmtime(t))
-        ltime = rctime.localtime()
-        assert rctime.asctime(tuple(ltime)) == rctime.asctime(ltime)
-        try:
-            rctime.asctime((12345,) + (0,) * 8)  # assert this doesn't crash
-        except ValueError:
-            pass  # some OS (ie POSIXes besides Linux) reject year > 9999
-
-    def test_accept2dyear_access(self):
-        import time as rctime
-
-        accept2dyear = rctime.accept2dyear
-        del rctime.accept2dyear
-        try:
-            # with year >= 1900 this shouldn't need to access accept2dyear
-            assert rctime.asctime((2000,) + (0,) * 8).split()[-1] == '2000'
-        finally:
-            rctime.accept2dyear = accept2dyear
-
-    def test_struct_time(self):
-        import time as rctime
-        raises(TypeError, rctime.struct_time)
-        raises(TypeError, rctime.struct_time, "foo")
-        raises(TypeError, rctime.struct_time, (1, 2, 3))
-        tup = (1, 2, 3, 4, 5, 6, 7, 8, 9)
-        st_time = rctime.struct_time(tup)
-        assert str(st_time).startswith('time.struct_time(tm_year=1, ')
-        assert len(st_time) == len(tup)
-
-    def test_tzset(self):
-        import time as rctime
-        import os
-
-        if not os.name == "posix":
-            skip("tzset available only under Unix")
-
-        # epoch time of midnight Dec 25th 2002. Never DST in northern
-        # hemisphere.
-        xmas2002 = 1040774400.0
-
-        # these formats are correct for 2002, and possibly future years
-        # this format is the 'standard' as documented at:
-        # http://www.opengroup.org/onlinepubs/007904975/basedefs/xbd_chap08.html
-        # They are also documented in the tzset(3) man page on most Unix
-        # systems.
-        eastern = 'EST+05EDT,M4.1.0,M10.5.0'
-        victoria = 'AEST-10AEDT-11,M10.5.0,M3.5.0'
-        utc = 'UTC+0'
-
-        org_TZ = os.environ.get('TZ', None)
-        try:
-            # Make sure we can switch to UTC time and results are correct
-            # Note that unknown timezones default to UTC.
-            # Note that altzone is undefined in UTC, as there is no DST
-            os.environ['TZ'] = eastern
-            rctime.tzset()
-            os.environ['TZ'] = utc
-            rctime.tzset()
-            assert rctime.gmtime(xmas2002) == rctime.localtime(xmas2002)
-            assert rctime.daylight == 0
-            assert rctime.timezone == 0
-            assert rctime.localtime(xmas2002).tm_isdst == 0
-
-            # make sure we can switch to US/Eastern
-            os.environ['TZ'] = eastern
-            rctime.tzset()
-            assert rctime.gmtime(xmas2002) != rctime.localtime(xmas2002)
-            assert rctime.tzname == ('EST', 'EDT')
-            assert len(rctime.tzname) == 2
-            assert rctime.daylight == 1
-            assert rctime.timezone == 18000
-            assert rctime.altzone == 14400
-            assert rctime.localtime(xmas2002).tm_isdst == 0
-
-            # now go to the southern hemisphere.
-            os.environ['TZ'] = victoria
-            rctime.tzset()
-            assert rctime.gmtime(xmas2002) != rctime.localtime(xmas2002)
-            assert rctime.tzname[0] == 'AEST'
-            assert rctime.tzname[1] == 'AEDT'
-            assert len(rctime.tzname) == 2
-            assert rctime.daylight == 1
-            assert rctime.timezone == -36000
-            assert rctime.altzone == -39600
-            assert rctime.localtime(xmas2002).tm_isdst == 1
-        finally:
-            # repair TZ environment variable in case any other tests
-            # rely on it.
-            if org_TZ is not None:
-                os.environ['TZ'] = org_TZ
-            elif os.environ.has_key('TZ'):
-                del os.environ['TZ']
-            rctime.tzset()
-
-    def test_strftime(self):
-        import time as rctime
-        import os, sys
-
-        t = rctime.time()
-        tt = rctime.gmtime(t)
-        for directive in ('a', 'A', 'b', 'B', 'c', 'd', 'H', 'I',
-                          'j', 'm', 'M', 'p', 'S',
-                          'U', 'w', 'W', 'x', 'X', 'y', 'Y', 'Z', '%'):
-            format = ' %' + directive
-            rctime.strftime(format, tt)
-
-        raises(TypeError, rctime.strftime, ())
-        raises(TypeError, rctime.strftime, (1,))
-        raises(TypeError, rctime.strftime, range(8))
-        exp = '2000 01 01 00 00 00 1 001'
-        assert rctime.strftime("%Y %m %d %H %M %S %w %j", (0,)*9) == exp
-
-        # Guard against invalid/non-supported format string
-        # so that Python don't crash (Windows crashes when the format string
-        # input to [w]strftime is not kosher.
-        if os.name == 'nt':
-            raises(ValueError, rctime.strftime, '%f')
-        elif sys.platform == 'darwin' or 'bsd' in sys.platform:
-            # darwin strips % of unknown format codes
-            # http://bugs.python.org/issue9811
-            assert rctime.strftime('%f') == 'f'
-        else:
-            assert rctime.strftime('%f') == '%f'
-
-    def test_strftime_ext(self):
-        import time as rctime
-
-        tt = rctime.gmtime()
-        try:
-            result = rctime.strftime('%D', tt)
-        except ValueError:
-            pass
-        else:
-            assert result == rctime.strftime('%m/%d/%y', tt)
-
-    def test_strftime_bounds_checking(self):
-        import time as rctime
-
-        # make sure that strftime() checks the bounds of the various parts
-        # of the time tuple.
-
-        # check year
-        raises(ValueError, rctime.strftime, '', (1899, 1, 1, 0, 0, 0, 0, 1, -1))
-        if rctime.accept2dyear:
-            raises(ValueError, rctime.strftime, '', (-1, 1, 1, 0, 0, 0, 0, 1, -1))
-            raises(ValueError, rctime.strftime, '', (100, 1, 1, 0, 0, 0, 0, 1, -1))
-        # check month
-        raises(ValueError, rctime.strftime, '', (1900, 13, 1, 0, 0, 0, 0, 1, -1))
-        # check day of month
-        raises(ValueError, rctime.strftime, '', (1900, 1, 32, 0, 0, 0, 0, 1, -1))
-        # check hour
-        raises(ValueError, rctime.strftime, '', (1900, 1, 1, -1, 0, 0, 0, 1, -1))
-        raises(ValueError, rctime.strftime, '', (1900, 1, 1, 24, 0, 0, 0, 1, -1))
-        # check minute
-        raises(ValueError, rctime.strftime, '', (1900, 1, 1, 0, -1, 0, 0, 1, -1))
-        raises(ValueError, rctime.strftime, '', (1900, 1, 1, 0, 60, 0, 0, 1, -1))
-        # check second
-        raises(ValueError, rctime.strftime, '', (1900, 1, 1, 0, 0, -1, 0, 1, -1))
-        # C99 only requires allowing for one leap second, but Python's docs say
-        # allow two leap seconds (0..61)
-        raises(ValueError, rctime.strftime, '', (1900, 1, 1, 0, 0, 62, 0, 1, -1))
-        # no check for upper-bound day of week;
-        #  value forced into range by a "% 7" calculation.
-        # start check at -2 since gettmarg() increments value before taking
-        #  modulo.
-        raises(ValueError, rctime.strftime, '', (1900, 1, 1, 0, 0, 0, -2, 1, -1))
-        # check day of the year
-        raises(ValueError, rctime.strftime, '', (1900, 1, 1, 0, 0, 0, 0, 367, -1))
-        # check daylight savings flag
-        raises(ValueError, rctime.strftime, '', (1900, 1, 1, 0, 0, 0, 0, 1, -2))
-        raises(ValueError, rctime.strftime, '', (1900, 1, 1, 0, 0, 0, 0, 1, 2))
-
-    def test_strptime(self):
-        import time as rctime
-
-        t = rctime.time()
-        tt = rctime.gmtime(t)
-        assert isinstance(rctime.strptime("", ""), type(tt))
-
-        for directive in ('a', 'A', 'b', 'B', 'c', 'd', 'H', 'I',
-                          'j', 'm', 'M', 'p', 'S',
-                          'U', 'w', 'W', 'x', 'X', 'y', 'Y', 'Z', '%'):
-            format = ' %' + directive
-            print format
-            rctime.strptime(rctime.strftime(format, tt), format)
-
-    def test_pickle(self):
-        import pickle
-        import time as rctime
-        now = rctime.localtime()
-        new = pickle.loads(pickle.dumps(now))
-        assert new == now
-        assert type(new) is type(now)
diff --git a/pypy/module/select/test/test_epoll.py b/pypy/module/select/test/test_epoll.py
--- a/pypy/module/select/test/test_epoll.py
+++ b/pypy/module/select/test/test_epoll.py
@@ -7,7 +7,7 @@
 
 class AppTestEpoll(object):
     spaceconfig = {
-        "usemodules": ["select", "_socket", "posix", "rctime"],
+        "usemodules": ["select", "_socket", "posix", "time"],
     }
 
     def setup_class(cls):
diff --git a/pypy/module/select/test/test_kqueue.py b/pypy/module/select/test/test_kqueue.py
--- a/pypy/module/select/test/test_kqueue.py
+++ b/pypy/module/select/test/test_kqueue.py
@@ -4,7 +4,7 @@
 import sys

From noreply at buildbot.pypy.org  Wed Nov 19 09:56:21 2014
From: noreply at buildbot.pypy.org (Raemi)
Date: Wed, 19 Nov 2014 09:56:21 +0100 (CET)
Subject: [pypy-commit] pypy stmgc-c7: attach clickable info to plot elements
Message-ID: <20141119085621.8094F1D2487@cobra.cs.uni-duesseldorf.de>

Author: Remi Meier 
Branch: stmgc-c7
Changeset: r74590:5e380588d479
Date: 2014-11-19 09:56 +0100
http://bitbucket.org/pypy/pypy/changeset/5e380588d479/

Log:	attach clickable info to plot elements

diff --git a/pypy/stm/plot_stm_log.py b/pypy/stm/plot_stm_log.py
--- a/pypy/stm/plot_stm_log.py
+++ b/pypy/stm/plot_stm_log.py
@@ -68,6 +68,7 @@
         self.stop_time = 0
         self.aborted = False
         self.pauses = []
+        self.info = []
 
 
 def plot_log(logentries, ax):
@@ -106,6 +107,17 @@
                 tr.pauses[-1] = (tr.pauses[-1][0], entry.timestamp)
 
 
+        # attach logentry as transaction info
+        tr = curr_trs.get(th_num)
+        if tr is not None:
+            tr.info.append(str(entry))
+        if entry.event in (psl.STM_ABORTING_OTHER_CONTENTION,):
+            tr2 = curr_trs.get(entry.otherthreadnum)
+            if tr2 is not None:
+                tr2.info.append(str(entry))
+
+
+
     # plt.ion()
     for th_num, trs in finished_trs.items():
         # plt.draw()
@@ -118,7 +130,8 @@
         for tr in trs:
             add_transaction(boxes, hlines,
                             tr.start_time, None, tr.stop_time,
-                            tr.aborted, tr.pauses)
+                            tr.aborted, tr.pauses,
+                            "\n".join(tr.info))
         plot_boxes(boxes, th_num, ax)
         plot_hlines(hlines, th_num, ax)
         print "> Pauses:", len(hlines)
@@ -159,7 +172,7 @@
     axs[0].set_ylabel("Thread")
     axs[0].set_ylim(0, thread_count)
     axs[0].set_yticks([r+0.5 for r in range(thread_count)])
-    axs[0].set_yticklabels(range(1, thread_count+1))
+    axs[0].set_yticklabels(range(thread_count))
     #axs[0].set_xticks([])
     print "Drawn."
 
diff --git a/pypy/stm/print_stm_log.py b/pypy/stm/print_stm_log.py
--- a/pypy/stm/print_stm_log.py
+++ b/pypy/stm/print_stm_log.py
@@ -51,6 +51,16 @@
         self.marker1 = marker1
         self.marker2 = marker2
 
+    def __str__(self):
+        s = '[%.3f][%s->%s]\t%s' % (
+            self.timestamp, self.threadnum, self.otherthreadnum,
+            event_name[self.event])
+        if self.marker1:
+            s += ':\n%s' % print_marker(self.marker1)
+        if self.marker2:
+            s += '\n%s' % print_marker(self.marker2)
+        return s
+
 
 def parse_log(filename):
     f = open(filename, 'rb')
@@ -131,17 +141,28 @@
     def sortkey(self):
         return self.aborted_time + self.paused_time
 
+    def __str__(self):
+        s = '%.3fs lost in aborts, %.3fs paused (%dx %s)\n' % (
+            self.aborted_time, self.paused_time, self.num_events, event_name[self.event])
+        s += print_marker(self.marker1)
+        if self.marker2:
+            s += '\n%s' % print_marker(self.marker2)
+        return s
+
+
+
 
 r_marker = re.compile(r'File "(.+)", line (\d+)')
 
 def print_marker(marker):
-    print '  ' + marker
+    s = '  %s' % marker
     match = r_marker.match(marker)
     if match:
         line = linecache.getline(match.group(1), int(match.group(2)))
         line = line.strip()
         if line:
-            print '    ' + line
+            s += '\n    %s' % line
+    return s
 
 def percent(fraction, total):
     r = '%.1f' % (fraction * 100.0 / total)
@@ -210,11 +231,7 @@
             idx = int((t - start_time) / total_time * intervals)
             timeline[idx] += 1
 
-        print '%.3fs lost in aborts, %.3fs paused (%dx %s)' % (
-            c.aborted_time, c.paused_time, c.num_events, event_name[c.event])
-        print_marker(c.marker1)
-        if c.marker2:
-            print_marker(c.marker2)
+        print str(c)
         print "time line:", "".join(['x' if i else '.' for i in timeline])
         print
 

From noreply at buildbot.pypy.org  Wed Nov 19 10:31:36 2014
From: noreply at buildbot.pypy.org (fijal)
Date: Wed, 19 Nov 2014 10:31:36 +0100 (CET)
Subject: [pypy-commit] pypy optresult: fix the imports, but not the tests
Message-ID: <20141119093136.764741C355D@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: optresult
Changeset: r74591:eb2e25680b4f
Date: 2014-11-19 09:10 +0200
http://bitbucket.org/pypy/pypy/changeset/eb2e25680b4f/

Log:	fix the imports, but not the tests

diff --git a/rpython/jit/metainterp/test/test_executor.py b/rpython/jit/metainterp/test/test_executor.py
--- a/rpython/jit/metainterp/test/test_executor.py
+++ b/rpython/jit/metainterp/test/test_executor.py
@@ -1,14 +1,11 @@
 import py
 import sys, random
 from rpython.rlib.rarithmetic import r_uint, intmask
-from rpython.rtyper.lltypesystem import lltype, llmemory
 from rpython.jit.metainterp.executor import execute
-from rpython.jit.metainterp.executor import execute_varargs, execute_nonspec
+from rpython.jit.metainterp.executor import execute_varargs, _execute_nonspec
 from rpython.jit.metainterp.resoperation import rop, opname, opclasses
-from rpython.jit.metainterp.history import BoxInt, ConstInt
-from rpython.jit.metainterp.history import BoxPtr, ConstPtr
-from rpython.jit.metainterp.history import BoxFloat, ConstFloat
-from rpython.jit.metainterp.history import AbstractDescr, Box
+from rpython.jit.metainterp.history import ConstInt, ConstPtr, ConstFloat
+from rpython.jit.metainterp.history import AbstractDescr
 from rpython.jit.metainterp import history
 from rpython.jit.codewriter import longlong
 from rpython.jit.backend.model import AbstractCPU

From noreply at buildbot.pypy.org  Wed Nov 19 10:31:37 2014
From: noreply at buildbot.pypy.org (fijal)
Date: Wed, 19 Nov 2014 10:31:37 +0100 (CET)
Subject: [pypy-commit] pypy optresult: fix test_logger
Message-ID: <20141119093137.DCEF51C355D@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: optresult
Changeset: r74592:b318f6fe9576
Date: 2014-11-19 11:31 +0200
http://bitbucket.org/pypy/pypy/changeset/b318f6fe9576/

Log:	fix test_logger

diff --git a/rpython/jit/metainterp/logger.py b/rpython/jit/metainterp/logger.py
--- a/rpython/jit/metainterp/logger.py
+++ b/rpython/jit/metainterp/logger.py
@@ -114,20 +114,20 @@
                 if name:
                     return 'ConstClass(' + name + ')'
             return str(arg.value)
-        elif isinstance(arg, BoxInt):
-            return 'i' + str(mv)
         elif isinstance(arg, self.ts.ConstRef):
             if arg.value:
                 return 'ConstPtr(ptr' + str(mv) + ')'
             return 'ConstPtr(null)'
-        elif isinstance(arg, self.ts.BoxRef):
-            return 'p' + str(mv)
         elif isinstance(arg, ConstFloat):
-            return str(arg.getfloat())
-        elif isinstance(arg, BoxFloat):
-            return 'f' + str(mv)
+            return str(arg.getfloatstorage())
         elif arg is None:
             return 'None'
+        elif arg.type == 'i':
+            return 'i' + str(mv)
+        elif arg.type == 'r':
+            return 'p' + str(mv)
+        elif arg.type == 'f':
+            return 'f' + str(mv)
         else:
             return '?'
 
@@ -155,10 +155,10 @@
             s_offset = "+%d: " % offset
         args = ", ".join([self.repr_of_arg(op.getarg(i)) for i in range(op.numargs())])
 
-        #if op.result is not None:
-        #    res = self.repr_of_arg(op.result) + " = "
-        #else:
-        res = ""
+        if op.type != 'v':
+            res = self.repr_of_arg(op) + " = "
+        else:
+            res = ""
         is_guard = op.is_guard()
         if op.getdescr() is not None:
             descr = op.getdescr()
diff --git a/rpython/jit/metainterp/test/test_logger.py b/rpython/jit/metainterp/test/test_logger.py
--- a/rpython/jit/metainterp/test/test_logger.py
+++ b/rpython/jit/metainterp/test/test_logger.py
@@ -76,8 +76,11 @@
         output = logger.log_loop(loop, namespace)
         oloop = pure_parse(output, namespace=namespace)
         if check_equal:
-            equaloplists(loop.operations, oloop.operations)
-            assert oloop.inputargs == loop.inputargs
+            remap = {}
+            for box1, box2 in zip(loop.inputargs, oloop.inputargs):
+                assert box1.__class__ == box2.__class__
+                remap[box2] = box1
+            equaloplists(loop.operations, oloop.operations, remap=remap)
         return logger, loop, oloop
 
     def test_simple(self):
@@ -154,7 +157,11 @@
         f1 = float_add(3.5, f0)
         '''
         _, loop, oloop = self.reparse(inp)
-        equaloplists(loop.operations, oloop.operations)
+        remap = {}
+        for box1, box2 in zip(loop.inputargs, oloop.inputargs):
+            assert box1.__class__ == box2.__class__
+            remap[box2] = box1
+        equaloplists(loop.operations, oloop.operations, remap=remap)
 
     def test_jump(self):
         namespace = {'target': JitCellToken()}

From noreply at buildbot.pypy.org  Wed Nov 19 11:01:31 2014
From: noreply at buildbot.pypy.org (fijal)
Date: Wed, 19 Nov 2014 11:01:31 +0100 (CET)
Subject: [pypy-commit] pypy optresult: stop cloning stuff in compile.py,
 we need to investigate why it happens
Message-ID: <20141119100131.ED3CE1D2889@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: optresult
Changeset: r74593:efc7b11cbda1
Date: 2014-11-19 12:01 +0200
http://bitbucket.org/pypy/pypy/changeset/efc7b11cbda1/

Log:	stop cloning stuff in compile.py, we need to investigate why it
	happens

diff --git a/rpython/jit/metainterp/compile.py b/rpython/jit/metainterp/compile.py
--- a/rpython/jit/metainterp/compile.py
+++ b/rpython/jit/metainterp/compile.py
@@ -10,8 +10,8 @@
 from rpython.tool.sourcetools import func_with_new_name
 
 from rpython.jit.metainterp.resoperation import ResOperation, rop, get_deep_immutable_oplist
-from rpython.jit.metainterp.history import (TreeLoop, Box, JitCellToken,
-    TargetToken, AbstractFailDescr, BoxInt, BoxPtr, BoxFloat, ConstInt)
+from rpython.jit.metainterp.history import (TreeLoop, Const, JitCellToken,
+    TargetToken, AbstractFailDescr, ConstInt)
 from rpython.jit.metainterp import history, jitexc
 from rpython.jit.metainterp.optimize import InvalidLoop
 from rpython.jit.metainterp.inliner import Inliner
@@ -130,7 +130,7 @@
     h_ops = history.operations
     # XXX why do we clone here?
     part.operations = [ResOperation(rop.LABEL, inputargs, descr=TargetToken(jitcell_token))] + \
-                      [h_ops[i].clone() for i in range(start, len(h_ops))] + \
+                      [h_ops[i] for i in range(start, len(h_ops))] + \
                       [ResOperation(rop.LABEL, jumpargs, descr=jitcell_token)]
 
     try:
@@ -172,7 +172,7 @@
     if not loop.quasi_immutable_deps:
         loop.quasi_immutable_deps = None
     for box in loop.inputargs:
-        assert isinstance(box, Box)
+        assert not isinstance(box, Const)
 
     loop.original_jitcell_token = jitcell_token
     for label in all_target_tokens:
@@ -206,10 +206,10 @@
     h_ops = history.operations
 
     part.operations = [partial_trace.operations[-1]] + \
-                      [h_ops[i].clone() for i in range(start, len(h_ops))] + \
+                      [h_ops[i] for i in range(start, len(h_ops))] + \
                       [ResOperation(rop.JUMP, jumpargs, None, descr=loop_jitcell_token)]
     label = part.operations[0]
-    orignial_label = label.clone()
+    orignial_label = label
     assert label.getopnum() == rop.LABEL
     try:
         optimize_trace(metainterp_sd, part, jitdriver_sd.warmstate.enable_opts)
@@ -784,7 +784,7 @@
     new_trace.inputargs = metainterp.history.inputargs[:]
     # clone ops, as optimize_bridge can mutate the ops
 
-    new_trace.operations = [op.clone() for op in metainterp.history.operations]
+    new_trace.operations = [op for op in metainterp.history.operations]
     metainterp_sd = metainterp.staticdata
     state = metainterp.jitdriver_sd.warmstate
     if isinstance(resumekey, ResumeAtPositionDescr):
diff --git a/rpython/jit/metainterp/history.py b/rpython/jit/metainterp/history.py
--- a/rpython/jit/metainterp/history.py
+++ b/rpython/jit/metainterp/history.py
@@ -412,6 +412,7 @@
     _attrs_ = ('value',)
 
     def __init__(self, value=0):
+        xxx
         if not we_are_translated():
             if is_valid_int(value):
                 value = int(value)    # bool -> int
@@ -451,6 +452,7 @@
     _attrs_ = ('value',)
 
     def __init__(self, valuestorage=longlong.ZEROF):
+        xxxx
         assert lltype.typeOf(valuestorage) is longlong.FLOATSTORAGE
         self.value = valuestorage
 
@@ -483,6 +485,7 @@
     _attrs_ = ('value',)
 
     def __init__(self, value=lltype.nullptr(llmemory.GCREF.TO)):
+        xxx
         assert lltype.typeOf(value) == llmemory.GCREF
         self.value = value
 
@@ -516,7 +519,6 @@
 
     _getrepr_ = repr_pointer
 
-NULLBOX = BoxPtr()
 
 # ____________________________________________________________
 
@@ -690,7 +692,8 @@
     @staticmethod
     def check_consistency_of(inputargs, operations):
         for box in inputargs:
-            assert isinstance(box, Box), "Loop.inputargs contains %r" % (box,)
+            assert (not isinstance(box, Const),
+                   "Loop.inputargs contains %r" % (box,))
         seen = dict.fromkeys(inputargs)
         assert len(seen) == len(inputargs), (
                "duplicate Box in the Loop.inputargs")
@@ -699,11 +702,10 @@
     @staticmethod
     def check_consistency_of_branch(operations, seen):
         "NOT_RPYTHON"
-        return # XXX for now
         for op in operations:
             for i in range(op.numargs()):
                 box = op.getarg(i)
-                if isinstance(box, Box):
+                if not isinstance(box, Const):
                     assert box in seen
             if op.is_guard():
                 assert op.getdescr() is not None
@@ -712,19 +714,16 @@
                     TreeLoop.check_consistency_of_branch(ops, seen.copy())
                 for box in op.getfailargs() or []:
                     if box is not None:
-                        assert isinstance(box, Box)
+                        assert not isinstance(box, Const)
                         assert box in seen
             else:
                 assert op.getfailargs() is None
-            box = op.result
-            if box is not None:
-                assert isinstance(box, Box)
-                assert box not in seen
-                seen[box] = True
+            if op.type != 'v':
+                seen[op] = True
             if op.getopnum() == rop.LABEL:
                 inputargs = op.getarglist()
                 for box in inputargs:
-                    assert isinstance(box, Box), "LABEL contains %r" % (box,)
+                    assert not isinstance(box, Const), "LABEL contains %r" % (box,)
                 seen = dict.fromkeys(inputargs)
                 assert len(seen) == len(inputargs), (
                     "duplicate Box in the LABEL arguments")
diff --git a/rpython/jit/metainterp/warmstate.py b/rpython/jit/metainterp/warmstate.py
--- a/rpython/jit/metainterp/warmstate.py
+++ b/rpython/jit/metainterp/warmstate.py
@@ -2,7 +2,7 @@
 import weakref
 
 from rpython.jit.codewriter import support, heaptracker, longlong
-from rpython.jit.metainterp import history
+from rpython.jit.metainterp import resoperation, history
 from rpython.rlib.debug import debug_start, debug_stop, debug_print
 from rpython.rlib.jit import PARAMETERS
 from rpython.rlib.nonconst import NonConstant
@@ -73,7 +73,7 @@
             if in_const_box:
                 return history.ConstPtr(value)
             else:
-                return history.BoxPtr(value)
+                return resoperation.InputArgRef(value)
         else:
             adr = llmemory.cast_ptr_to_adr(value)
             value = heaptracker.adr2int(adr)
@@ -87,7 +87,7 @@
         if in_const_box:
             return history.ConstFloat(value)
         else:
-            return history.BoxFloat(value)
+            return resoperation.InputArgFloat(value)
     elif isinstance(value, str) or isinstance(value, unicode):
         assert len(value) == 1     # must be a character
         value = ord(value)
@@ -98,7 +98,7 @@
     if in_const_box:
         return history.ConstInt(value)
     else:
-        return history.BoxInt(value)
+        return resoperation.InputArgInt(value)
 
 @specialize.arg(0)
 def equal_whatever(TYPE, x, y):

From noreply at buildbot.pypy.org  Wed Nov 19 11:16:03 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Wed, 19 Nov 2014 11:16:03 +0100 (CET)
Subject: [pypy-commit] pypy stmgc-c7: fix
Message-ID: <20141119101603.566831D2899@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: stmgc-c7
Changeset: r74594:66bca6f88d3e
Date: 2014-11-19 11:15 +0100
http://bitbucket.org/pypy/pypy/changeset/66bca6f88d3e/

Log:	fix

diff --git a/rpython/rtyper/lltypesystem/rffi.py b/rpython/rtyper/lltypesystem/rffi.py
--- a/rpython/rtyper/lltypesystem/rffi.py
+++ b/rpython/rtyper/lltypesystem/rffi.py
@@ -836,6 +836,7 @@
         # following code is executed if:
         # - rgc.can_move(data) and rgc.pin(data) both returned true
         # - rgc.can_move(data) returned false
+        assert not rgc.stm_is_enabled()
         data_start = cast_ptr_to_adr(lldata) + \
             offsetof(STRTYPE, 'chars') + itemoffsetof(STRTYPE.chars, 0)
 

From noreply at buildbot.pypy.org  Wed Nov 19 11:47:58 2014
From: noreply at buildbot.pypy.org (Wouter van Heyst)
Date: Wed, 19 Nov 2014 11:47:58 +0100 (CET)
Subject: [pypy-commit] pypy issue1922-take2: match different virtualenv
 versions on top of arigo's patch
Message-ID: <20141119104758.701971C34F4@cobra.cs.uni-duesseldorf.de>

Author: Wouter van Heyst
Branch: issue1922-take2
Changeset: r74595:f769234d3938
Date: 2014-11-14 15:25 +0200
http://bitbucket.org/pypy/pypy/changeset/f769234d3938/

Log:	match different virtualenv versions on top of arigo's patch

diff --git a/lib-python/2.7/subprocess.py b/lib-python/2.7/subprocess.py
--- a/lib-python/2.7/subprocess.py
+++ b/lib-python/2.7/subprocess.py
@@ -655,6 +655,22 @@
         """Create new Popen instance."""
         _cleanup()
 
+        # --- PyPy hack, see _pypy_install_libs_after_virtualenv() ---
+        # match arguments passed by different versions of virtualenv
+        if args[1:] in (
+            ['-c', 'import sys; print sys.prefix'],          # 9c47597b
+            ['-c', 'import sys; print(sys.prefix)'],        # 10ba3f3c
+            ['-c', "\nimport sys\nprefix = sys.prefix\n"    # 0e9342ce
+             "if sys.version_info[0] == 3:\n"
+             "    prefix = prefix.encode('utf8')\n"
+             "if hasattr(sys.stdout, 'detach'):\n"
+             "    sys.stdout = sys.stdout.detach()\n"
+             "elif hasattr(sys.stdout, 'buffer'):\n"
+             "    sys.stdout = sys.stdout.buffer\nsys.stdout.write(prefix)\n"],
+            ['-c', 'import sys;out=sys.stdout;getattr(out, "buffer"'
+             ', out).write(sys.prefix.encode("utf-8"))']):  # a9454bce
+            _pypy_install_libs_after_virtualenv(args[0])
+
         if not isinstance(bufsize, (int, long)):
             raise TypeError("bufsize must be an integer")
 
@@ -1560,6 +1576,27 @@
             self.send_signal(signal.SIGKILL)
 
 
+def _pypy_install_libs_after_virtualenv(target_executable):
+    # https://bitbucket.org/pypy/pypy/issue/1922/future-proofing-virtualenv
+    #
+    # PyPy 2.4.1 turned --shared on by default.  This means the pypy binary
+    # depends on the 'libpypy-c.so' shared library to be able to run.
+    # The virtualenv code existing at the time did not account for this
+    # and would break.  Try to detect that we're running under such a
+    # virtualenv in the "Testing executable with" phase and copy the
+    # library ourselves.
+    caller = sys._getframe(2)
+    if ('virtualenv_version' in caller.f_globals and
+                  'copyfile' in caller.f_globals):
+        dest_dir = sys.pypy_resolvedirof(target_executable)
+        src_dir = sys.pypy_resolvedirof(sys.executable)
+        for libname in ['libpypy-c.so']:
+            dest_library = os.path.join(dest_dir, libname)
+            src_library = os.path.join(src_dir, libname)
+            if os.path.exists(src_library):
+                caller.f_globals['copyfile'](src_library, dest_library)
+
+
 def _demo_posix():
     #
     # Example 1: Simple redirection: Get process list

From noreply at buildbot.pypy.org  Wed Nov 19 11:47:59 2014
From: noreply at buildbot.pypy.org (Wouter van Heyst)
Date: Wed, 19 Nov 2014 11:47:59 +0100 (CET)
Subject: [pypy-commit] pypy issue1922-take2: add versions
Message-ID: <20141119104759.D4FAC1C34F4@cobra.cs.uni-duesseldorf.de>

Author: Wouter van Heyst
Branch: issue1922-take2
Changeset: r74596:eba70e4af6dd
Date: 2014-11-14 16:51 +0200
http://bitbucket.org/pypy/pypy/changeset/eba70e4af6dd/

Log:	add versions

diff --git a/lib-python/2.7/subprocess.py b/lib-python/2.7/subprocess.py
--- a/lib-python/2.7/subprocess.py
+++ b/lib-python/2.7/subprocess.py
@@ -658,9 +658,8 @@
         # --- PyPy hack, see _pypy_install_libs_after_virtualenv() ---
         # match arguments passed by different versions of virtualenv
         if args[1:] in (
-            ['-c', 'import sys; print sys.prefix'],          # 9c47597b
-            ['-c', 'import sys; print(sys.prefix)'],        # 10ba3f3c
-            ['-c', "\nimport sys\nprefix = sys.prefix\n"    # 0e9342ce
+            ['-c', 'import sys; print(sys.prefix)'],        # 1.6 10ba3f3c
+            ['-c', "\nimport sys\nprefix = sys.prefix\n"    # 1.7 0e9342ce
              "if sys.version_info[0] == 3:\n"
              "    prefix = prefix.encode('utf8')\n"
              "if hasattr(sys.stdout, 'detach'):\n"
@@ -668,7 +667,7 @@
              "elif hasattr(sys.stdout, 'buffer'):\n"
              "    sys.stdout = sys.stdout.buffer\nsys.stdout.write(prefix)\n"],
             ['-c', 'import sys;out=sys.stdout;getattr(out, "buffer"'
-             ', out).write(sys.prefix.encode("utf-8"))']):  # a9454bce
+             ', out).write(sys.prefix.encode("utf-8"))']):  # 1.7.2 a9454bce
             _pypy_install_libs_after_virtualenv(args[0])
 
         if not isinstance(bufsize, (int, long)):

From noreply at buildbot.pypy.org  Wed Nov 19 11:48:01 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Wed, 19 Nov 2014 11:48:01 +0100 (CET)
Subject: [pypy-commit] pypy default: Merged
 larstiq/pypy-virtualenv/issue1922-take2 into default
Message-ID: <20141119104801.1E17A1C34F4@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r74597:b50c82f2d282
Date: 2014-11-19 11:47 +0100
http://bitbucket.org/pypy/pypy/changeset/b50c82f2d282/

Log:	Merged larstiq/pypy-virtualenv/issue1922-take2 into default

	Patch by larstiq and me: add an obscure hack to make existing
	virtualenv versions work correctly with the new PyPy 2.4.1

diff --git a/lib-python/2.7/subprocess.py b/lib-python/2.7/subprocess.py
--- a/lib-python/2.7/subprocess.py
+++ b/lib-python/2.7/subprocess.py
@@ -655,6 +655,21 @@
         """Create new Popen instance."""
         _cleanup()
 
+        # --- PyPy hack, see _pypy_install_libs_after_virtualenv() ---
+        # match arguments passed by different versions of virtualenv
+        if args[1:] in (
+            ['-c', 'import sys; print(sys.prefix)'],        # 1.6 10ba3f3c
+            ['-c', "\nimport sys\nprefix = sys.prefix\n"    # 1.7 0e9342ce
+             "if sys.version_info[0] == 3:\n"
+             "    prefix = prefix.encode('utf8')\n"
+             "if hasattr(sys.stdout, 'detach'):\n"
+             "    sys.stdout = sys.stdout.detach()\n"
+             "elif hasattr(sys.stdout, 'buffer'):\n"
+             "    sys.stdout = sys.stdout.buffer\nsys.stdout.write(prefix)\n"],
+            ['-c', 'import sys;out=sys.stdout;getattr(out, "buffer"'
+             ', out).write(sys.prefix.encode("utf-8"))']):  # 1.7.2 a9454bce
+            _pypy_install_libs_after_virtualenv(args[0])
+
         if not isinstance(bufsize, (int, long)):
             raise TypeError("bufsize must be an integer")
 
@@ -1560,6 +1575,27 @@
             self.send_signal(signal.SIGKILL)
 
 
+def _pypy_install_libs_after_virtualenv(target_executable):
+    # https://bitbucket.org/pypy/pypy/issue/1922/future-proofing-virtualenv
+    #
+    # PyPy 2.4.1 turned --shared on by default.  This means the pypy binary
+    # depends on the 'libpypy-c.so' shared library to be able to run.
+    # The virtualenv code existing at the time did not account for this
+    # and would break.  Try to detect that we're running under such a
+    # virtualenv in the "Testing executable with" phase and copy the
+    # library ourselves.
+    caller = sys._getframe(2)
+    if ('virtualenv_version' in caller.f_globals and
+                  'copyfile' in caller.f_globals):
+        dest_dir = sys.pypy_resolvedirof(target_executable)
+        src_dir = sys.pypy_resolvedirof(sys.executable)
+        for libname in ['libpypy-c.so']:
+            dest_library = os.path.join(dest_dir, libname)
+            src_library = os.path.join(src_dir, libname)
+            if os.path.exists(src_library):
+                caller.f_globals['copyfile'](src_library, dest_library)
+
+
 def _demo_posix():
     #
     # Example 1: Simple redirection: Get process list

From noreply at buildbot.pypy.org  Wed Nov 19 12:18:37 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Wed, 19 Nov 2014 12:18:37 +0100 (CET)
Subject: [pypy-commit] pypy default: Remove the calls to pin/unpin
 statically if our GC doesn't support them.
Message-ID: <20141119111837.7608E1C06A7@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r74598:f2f96a913975
Date: 2014-11-19 12:18 +0100
http://bitbucket.org/pypy/pypy/changeset/f2f96a913975/

Log:	Remove the calls to pin/unpin statically if our GC doesn't support
	them. (This is mainly for STM being unhappy about the
	cast_ptr_to_adr.)

diff --git a/rpython/memory/gctransform/framework.py b/rpython/memory/gctransform/framework.py
--- a/rpython/memory/gctransform/framework.py
+++ b/rpython/memory/gctransform/framework.py
@@ -461,17 +461,18 @@
                                             annmodel.SomeInteger(nonneg=True)],
                                            annmodel.s_None)
 
-        self.pin_ptr = getfn(GCClass.pin,
-                             [s_gc, SomeAddress()],
-                             annmodel.SomeBool())
+        if GCClass.can_usually_pin_objects:
+            self.pin_ptr = getfn(GCClass.pin,
+                                 [s_gc, SomeAddress()],
+                                 annmodel.SomeBool())
 
-        self.unpin_ptr = getfn(GCClass.unpin,
-                               [s_gc, SomeAddress()],
-                               annmodel.s_None)
+            self.unpin_ptr = getfn(GCClass.unpin,
+                                   [s_gc, SomeAddress()],
+                                   annmodel.s_None)
 
-        self._is_pinned_ptr = getfn(GCClass._is_pinned,
-                                    [s_gc, SomeAddress()],
-                                    annmodel.SomeBool())
+            self._is_pinned_ptr = getfn(GCClass._is_pinned,
+                                        [s_gc, SomeAddress()],
+                                        annmodel.SomeBool())
 
         self.write_barrier_ptr = None
         self.write_barrier_from_array_ptr = None
@@ -1042,6 +1043,10 @@
                                   v_size])
 
     def gct_gc_pin(self, hop):
+        if not hasattr(self, 'pin_ptr'):
+            c_false = rmodel.inputconst(lltype.Bool, False)
+            hop.genop("same_as", [c_false], resultvar=hop.spaceop.result)
+            return
         op = hop.spaceop
         v_addr = hop.genop('cast_ptr_to_adr', [op.args[0]],
             resulttype=llmemory.Address)
@@ -1049,6 +1054,8 @@
                   resultvar=op.result)
 
     def gct_gc_unpin(self, hop):
+        if not hasattr(self, 'unpin_ptr'):
+            return
         op = hop.spaceop
         v_addr = hop.genop('cast_ptr_to_adr', [op.args[0]],
             resulttype=llmemory.Address)
@@ -1056,6 +1063,10 @@
                   resultvar=op.result)
 
     def gct_gc__is_pinned(self, hop):
+        if not hasattr(self, '_is_pinned_ptr'):
+            c_false = rmodel.inputconst(lltype.Bool, False)
+            hop.genop("same_as", [c_false], resultvar=hop.spaceop.result)
+            return
         op = hop.spaceop
         v_addr = hop.genop('cast_ptr_to_adr', [op.args[0]],
             resulttype=llmemory.Address)

From noreply at buildbot.pypy.org  Wed Nov 19 12:19:33 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Wed, 19 Nov 2014 12:19:33 +0100 (CET)
Subject: [pypy-commit] pypy stmgc-c7: Small fixes
Message-ID: <20141119111933.1BBB81C06A7@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: stmgc-c7
Changeset: r74599:9ddb804903be
Date: 2014-11-19 12:19 +0100
http://bitbucket.org/pypy/pypy/changeset/9ddb804903be/

Log:	Small fixes

diff --git a/rpython/memory/test/test_transformed_gc.py b/rpython/memory/test/test_transformed_gc.py
--- a/rpython/memory/test/test_transformed_gc.py
+++ b/rpython/memory/test/test_transformed_gc.py
@@ -97,7 +97,7 @@
             return res
 
         from rpython.translator.c.genc import CStandaloneBuilder
-        from rpython.annotation.listdef import s_list_of_strings
+        from rpython.annotator.listdef import s_list_of_strings
 
         s_args = s_list_of_strings
         t = rtype(entrypoint, [s_args], gcname=cls.gcname,
diff --git a/rpython/rtyper/lltypesystem/rffi.py b/rpython/rtyper/lltypesystem/rffi.py
--- a/rpython/rtyper/lltypesystem/rffi.py
+++ b/rpython/rtyper/lltypesystem/rffi.py
@@ -836,7 +836,6 @@
         # following code is executed if:
         # - rgc.can_move(data) and rgc.pin(data) both returned true
         # - rgc.can_move(data) returned false
-        assert not rgc.stm_is_enabled()
         data_start = cast_ptr_to_adr(lldata) + \
             offsetof(STRTYPE, 'chars') + itemoffsetof(STRTYPE.chars, 0)
 
diff --git a/rpython/translator/stm/inevitable.py b/rpython/translator/stm/inevitable.py
--- a/rpython/translator/stm/inevitable.py
+++ b/rpython/translator/stm/inevitable.py
@@ -17,6 +17,7 @@
     'jit_record_known_class',
     'gc_identityhash', 'gc_id', 'gc_can_move', 'gc__collect',
     'gc_adr_of_root_stack_top', 'gc_add_memory_pressure',
+    'gc_pin', 'gc_unpin', 'gc__is_pinned',
     'weakref_create', 'weakref_deref',
     'jit_assembler_call', 'gc_writebarrier',
     'shrink_array',

From noreply at buildbot.pypy.org  Wed Nov 19 12:19:34 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Wed, 19 Nov 2014 12:19:34 +0100 (CET)
Subject: [pypy-commit] pypy stmgc-c7: hg merge default
Message-ID: <20141119111934.999971C06A7@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: stmgc-c7
Changeset: r74600:68acf29aeeae
Date: 2014-11-19 12:19 +0100
http://bitbucket.org/pypy/pypy/changeset/68acf29aeeae/

Log:	hg merge default

diff --git a/lib-python/2.7/subprocess.py b/lib-python/2.7/subprocess.py
--- a/lib-python/2.7/subprocess.py
+++ b/lib-python/2.7/subprocess.py
@@ -655,6 +655,21 @@
         """Create new Popen instance."""
         _cleanup()
 
+        # --- PyPy hack, see _pypy_install_libs_after_virtualenv() ---
+        # match arguments passed by different versions of virtualenv
+        if args[1:] in (
+            ['-c', 'import sys; print(sys.prefix)'],        # 1.6 10ba3f3c
+            ['-c', "\nimport sys\nprefix = sys.prefix\n"    # 1.7 0e9342ce
+             "if sys.version_info[0] == 3:\n"
+             "    prefix = prefix.encode('utf8')\n"
+             "if hasattr(sys.stdout, 'detach'):\n"
+             "    sys.stdout = sys.stdout.detach()\n"
+             "elif hasattr(sys.stdout, 'buffer'):\n"
+             "    sys.stdout = sys.stdout.buffer\nsys.stdout.write(prefix)\n"],
+            ['-c', 'import sys;out=sys.stdout;getattr(out, "buffer"'
+             ', out).write(sys.prefix.encode("utf-8"))']):  # 1.7.2 a9454bce
+            _pypy_install_libs_after_virtualenv(args[0])
+
         if not isinstance(bufsize, (int, long)):
             raise TypeError("bufsize must be an integer")
 
@@ -1560,6 +1575,27 @@
             self.send_signal(signal.SIGKILL)
 
 
+def _pypy_install_libs_after_virtualenv(target_executable):
+    # https://bitbucket.org/pypy/pypy/issue/1922/future-proofing-virtualenv
+    #
+    # PyPy 2.4.1 turned --shared on by default.  This means the pypy binary
+    # depends on the 'libpypy-c.so' shared library to be able to run.
+    # The virtualenv code existing at the time did not account for this
+    # and would break.  Try to detect that we're running under such a
+    # virtualenv in the "Testing executable with" phase and copy the
+    # library ourselves.
+    caller = sys._getframe(2)
+    if ('virtualenv_version' in caller.f_globals and
+                  'copyfile' in caller.f_globals):
+        dest_dir = sys.pypy_resolvedirof(target_executable)
+        src_dir = sys.pypy_resolvedirof(sys.executable)
+        for libname in ['libpypy-c.so']:
+            dest_library = os.path.join(dest_dir, libname)
+            src_library = os.path.join(src_dir, libname)
+            if os.path.exists(src_library):
+                caller.f_globals['copyfile'](src_library, dest_library)
+
+
 def _demo_posix():
     #
     # Example 1: Simple redirection: Get process list
diff --git a/rpython/flowspace/flowcontext.py b/rpython/flowspace/flowcontext.py
--- a/rpython/flowspace/flowcontext.py
+++ b/rpython/flowspace/flowcontext.py
@@ -856,14 +856,9 @@
     def WITH_CLEANUP(self, oparg):
         # Note: RPython context managers receive None in lieu of tracebacks
         # and cannot suppress the exception.
-        # This opcode changed a lot between CPython versions
-        if sys.version_info >= (2, 6):
-            unroller = self.popvalue()
-            w_exitfunc = self.popvalue()
-            self.pushvalue(unroller)
-        else:
-            w_exitfunc = self.popvalue()
-            unroller = self.peekvalue(0)
+        unroller = self.popvalue()
+        w_exitfunc = self.popvalue()
+        self.pushvalue(unroller)
 
         if isinstance(unroller, Raise):
             w_exc = unroller.w_exc
diff --git a/rpython/memory/gctransform/framework.py b/rpython/memory/gctransform/framework.py
--- a/rpython/memory/gctransform/framework.py
+++ b/rpython/memory/gctransform/framework.py
@@ -466,17 +466,18 @@
                                             annmodel.SomeInteger(nonneg=True)],
                                            annmodel.s_None)
 
-        self.pin_ptr = getfn(GCClass.pin,
-                             [s_gc, SomeAddress()],
-                             annmodel.SomeBool())
+        if GCClass.can_usually_pin_objects:
+            self.pin_ptr = getfn(GCClass.pin,
+                                 [s_gc, SomeAddress()],
+                                 annmodel.SomeBool())
 
-        self.unpin_ptr = getfn(GCClass.unpin,
-                               [s_gc, SomeAddress()],
-                               annmodel.s_None)
+            self.unpin_ptr = getfn(GCClass.unpin,
+                                   [s_gc, SomeAddress()],
+                                   annmodel.s_None)
 
-        self._is_pinned_ptr = getfn(GCClass._is_pinned,
-                                    [s_gc, SomeAddress()],
-                                    annmodel.SomeBool())
+            self._is_pinned_ptr = getfn(GCClass._is_pinned,
+                                        [s_gc, SomeAddress()],
+                                        annmodel.SomeBool())
 
         self.write_barrier_ptr = None
         self.write_barrier_from_array_ptr = None
@@ -1089,6 +1090,10 @@
                                   v_size])
 
     def gct_gc_pin(self, hop):
+        if not hasattr(self, 'pin_ptr'):
+            c_false = rmodel.inputconst(lltype.Bool, False)
+            hop.genop("same_as", [c_false], resultvar=hop.spaceop.result)
+            return
         op = hop.spaceop
         v_addr = hop.genop('cast_ptr_to_adr', [op.args[0]],
             resulttype=llmemory.Address)
@@ -1096,6 +1101,8 @@
                   resultvar=op.result)
 
     def gct_gc_unpin(self, hop):
+        if not hasattr(self, 'unpin_ptr'):
+            return
         op = hop.spaceop
         v_addr = hop.genop('cast_ptr_to_adr', [op.args[0]],
             resulttype=llmemory.Address)
@@ -1103,6 +1110,10 @@
                   resultvar=op.result)
 
     def gct_gc__is_pinned(self, hop):
+        if not hasattr(self, '_is_pinned_ptr'):
+            c_false = rmodel.inputconst(lltype.Bool, False)
+            hop.genop("same_as", [c_false], resultvar=hop.spaceop.result)
+            return
         op = hop.spaceop
         v_addr = hop.genop('cast_ptr_to_adr', [op.args[0]],
             resulttype=llmemory.Address)

From noreply at buildbot.pypy.org  Wed Nov 19 13:16:01 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Wed, 19 Nov 2014 13:16:01 +0100 (CET)
Subject: [pypy-commit] pypy stmgc-c7: next one
Message-ID: <20141119121601.770581D2304@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: stmgc-c7
Changeset: r74601:9a74415a5e88
Date: 2014-11-19 13:15 +0100
http://bitbucket.org/pypy/pypy/changeset/9a74415a5e88/

Log:	next one

diff --git a/rpython/rtyper/lltypesystem/rffi.py b/rpython/rtyper/lltypesystem/rffi.py
--- a/rpython/rtyper/lltypesystem/rffi.py
+++ b/rpython/rtyper/lltypesystem/rffi.py
@@ -877,7 +877,7 @@
         """
         new_buf = mallocfn(count)
         pinned = 0
-        if rgc.can_move(new_buf):
+        if rgc.stm_is_enabled() or rgc.can_move(new_buf):
             if rgc.pin(new_buf):
                 pinned = 1
             else:

From noreply at buildbot.pypy.org  Wed Nov 19 20:36:09 2014
From: noreply at buildbot.pypy.org (pjenvey)
Date: Wed, 19 Nov 2014 20:36:09 +0100 (CET)
Subject: [pypy-commit] pypy py3k: reapply our time module deltas from rctime
Message-ID: <20141119193609.6D1C81D387C@cobra.cs.uni-duesseldorf.de>

Author: Philip Jenvey 
Branch: py3k
Changeset: r74602:ae21f00a0b84
Date: 2014-11-19 11:35 -0800
http://bitbucket.org/pypy/pypy/changeset/ae21f00a0b84/

Log:	reapply our time module deltas from rctime

diff --git a/pypy/module/time/interp_time.py b/pypy/module/time/interp_time.py
--- a/pypy/module/time/interp_time.py
+++ b/pypy/module/time/interp_time.py
@@ -168,10 +168,8 @@
 TM_P = lltype.Ptr(tm)
 c_clock = external('clock', [rffi.TIME_TP], clock_t)
 c_time = external('time', [rffi.TIME_TP], rffi.TIME_T)
-c_ctime = external('ctime', [rffi.TIME_TP], rffi.CCHARP)
 c_gmtime = external('gmtime', [rffi.TIME_TP], TM_P)
 c_mktime = external('mktime', [TM_P], rffi.TIME_T)
-c_asctime = external('asctime', [TM_P], rffi.CCHARP)
 c_localtime = external('localtime', [rffi.TIME_TP], TM_P)
 if _POSIX:
     c_tzset = external('tzset', [], lltype.Void)
@@ -185,12 +183,12 @@
                              "RPY_EXTERN "
                              "char** pypy_get_tzname();\n"
                              "RPY_EXTERN "
-                             "void pypy__tzset();"],
+                             "void* pypy__tzset();"],
         separate_module_sources = ["""
         long pypy_get_timezone() { return timezone; }
         int pypy_get_daylight() { return daylight; }
         char** pypy_get_tzname() { return tzname; }
-        void pypy__tzset() { _tzset(); }
+        void pypy__tzset() { return _tzset(); }
         """])
     # Ensure sure that we use _tzset() and timezone from the same C Runtime.
     c_tzset = external('pypy__tzset', [], lltype.Void, win_eci)
@@ -412,24 +410,24 @@
                     "argument must be sequence of length 9, not %d",
                     len(tup_w))
 
-    y = space.int_w(tup_w[0])
-    tm_mon = space.int_w(tup_w[1])
+    y = space.c_int_w(tup_w[0])
+    tm_mon = space.c_int_w(tup_w[1])
     if tm_mon == 0:
         tm_mon = 1
-    tm_mday = space.int_w(tup_w[2])
+    tm_mday = space.c_int_w(tup_w[2])
     if tm_mday == 0:
         tm_mday = 1
-    tm_yday = space.int_w(tup_w[7])
+    tm_yday = space.c_int_w(tup_w[7])
     if tm_yday == 0:
         tm_yday = 1
     rffi.setintfield(glob_buf, 'c_tm_mon', tm_mon)
     rffi.setintfield(glob_buf, 'c_tm_mday', tm_mday)
-    rffi.setintfield(glob_buf, 'c_tm_hour', space.int_w(tup_w[3]))
-    rffi.setintfield(glob_buf, 'c_tm_min', space.int_w(tup_w[4]))
-    rffi.setintfield(glob_buf, 'c_tm_sec', space.int_w(tup_w[5]))
-    rffi.setintfield(glob_buf, 'c_tm_wday', space.int_w(tup_w[6]))
+    rffi.setintfield(glob_buf, 'c_tm_hour', space.c_int_w(tup_w[3]))
+    rffi.setintfield(glob_buf, 'c_tm_min', space.c_int_w(tup_w[4]))
+    rffi.setintfield(glob_buf, 'c_tm_sec', space.c_int_w(tup_w[5]))
+    rffi.setintfield(glob_buf, 'c_tm_wday', space.c_int_w(tup_w[6]))
     rffi.setintfield(glob_buf, 'c_tm_yday', tm_yday)
-    rffi.setintfield(glob_buf, 'c_tm_isdst', space.int_w(tup_w[8]))
+    rffi.setintfield(glob_buf, 'c_tm_isdst', space.c_int_w(tup_w[8]))
     if _POSIX:
         if _CYGWIN:
             pass
@@ -438,24 +436,23 @@
             glob_buf.c_tm_zone = lltype.nullptr(rffi.CCHARP.TO)
             rffi.setintfield(glob_buf, 'c_tm_gmtoff', 0)
 
-    if y < 1900:
+    if y < 1000:
         w_accept2dyear = _get_module_object(space, "accept2dyear")
-        accept2dyear = space.int_w(w_accept2dyear)
+        accept2dyear = space.is_true(w_accept2dyear)
 
-        if not accept2dyear:
-            raise OperationError(space.w_ValueError,
-                space.wrap("year >= 1900 required"))
-
-        if 69 <= y <= 99:
-            y += 1900
-        elif 0 <= y <= 68:
-            y += 2000
-        else:
-            raise OperationError(space.w_ValueError,
-                space.wrap("year out of range"))
+        if accept2dyear:
+            if 69 <= y <= 99:
+                y += 1900
+            elif 0 <= y <= 68:
+                y += 2000
+            else:
+                raise OperationError(space.w_ValueError,
+                                     space.wrap("year out of range"))
+            space.warn(space.wrap("Century info guessed for a 2-digit year."),
+                       space.w_DeprecationWarning)
 
     # tm_wday does not need checking of its upper-bound since taking "%
-    #  7" in gettmarg() automatically restricts the range.
+    #  7" in _gettmarg() automatically restricts the range.
     if rffi.getintfield(glob_buf, 'c_tm_wday') < -1:
         raise OperationError(space.w_ValueError,
                              space.wrap("day of week out of range"))
@@ -470,6 +467,32 @@
 
     return glob_buf
 
+def _checktm(space, t_ref):
+    """Checks added to make sure strftime() and asctime() do not crash
+    Python by indexing blindly into some array for a textual
+    representation by some bad index (fixes bug #897625).  No check for
+    year or wday since handled in _gettmarg()."""
+    if not 0 <= rffi.getintfield(t_ref, 'c_tm_mon') <= 11:
+        raise OperationError(space.w_ValueError,
+                             space.wrap("month out of range"))
+    if not 1 <= rffi.getintfield(t_ref, 'c_tm_mday') <= 31:
+        raise OperationError(space.w_ValueError,
+                             space.wrap("day of month out of range"))
+    if not 0 <= rffi.getintfield(t_ref, 'c_tm_hour') <= 23:
+        raise OperationError(space.w_ValueError,
+                             space.wrap("hour out of range"))
+    if not 0 <= rffi.getintfield(t_ref, 'c_tm_min') <= 59:
+        raise OperationError(space.w_ValueError,
+                             space.wrap("minute out of range"))
+    if not 0 <= rffi.getintfield(t_ref, 'c_tm_sec') <= 61:
+        raise OperationError(space.w_ValueError,
+                             space.wrap("seconds out of range"))
+    # tm_wday does not need checking: "% 7" in _gettmarg() automatically
+    # restricts the range
+    if not 0 <= rffi.getintfield(t_ref, 'c_tm_yday') <= 365:
+        raise OperationError(space.w_ValueError,
+                             space.wrap("day of year out of range"))
+
 def time(space):
     """time() -> floating point number
 
@@ -503,16 +526,13 @@
     not present, current time as returned by localtime() is used."""
 
     seconds = _get_inttime(space, w_seconds)
-
-    t_ref = lltype.malloc(rffi.TIME_TP.TO, 1, flavor='raw')
-    t_ref[0] = seconds
-    p = c_ctime(t_ref)
-    lltype.free(t_ref, flavor='raw')
+    with lltype.scoped_alloc(rffi.TIME_TP.TO, 1) as t_ref:
+        t_ref[0] = seconds
+        p = c_localtime(t_ref)
     if not p:
         raise OperationError(space.w_ValueError,
-            space.wrap("unconvertible time"))
-
-    return space.wrap(rffi.charp2str(p)[:-1]) # get rid of new line
+                             space.wrap("unconvertible time"))
+    return _asctime(space, p)
 
 # by now w_tup is an optional argument (and not *args)
 # because of the ext. compiler bugs in handling such arguments (*args, **kwds)
@@ -523,12 +543,26 @@
     When the time tuple is not present, current time as returned by localtime()
     is used."""
     buf_value = _gettmarg(space, w_tup)
-    p = c_asctime(buf_value)
-    if not p:
-        raise OperationError(space.w_ValueError,
-            space.wrap("unconvertible time"))
+    _checktm(space, buf_value)
+    return _asctime(space, buf_value)
 
-    return space.wrap(rffi.charp2str(p)[:-1]) # get rid of new line
+_wday_names = ["Sun", "Mon", "Tue", "Wed", "Thu", "Fri", "Sat"]
+_mon_names = ["Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sep",
+              "Oct", "Nov", "Dec"]
+
+def _asctime(space, t_ref):
+    # Inspired by Open Group reference implementation available at
+    # http://pubs.opengroup.org/onlinepubs/009695399/functions/asctime.html
+    w, getif = space.wrap, rffi.getintfield
+    args = [w(_wday_names[getif(t_ref, 'c_tm_wday')]),
+            w(_mon_names[getif(t_ref, 'c_tm_mon')]),
+            w(getif(t_ref, 'c_tm_mday')),
+            w(getif(t_ref, 'c_tm_hour')),
+            w(getif(t_ref, 'c_tm_min')),
+            w(getif(t_ref, 'c_tm_sec')),
+            w(getif(t_ref, 'c_tm_year') + 1900)]
+    return space.mod(w("%.3s %.3s%3d %.2d:%.2d:%.2d %d"),
+                     space.newtuple(args))
 
 def gmtime(space, w_seconds=None):
     """gmtime([seconds]) -> (tm_year, tm_mon, tm_day, tm_hour, tm_min,
@@ -610,32 +644,15 @@
     See the library reference manual for formatting codes. When the time tuple
     is not present, current time as returned by localtime() is used."""
     buf_value = _gettmarg(space, w_tup)
+    _checktm(space, buf_value)
 
-    # Checks added to make sure strftime() does not crash Python by
-    # indexing blindly into some array for a textual representation
-    # by some bad index (fixes bug #897625).
-    # No check for year since handled in gettmarg().
-    if rffi.getintfield(buf_value, 'c_tm_mon') < 0 or rffi.getintfield(buf_value, 'c_tm_mon') > 11:
-        raise OperationError(space.w_ValueError,
-                             space.wrap("month out of range"))
-    if rffi.getintfield(buf_value, 'c_tm_mday') < 1 or rffi.getintfield(buf_value, 'c_tm_mday') > 31:
-        raise OperationError(space.w_ValueError,
-                             space.wrap("day of month out of range"))
-    if rffi.getintfield(buf_value, 'c_tm_hour') < 0 or rffi.getintfield(buf_value, 'c_tm_hour') > 23:
-        raise OperationError(space.w_ValueError,
-                             space.wrap("hour out of range"))
-    if rffi.getintfield(buf_value, 'c_tm_min') < 0 or rffi.getintfield(buf_value, 'c_tm_min') > 59:
-        raise OperationError(space.w_ValueError,
-                             space.wrap("minute out of range"))
-    if rffi.getintfield(buf_value, 'c_tm_sec') < 0 or rffi.getintfield(buf_value, 'c_tm_sec') > 61:
-        raise OperationError(space.w_ValueError,
-                             space.wrap("seconds out of range"))
-    if rffi.getintfield(buf_value, 'c_tm_yday') < 0 or rffi.getintfield(buf_value, 'c_tm_yday') > 365:
-        raise OperationError(space.w_ValueError,
-                             space.wrap("day of year out of range"))
-    if rffi.getintfield(buf_value, 'c_tm_isdst') < -1 or rffi.getintfield(buf_value, 'c_tm_isdst') > 1:
-        raise OperationError(space.w_ValueError,
-                             space.wrap("daylight savings flag out of range"))
+    # Normalize tm_isdst just in case someone foolishly implements %Z
+    # based on the assumption that tm_isdst falls within the range of
+    # [-1, 1]
+    if rffi.getintfield(buf_value, 'c_tm_isdst') < -1:
+        rffi.setintfield(buf_value, 'c_tm_isdst', -1)
+    elif rffi.getintfield(buf_value, 'c_tm_isdst') > 1:
+        rffi.setintfield(buf_value, 'c_tm_isdst', 1)
 
     if _WIN:
         # check that the format string contains only valid directives
diff --git a/pypy/module/time/test/test_time.py b/pypy/module/time/test/test_time.py
--- a/pypy/module/time/test/test_time.py
+++ b/pypy/module/time/test/test_time.py
@@ -44,6 +44,15 @@
         time.ctime(time.time())
         raises(ValueError, time.ctime, 1E200)
         raises(OverflowError, time.ctime, 10**900)
+        for year in [-100, 100, 1000, 2000, 10000]:
+            try:
+                testval = time.mktime((year, 1, 10) + (0,)*6)
+            except (ValueError, OverflowError):
+                # If mktime fails, ctime will fail too.  This may happen
+                # on some platforms.
+                pass
+            else:
+                assert time.ctime(testval)[20:] == str(year)
 
     def test_gmtime(self):
         import time
@@ -94,14 +103,14 @@
         ltime = time.localtime()
         time.accept2dyear == 0
         ltime = list(ltime)
-        ltime[0] = 1899
+        ltime[0] = -1
         raises(ValueError, time.mktime, tuple(ltime))
         time.accept2dyear == 1
 
         ltime = list(ltime)
         ltime[0] = 67
         ltime = tuple(ltime)
-        if os.name != "nt" and sys.maxint < 1<<32:   # time_t may be 64bit
+        if os.name != "nt" and sys.maxsize < 1<<32:   # time_t may be 64bit
             raises(OverflowError, time.mktime, ltime)
 
         ltime = list(ltime)
@@ -109,8 +118,8 @@
         raises(ValueError, time.mktime, tuple(ltime))
 
         t = time.time()
-        assert long(time.mktime(time.localtime(t))) == long(t)
-        assert long(time.mktime(time.gmtime(t))) - time.timezone == long(t)
+        assert int(time.mktime(time.localtime(t))) == int(t)
+        assert int(time.mktime(time.gmtime(t))) - time.timezone == int(t)
         ltime = time.localtime()
         assert time.mktime(tuple(ltime)) == time.mktime(ltime)
         if os.name != 'nt':
@@ -132,6 +141,7 @@
         raises(TypeError, time.asctime, (1, 2))
         raises(TypeError, time.asctime, (1, 2, 3, 4, 5, 6, 'f', 8, 9))
         raises(TypeError, time.asctime, "foo")
+        raises(ValueError, time.asctime, (1900, -1, 1, 0, 0, 0, 0, 1, -1))
         res = time.asctime()
         assert isinstance(res, str)
         time.asctime(time.localtime())
@@ -146,6 +156,18 @@
         except ValueError:
             pass  # some OS (ie POSIXes besides Linux) reject year > 9999
 
+    def test_asctime_large_year(self):
+        import time
+        assert time.asctime((12345,) +
+                              (0,) * 8) == 'Mon Jan  1 00:00:00 12345'
+        assert time.asctime((123456789,) +
+                              (0,) * 8) == 'Mon Jan  1 00:00:00 123456789'
+        sizeof_int = 4
+        bigyear = (1 << 8 * sizeof_int - 1) - 1
+        asc = time.asctime((bigyear, 6, 1) + (0,)*6)
+        assert asc[-len(str(bigyear)):] == str(bigyear)
+        raises(OverflowError, time.asctime, (bigyear + 1,) + (0,)*8)
+
     def test_accept2dyear_access(self):
         import time
 
@@ -157,6 +179,17 @@
         finally:
             time.accept2dyear = accept2dyear
 
+    def test_accept2dyear_bad(self):
+        import time
+        class X:
+            def __bool__(self):
+                raise RuntimeError('boo')
+        orig, time.accept2dyear = time.accept2dyear, X()
+        try:
+            raises(RuntimeError, time.asctime, (200,)  + (0,) * 8)
+        finally:
+            time.accept2dyear = orig
+
     def test_struct_time(self):
         import time
         raises(TypeError, time.struct_time)
@@ -228,7 +261,7 @@
             # rely on it.
             if org_TZ is not None:
                 os.environ['TZ'] = org_TZ
-            elif os.environ.has_key('TZ'):
+            elif 'TZ' in os.environ:
                 del os.environ['TZ']
             time.tzset()
 
@@ -280,10 +313,12 @@
         # of the time tuple.
 
         # check year
-        raises(ValueError, time.strftime, '', (1899, 1, 1, 0, 0, 0, 0, 1, -1))
         if time.accept2dyear:
             raises(ValueError, time.strftime, '', (-1, 1, 1, 0, 0, 0, 0, 1, -1))
             raises(ValueError, time.strftime, '', (100, 1, 1, 0, 0, 0, 0, 1, -1))
+        time.strftime('', (1899, 1, 1, 0, 0, 0, 0, 1, -1))
+        time.strftime('', (0, 1, 1, 0, 0, 0, 0, 1, -1))
+
         # check month
         raises(ValueError, time.strftime, '', (1900, 13, 1, 0, 0, 0, 0, 1, -1))
         # check day of month
@@ -307,8 +342,8 @@
         # check day of the year
         raises(ValueError, time.strftime, '', (1900, 1, 1, 0, 0, 0, 0, 367, -1))
         # check daylight savings flag
-        raises(ValueError, time.strftime, '', (1900, 1, 1, 0, 0, 0, 0, 1, -2))
-        raises(ValueError, time.strftime, '', (1900, 1, 1, 0, 0, 0, 0, 1, 2))
+        time.strftime('', (1900, 1, 1, 0, 0, 0, 0, 1, -2))
+        time.strftime('', (1900, 1, 1, 0, 0, 0, 0, 1, 2))
 
     def test_strptime(self):
         import time
@@ -321,7 +356,7 @@
                           'j', 'm', 'M', 'p', 'S',
                           'U', 'w', 'W', 'x', 'X', 'y', 'Y', 'Z', '%'):
             format = ' %' + directive
-            print format
+            print(format)
             time.strptime(time.strftime(format, tt), format)
 
     def test_pickle(self):

From noreply at buildbot.pypy.org  Wed Nov 19 20:52:54 2014
From: noreply at buildbot.pypy.org (rlamy)
Date: Wed, 19 Nov 2014 20:52:54 +0100 (CET)
Subject: [pypy-commit] pypy default: Use consistent terminology for offset
	vs instruction:
Message-ID: <20141119195254.53A1A1C34F4@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy 
Branch: 
Changeset: r74603:0fe478d043c1
Date: 2014-05-04 21:00 +0100
http://bitbucket.org/pypy/pypy/changeset/0fe478d043c1/

Log:	Use consistent terminology for offset vs instruction:

	offset = index into the code bytestring instruction = an opcode
	together with its argument

diff --git a/rpython/flowspace/bytecode.py b/rpython/flowspace/bytecode.py
--- a/rpython/flowspace/bytecode.py
+++ b/rpython/flowspace/bytecode.py
@@ -81,37 +81,37 @@
         and **varkwarg, if they exist."""
         return self.signature.scope_length()
 
-    def read(self, pos):
+    def read(self, offset):
         """
-        Decode the instruction starting at position ``next_instr``.
+        Decode the instruction starting at position ``offset``.
 
-        Returns (next_instr, opname, oparg).
+        Returns (next_offset, opname, oparg).
         """
         co_code = self.co_code
-        opnum = ord(co_code[pos])
-        next_instr = pos + 1
+        opnum = ord(co_code[offset])
+        next_offset = offset + 1
 
         if opnum >= HAVE_ARGUMENT:
-            lo = ord(co_code[next_instr])
-            hi = ord(co_code[next_instr+1])
-            next_instr += 2
+            lo = ord(co_code[next_offset])
+            hi = ord(co_code[next_offset + 1])
+            next_offset += 2
             oparg = (hi * 256) | lo
         else:
             oparg = 0
 
         while opnum == EXTENDED_ARG:
-            opnum = ord(co_code[next_instr])
+            opnum = ord(co_code[next_offset])
             if opnum < HAVE_ARGUMENT:
                 raise BytecodeCorruption
-            lo = ord(co_code[next_instr+1])
-            hi = ord(co_code[next_instr+2])
-            next_instr += 3
+            lo = ord(co_code[next_offset + 1])
+            hi = ord(co_code[next_offset + 2])
+            next_offset += 3
             oparg = (oparg * 65536) | (hi * 256) | lo
 
         if opnum in opcode.hasjrel:
-            oparg += next_instr
+            oparg += next_offset
         opname = self.opnames[opnum]
-        return next_instr, opname, oparg
+        return next_offset, opname, oparg
 
     @property
     def is_generator(self):
diff --git a/rpython/flowspace/flowcontext.py b/rpython/flowspace/flowcontext.py
--- a/rpython/flowspace/flowcontext.py
+++ b/rpython/flowspace/flowcontext.py
@@ -29,7 +29,7 @@
         msg = ["\n"]
         msg += map(str, self.args)
         msg += [""]
-        msg += source_lines(self.ctx.graph, None, offset=self.ctx.last_instr)
+        msg += source_lines(self.ctx.graph, None, offset=self.ctx.last_offset)
         return "\n".join(msg)
 
 
@@ -288,7 +288,7 @@
 
         self.init_closure(func.func_closure)
         self.f_lineno = code.co_firstlineno
-        self.last_instr = 0
+        self.last_offset = 0
 
         self.init_locals_stack(code)
 
@@ -359,7 +359,7 @@
         self.locals_stack_w[:len(items_w)] = items_w
         self.dropvaluesuntil(len(items_w))
 
-    def getstate(self, next_pos):
+    def getstate(self, next_offset):
         # getfastscope() can return real None, for undefined locals
         data = self.save_locals_stack()
         if self.last_exception is None:
@@ -369,7 +369,7 @@
             data.append(self.last_exception.w_type)
             data.append(self.last_exception.w_value)
         recursively_flatten(data)
-        return FrameState(data, self.blockstack[:], next_pos)
+        return FrameState(data, self.blockstack[:], next_offset)
 
     def setstate(self, state):
         """ Reset the context to the given frame state. """
@@ -393,7 +393,7 @@
         if getattr(recorder, 'final_state', None) is not None:
             self.mergeblock(recorder.crnt_block, recorder.final_state)
             raise StopFlowing
-        spaceop.offset = self.last_instr
+        spaceop.offset = self.last_offset
         recorder.append(spaceop)
 
     def do_op(self, op):
@@ -424,12 +424,12 @@
 
     def record_block(self, block):
         self.setstate(block.framestate)
-        next_pos = block.framestate.next_instr
+        next_offset = block.framestate.next_offset
         self.recorder = block.make_recorder()
         try:
             while True:
-                next_pos = self.handle_bytecode(next_pos)
-                self.recorder.final_state = self.getstate(next_pos)
+                next_offset = self.handle_bytecode(next_offset)
+                self.recorder.final_state = self.getstate(next_offset)
 
         except RaiseImplicit as e:
             w_exc = e.w_exc
@@ -467,10 +467,10 @@
         self.recorder = None
 
     def mergeblock(self, currentblock, currentstate):
-        next_instr = currentstate.next_instr
+        next_offset = currentstate.next_offset
         # can 'currentstate' be merged with one of the blocks that
         # already exist for this bytecode position?
-        candidates = self.joinpoints.setdefault(next_instr, [])
+        candidates = self.joinpoints.setdefault(next_offset, [])
         for block in candidates:
             newstate = block.framestate.union(currentstate)
             if newstate is not None:
@@ -526,12 +526,12 @@
                     stack_items_w[i] = w_new
                     break
 
-    def handle_bytecode(self, next_instr):
-        self.last_instr = next_instr
-        next_instr, methodname, oparg = self.pycode.read(next_instr)
+    def handle_bytecode(self, next_offset):
+        self.last_offset = next_offset
+        next_offset, methodname, oparg = self.pycode.read(next_offset)
         try:
-            res = getattr(self, methodname)(oparg)
-            return res if res is not None else next_instr
+            offset = getattr(self, methodname)(oparg)
+            return offset if offset is not None else next_offset
         except FlowSignal as signal:
             return self.unroll(signal)
 
diff --git a/rpython/flowspace/framestate.py b/rpython/flowspace/framestate.py
--- a/rpython/flowspace/framestate.py
+++ b/rpython/flowspace/framestate.py
@@ -3,10 +3,10 @@
 
 
 class FrameState(object):
-    def __init__(self, mergeable, blocklist, next_instr):
+    def __init__(self, mergeable, blocklist, next_offset):
         self.mergeable = mergeable
         self.blocklist = blocklist
-        self.next_instr = next_instr
+        self.next_offset = next_offset
 
     def copy(self):
         "Make a copy of this state in which all Variables are fresh."
@@ -15,7 +15,7 @@
             if isinstance(w, Variable):
                 w = Variable(w)
             newstate.append(w)
-        return FrameState(newstate, self.blocklist, self.next_instr)
+        return FrameState(newstate, self.blocklist, self.next_offset)
 
     def getvariables(self):
         return [w for w in self.mergeable if isinstance(w, Variable)]
@@ -26,7 +26,7 @@
         # safety check, don't try to compare states with different
         # nonmergeable states
         assert self.blocklist == other.blocklist
-        assert self.next_instr == other.next_instr
+        assert self.next_offset == other.next_offset
         for w1, w2 in zip(self.mergeable, other.mergeable):
             if not (w1 == w2 or (isinstance(w1, Variable) and
                                  isinstance(w2, Variable))):
@@ -44,7 +44,7 @@
                 newstate.append(union(w1, w2))
         except UnionError:
             return None
-        return FrameState(newstate, self.blocklist, self.next_instr)
+        return FrameState(newstate, self.blocklist, self.next_offset)
 
     def getoutputargs(self, targetstate):
         "Return the output arguments needed to link self to targetstate."

From noreply at buildbot.pypy.org  Wed Nov 19 21:18:02 2014
From: noreply at buildbot.pypy.org (mattip)
Date: Wed, 19 Nov 2014 21:18:02 +0100 (CET)
Subject: [pypy-commit] pypy ufuncapi: pass cpyext ndarrayobject ufunc test
Message-ID: <20141119201802.BDE441C34F4@cobra.cs.uni-duesseldorf.de>

Author: mattip 
Branch: ufuncapi
Changeset: r74604:75e3444eb0a1
Date: 2014-11-19 21:35 +0200
http://bitbucket.org/pypy/pypy/changeset/75e3444eb0a1/

Log:	pass cpyext ndarrayobject ufunc test

diff --git a/pypy/module/micronumpy/ufuncs.py b/pypy/module/micronumpy/ufuncs.py
--- a/pypy/module/micronumpy/ufuncs.py
+++ b/pypy/module/micronumpy/ufuncs.py
@@ -1289,7 +1289,7 @@
                          space.wrap("cannot mix ndarray and %r (arg %d) in call to ufunc" % (
                                     arg_i, i)))
                 raw_storage_setitem(dataps, CCHARP_SIZE * i,
-                        rffi.cast(rffi.CCHARP, arg_i.implementation.storage))
+                        rffi.cast(rffi.CCHARP, arg_i.implementation.get_storage_as_int(space)))
                 #This assumes we iterate over the whole array (it should be a view...)
                 raw_storage_setitem(self.dims, LONG_SIZE * i, rffi.cast(rffi.LONG, arg_i.get_size()))
                 raw_storage_setitem(self.steps, LONG_SIZE * i, rffi.cast(rffi.LONG, arg_i.get_dtype().elsize))

From noreply at buildbot.pypy.org  Thu Nov 20 01:34:21 2014
From: noreply at buildbot.pypy.org (rlamy)
Date: Thu, 20 Nov 2014 01:34:21 +0100 (CET)
Subject: [pypy-commit] pypy framestate: Split locals from stack in
	FlowContext - WIP
Message-ID: <20141120003421.219391D283E@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy 
Branch: framestate
Changeset: r74605:82688ed76170
Date: 2014-11-20 00:33 +0000
http://bitbucket.org/pypy/pypy/changeset/82688ed76170/

Log:	Split locals from stack in FlowContext - WIP

diff --git a/rpython/flowspace/flowcontext.py b/rpython/flowspace/flowcontext.py
--- a/rpython/flowspace/flowcontext.py
+++ b/rpython/flowspace/flowcontext.py
@@ -278,6 +278,25 @@
     "cmp_exc_match",
     ]
 
+class sliceview(object):
+    def __init__(self, lst, start, stop):
+        self.lst = lst
+        self.start = start
+        self.stop = stop
+
+    def __getitem__(self, n):
+        assert 0 <= self.start + n < self.stop
+        return self.lst[self.start + n]
+
+    def __setitem__(self, n, value):
+        assert 0 <= self.start + n < self.stop
+        self.lst[self.start + n] = value
+
+    def __delitem__(self, n):
+        assert 0 <= self.start + n < self.stop
+        del self.lst[self.start + n]
+
+
 class FlowContext(object):
     def __init__(self, graph, code):
         self.graph = graph
@@ -301,63 +320,71 @@
             self.closure = list(closure)
         assert len(self.closure) == len(self.pycode.co_freevars)
 
+    @property
+    def locals_w(self):
+        return sliceview(self.locals_stack_w, 0, self.nlocals)
+
+    @property
+    def stack(self):
+        return sliceview(self.locals_stack_w, self.nlocals, len(self.locals_stack_w))
+
+    @property
+    def stackdepth(self):
+        assert self.valuestackdepth >= self.nlocals
+        return self.valuestackdepth - self.nlocals
+
+    @stackdepth.setter
+    def stackdepth(self, n):
+        assert 0 <= n <= self.stacksize
+        self.valuestackdepth = self.nlocals + n
+
     def init_locals_stack(self, code):
         """
         Initialize the locals and the stack.
 
         The locals are ordered according to self.pycode.signature.
         """
-        self.valuestackdepth = code.co_nlocals
+        self.nlocals = self.valuestackdepth = code.co_nlocals
+        self.stacksize = code.co_stacksize
         self.locals_stack_w = [None] * (code.co_stacksize + code.co_nlocals)
 
     def pushvalue(self, w_object):
-        depth = self.valuestackdepth
-        self.locals_stack_w[depth] = w_object
-        self.valuestackdepth = depth + 1
+        depth = self.stackdepth
+        self.stackdepth = depth + 1
+        self.stack[depth] = w_object
 
     def popvalue(self):
-        depth = self.valuestackdepth - 1
-        assert depth >= self.pycode.co_nlocals, "pop from empty value stack"
-        w_object = self.locals_stack_w[depth]
-        self.locals_stack_w[depth] = None
-        self.valuestackdepth = depth
+        depth = self.stackdepth - 1
+        w_object = self.stack[depth]
+        self.stack[depth] = None
+        self.stackdepth = depth
         return w_object
 
     def peekvalue(self, index_from_top=0):
         # NOTE: top of the stack is peekvalue(0).
-        index = self.valuestackdepth + ~index_from_top
-        assert index >= self.pycode.co_nlocals, (
-            "peek past the bottom of the stack")
-        return self.locals_stack_w[index]
+        index = self.stackdepth + ~index_from_top
+        return self.stack[index]
 
     def settopvalue(self, w_object, index_from_top=0):
-        index = self.valuestackdepth + ~index_from_top
-        assert index >= self.pycode.co_nlocals, (
-            "settop past the bottom of the stack")
-        self.locals_stack_w[index] = w_object
+        index = self.stackdepth + ~index_from_top
+        self.stack[index] = w_object
 
     def popvalues(self, n):
         values_w = [self.popvalue() for i in range(n)]
         values_w.reverse()
         return values_w
 
-    def dropvalues(self, n):
-        finaldepth = self.valuestackdepth - n
-        for n in range(finaldepth, self.valuestackdepth):
-            self.locals_stack_w[n] = None
-        self.valuestackdepth = finaldepth
-
     def dropvaluesuntil(self, finaldepth):
-        for n in range(finaldepth, self.valuestackdepth):
-            self.locals_stack_w[n] = None
-        self.valuestackdepth = finaldepth
+        for n in range(finaldepth, self.stackdepth):
+            self.stack[n] = None
+        self.stackdepth = finaldepth
 
     def save_locals_stack(self):
         return self.locals_stack_w[:self.valuestackdepth]
 
     def restore_locals_stack(self, items_w):
         self.locals_stack_w[:len(items_w)] = items_w
-        self.dropvaluesuntil(len(items_w))
+        self.dropvaluesuntil(len(items_w) - self.nlocals)
 
     def getstate(self, next_offset):
         # getfastscope() can return real None, for undefined locals
@@ -516,8 +543,8 @@
     # hack for unrolling iterables, don't use this
     def replace_in_stack(self, oldvalue, newvalue):
         w_new = Constant(newvalue)
-        stack_items_w = self.locals_stack_w
-        for i in range(self.valuestackdepth - 1, self.pycode.co_nlocals - 1, -1):
+        stack_items_w = self.stack
+        for i in range(self.stackdepth - 1, - 1, -1):
             w_v = stack_items_w[i]
             if isinstance(w_v, Constant):
                 if w_v.value is oldvalue:
@@ -870,7 +897,7 @@
             op.simple_call(w_exitfunc, w_None, w_None, w_None).eval(self)
 
     def LOAD_FAST(self, varindex):
-        w_value = self.locals_stack_w[varindex]
+        w_value = self.locals_w[varindex]
         if w_value is None:
             raise FlowingError("Local variable referenced before assignment")
         self.pushvalue(w_value)
@@ -915,7 +942,7 @@
     def STORE_FAST(self, varindex):
         w_newvalue = self.popvalue()
         assert w_newvalue is not None
-        self.locals_stack_w[varindex] = w_newvalue
+        self.locals_w[varindex] = w_newvalue
         if isinstance(w_newvalue, Variable):
             w_newvalue.rename(self.getlocalvarname(varindex))
 
@@ -1128,11 +1155,11 @@
         op.simple_call(w_append_meth, w_value).eval(self)
 
     def DELETE_FAST(self, varindex):
-        if self.locals_stack_w[varindex] is None:
+        if self.locals_w[varindex] is None:
             varname = self.getlocalvarname(varindex)
             message = "local variable '%s' referenced before assignment"
             raise UnboundLocalError(message, varname)
-        self.locals_stack_w[varindex] = None
+        self.locals_w[varindex] = None
 
     def STORE_MAP(self, oparg):
         w_key = self.popvalue()
@@ -1295,21 +1322,21 @@
 
     def __init__(self, ctx, handlerposition):
         self.handlerposition = handlerposition
-        self.valuestackdepth = ctx.valuestackdepth
+        self.stackdepth = ctx.stackdepth
 
     def __eq__(self, other):
         return (self.__class__ is other.__class__ and
                 self.handlerposition == other.handlerposition and
-                self.valuestackdepth == other.valuestackdepth)
+                self.stackdepth == other.stackdepth)
 
     def __ne__(self, other):
         return not (self == other)
 
     def __hash__(self):
-        return hash((self.handlerposition, self.valuestackdepth))
+        return hash((self.handlerposition, self.stackdepth))
 
     def cleanupstack(self, ctx):
-        ctx.dropvaluesuntil(self.valuestackdepth)
+        ctx.dropvaluesuntil(self.stackdepth)
 
     def handle(self, ctx, unroller):
         raise NotImplementedError

From noreply at buildbot.pypy.org  Thu Nov 20 03:59:59 2014
From: noreply at buildbot.pypy.org (rlamy)
Date: Thu, 20 Nov 2014 03:59:59 +0100 (CET)
Subject: [pypy-commit] pypy framestate: Actually split locals from stack in
 FlowContext + cleanup
Message-ID: <20141120025959.5EF8B1D283E@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy 
Branch: framestate
Changeset: r74606:723281c212b3
Date: 2014-11-20 02:59 +0000
http://bitbucket.org/pypy/pypy/changeset/723281c212b3/

Log:	Actually split locals from stack in FlowContext + cleanup

diff --git a/rpython/flowspace/flowcontext.py b/rpython/flowspace/flowcontext.py
--- a/rpython/flowspace/flowcontext.py
+++ b/rpython/flowspace/flowcontext.py
@@ -278,24 +278,6 @@
     "cmp_exc_match",
     ]
 
-class sliceview(object):
-    def __init__(self, lst, start, stop):
-        self.lst = lst
-        self.start = start
-        self.stop = stop
-
-    def __getitem__(self, n):
-        assert 0 <= self.start + n < self.stop
-        return self.lst[self.start + n]
-
-    def __setitem__(self, n, value):
-        assert 0 <= self.start + n < self.stop
-        self.lst[self.start + n] = value
-
-    def __delitem__(self, n):
-        assert 0 <= self.start + n < self.stop
-        del self.lst[self.start + n]
-
 
 class FlowContext(object):
     def __init__(self, graph, code):
@@ -320,33 +302,16 @@
             self.closure = list(closure)
         assert len(self.closure) == len(self.pycode.co_freevars)
 
-    @property
-    def locals_w(self):
-        return sliceview(self.locals_stack_w, 0, self.nlocals)
-
-    @property
-    def stack(self):
-        return sliceview(self.locals_stack_w, self.nlocals, len(self.locals_stack_w))
-
-    @property
-    def stackdepth(self):
-        assert self.valuestackdepth >= self.nlocals
-        return self.valuestackdepth - self.nlocals
-
-    @stackdepth.setter
-    def stackdepth(self, n):
-        assert 0 <= n <= self.stacksize
-        self.valuestackdepth = self.nlocals + n
-
     def init_locals_stack(self, code):
         """
         Initialize the locals and the stack.
 
         The locals are ordered according to self.pycode.signature.
         """
-        self.nlocals = self.valuestackdepth = code.co_nlocals
-        self.stacksize = code.co_stacksize
-        self.locals_stack_w = [None] * (code.co_stacksize + code.co_nlocals)
+        self.nlocals = code.co_nlocals
+        self.stackdepth = 0
+        self.locals_w = [None] * code.co_nlocals
+        self.stack = [None] * code.co_stacksize
 
     def pushvalue(self, w_object):
         depth = self.stackdepth
@@ -380,10 +345,11 @@
         self.stackdepth = finaldepth
 
     def save_locals_stack(self):
-        return self.locals_stack_w[:self.valuestackdepth]
+        return self.locals_w + self.stack[:self.stackdepth]
 
     def restore_locals_stack(self, items_w):
-        self.locals_stack_w[:len(items_w)] = items_w
+        self.locals_w = items_w[:self.nlocals]
+        self.stack[:len(items_w) - self.nlocals] = items_w[self.nlocals:]
         self.dropvaluesuntil(len(items_w) - self.nlocals)
 
     def getstate(self, next_offset):
diff --git a/rpython/flowspace/test/test_framestate.py b/rpython/flowspace/test/test_framestate.py
--- a/rpython/flowspace/test/test_framestate.py
+++ b/rpython/flowspace/test/test_framestate.py
@@ -15,7 +15,7 @@
         ctx = FlowContext(graph, code)
         # hack the frame
         ctx.setstate(graph.startblock.framestate)
-        ctx.locals_stack_w[ctx.pycode.co_nlocals-1] = Constant(None)
+        ctx.locals_w[-1] = Constant(None)
         return ctx
 
     def func_simple(x):
@@ -31,7 +31,7 @@
     def test_neq_hacked_framestate(self):
         ctx = self.get_context(self.func_simple)
         fs1 = ctx.getstate(0)
-        ctx.locals_stack_w[ctx.pycode.co_nlocals-1] = Variable()
+        ctx.locals_w[-1] = Variable()
         fs2 = ctx.getstate(0)
         assert not fs1.matches(fs2)
 
@@ -44,7 +44,7 @@
     def test_union_on_hacked_framestates(self):
         ctx = self.get_context(self.func_simple)
         fs1 = ctx.getstate(0)
-        ctx.locals_stack_w[ctx.pycode.co_nlocals-1] = Variable()
+        ctx.locals_w[-1] = Variable()
         fs2 = ctx.getstate(0)
         assert fs1.union(fs2).matches(fs2)  # fs2 is more general
         assert fs2.union(fs1).matches(fs2)  # fs2 is more general
@@ -52,7 +52,7 @@
     def test_restore_frame(self):
         ctx = self.get_context(self.func_simple)
         fs1 = ctx.getstate(0)
-        ctx.locals_stack_w[ctx.pycode.co_nlocals-1] = Variable()
+        ctx.locals_w[-1] = Variable()
         ctx.setstate(fs1)
         assert fs1.matches(ctx.getstate(0))
 
@@ -71,26 +71,25 @@
     def test_getoutputargs(self):
         ctx = self.get_context(self.func_simple)
         fs1 = ctx.getstate(0)
-        ctx.locals_stack_w[ctx.pycode.co_nlocals-1] = Variable()
+        ctx.locals_w[-1] = Variable()
         fs2 = ctx.getstate(0)
         outputargs = fs1.getoutputargs(fs2)
         # 'x' -> 'x' is a Variable
         # locals_w[n-1] -> locals_w[n-1] is Constant(None)
-        assert outputargs == [ctx.locals_stack_w[0], Constant(None)]
+        assert outputargs == [ctx.locals_w[0], Constant(None)]
 
     def test_union_different_constants(self):
         ctx = self.get_context(self.func_simple)
         fs1 = ctx.getstate(0)
-        ctx.locals_stack_w[ctx.pycode.co_nlocals-1] = Constant(42)
+        ctx.locals_w[-1] = Constant(42)
         fs2 = ctx.getstate(0)
         fs3 = fs1.union(fs2)
         ctx.setstate(fs3)
-        assert isinstance(ctx.locals_stack_w[ctx.pycode.co_nlocals-1],
-                          Variable)   # generalized
+        assert isinstance(ctx.locals_w[-1], Variable)   # generalized
 
     def test_union_spectag(self):
         ctx = self.get_context(self.func_simple)
         fs1 = ctx.getstate(0)
-        ctx.locals_stack_w[ctx.pycode.co_nlocals-1] = Constant(SpecTag())
+        ctx.locals_w[-1] = Constant(SpecTag())
         fs2 = ctx.getstate(0)
         assert fs1.union(fs2) is None   # UnionError

From noreply at buildbot.pypy.org  Thu Nov 20 10:46:38 2014
From: noreply at buildbot.pypy.org (techtonik)
Date: Thu, 20 Nov 2014 10:46:38 +0100 (CET)
Subject: [pypy-commit] pypy.org extradoc: Still add README for Bitbucket
Message-ID: <20141120094638.8D0B61C3140@cobra.cs.uni-duesseldorf.de>

Author: anatoly techtonik 
Branch: extradoc
Changeset: r553:bd0d982e013b
Date: 2014-11-19 12:24 +0300
http://bitbucket.org/pypy/pypy.org/changeset/bd0d982e013b/

Log:	Still add README for Bitbucket

diff --git a/README-DO-NOT-EDIT b/README
copy from README-DO-NOT-EDIT
copy to README
--- a/README-DO-NOT-EDIT
+++ b/README
@@ -1,2 +1,4 @@
+README-DO-NOT-EDIT
+
 These html files are not written by hand!
 See source/README.

From noreply at buildbot.pypy.org  Thu Nov 20 10:46:39 2014
From: noreply at buildbot.pypy.org (techtonik)
Date: Thu, 20 Nov 2014 10:46:39 +0100 (CET)
Subject: [pypy-commit] pypy.org extradoc: Beautify navigation - insert 
    
	after Contact
Message-ID: <20141120094639.BD9CE1C3140@cobra.cs.uni-duesseldorf.de>

Author: anatoly techtonik 
Branch: extradoc
Changeset: r554:64b115a0a5c8
Date: 2014-11-19 12:28 +0300
http://bitbucket.org/pypy/pypy.org/changeset/64b115a0a5c8/

Log:	Beautify navigation - insert 
     after Contact

diff --git a/archive.html b/archive.html
--- a/archive.html
+++ b/archive.html
@@ -30,15 +30,39 @@
 
 
    
 
 
    
diff --git a/compat.html b/compat.html
--- a/compat.html
+++ b/compat.html
@@ -30,15 +30,39 @@
 
 
    
 
 
    
diff --git a/contact.html b/contact.html
--- a/contact.html
+++ b/contact.html
@@ -30,15 +30,39 @@
 
 
    
 
 
    
diff --git a/download.html b/download.html
--- a/download.html
+++ b/download.html
@@ -30,15 +30,39 @@
 
 
    
 
 
    
diff --git a/features.html b/features.html
--- a/features.html
+++ b/features.html
@@ -30,15 +30,39 @@
 
 
    
 
 
    
diff --git a/howtohelp.html b/howtohelp.html
--- a/howtohelp.html
+++ b/howtohelp.html
@@ -30,15 +30,39 @@
 
 
    
 
 
    
diff --git a/index.html b/index.html
--- a/index.html
+++ b/index.html
@@ -30,15 +30,39 @@
 
 
    
 
 
    
diff --git a/numpydonate.html b/numpydonate.html
--- a/numpydonate.html
+++ b/numpydonate.html
@@ -30,15 +30,39 @@
 
 
    
 
 
    
diff --git a/people.html b/people.html
--- a/people.html
+++ b/people.html
@@ -30,15 +30,39 @@
 
 
    
 
 
    
diff --git a/performance.html b/performance.html
--- a/performance.html
+++ b/performance.html
@@ -30,15 +30,39 @@
 
 
    
 
 
    
diff --git a/py3donate.html b/py3donate.html
--- a/py3donate.html
+++ b/py3donate.html
@@ -30,15 +30,39 @@
 
 
    
 
 
    
diff --git a/source/_layouts/site.genshi b/source/_layouts/site.genshi
--- a/source/_layouts/site.genshi
+++ b/source/_layouts/site.genshi
@@ -63,15 +63,21 @@
 
    
 
 
 
diff --git a/sponsor.html b/sponsor.html
--- a/sponsor.html
+++ b/sponsor.html
@@ -30,15 +30,39 @@
 
 
    
 
 
    
diff --git a/success.html b/success.html
--- a/success.html
+++ b/success.html
@@ -30,15 +30,39 @@
 
 
    
 
 
    
diff --git a/tmdonate.html b/tmdonate.html
--- a/tmdonate.html
+++ b/tmdonate.html
@@ -30,15 +30,39 @@
 
 
    
 
 
    
diff --git a/tmdonate2.html b/tmdonate2.html
--- a/tmdonate2.html
+++ b/tmdonate2.html
@@ -30,15 +30,39 @@
 
 
    
 
 
    

From noreply at buildbot.pypy.org  Thu Nov 20 10:46:41 2014
From: noreply at buildbot.pypy.org (techtonik)
Date: Thu, 20 Nov 2014 10:46:41 +0100 (CET)
Subject: [pypy-commit] pypy.org extradoc: numpydonate: Link to numpy blog
 feed and to buildbot dashboard
Message-ID: <20141120094641.3949F1C3140@cobra.cs.uni-duesseldorf.de>

Author: anatoly techtonik 
Branch: extradoc
Changeset: r555:2b866097e40d
Date: 2014-11-19 15:32 +0300
http://bitbucket.org/pypy/pypy.org/changeset/2b866097e40d/

Log:	numpydonate: Link to numpy blog feed and to buildbot dashboard

diff --git a/numpydonate.html b/numpydonate.html
--- a/numpydonate.html
+++ b/numpydonate.html
@@ -79,8 +79,9 @@
 quickly as predicted.  The ratio “progress / $ used” so far
 corresponds roughly to what we expected.  The document below is the
 original call for proposal, and we still accept donations for this
-topic. Latest status reports can be found on our blog like the ones from
-March or February
+topic. See the latest status reports on our blog for updates.
+There is also an automatically generated coverage dashboard showing
+what parts of NumPy are already usable.
 
    This is a proposal to provide a fully compatible working NumPy implementation
 for PyPy. This has long been a very commonly requested feature for PyPy
 as well as a worthy goal given that PyPy performs extremely well on numeric
diff --git a/source/numpydonate.txt b/source/numpydonate.txt
--- a/source/numpydonate.txt
+++ b/source/numpydonate.txt
@@ -13,8 +13,9 @@
   quickly as predicted.  The ratio "progress / $ used" so far
   corresponds roughly to what we expected.  The document below is the
   original call for proposal, and we still accept donations for this
-  topic. Latest status reports can be found on our blog like the ones from*
-  `March`_ *or* `February`_
+  topic. See the* `latest status reports`_ *on our blog for updates.
+  There is also an automatically generated* `coverage dashboard`_ *showing
+  what parts of NumPy are already usable.*
 
 
 This is a proposal to provide a fully compatible working `NumPy`_ implementation
@@ -75,8 +76,8 @@
 .. _`requested feature`: http://morepypy.blogspot.com/2011/06/report-back-from-our-survey.html
 .. _`NumPy implementation`: http://morepypy.blogspot.com/2011/05/numpy-in-pypy-status-and-roadmap.html
 .. _`performs well`: http://morepypy.blogspot.com/2011/07/realtime-image-processing-in-python.html
-.. _`March`: http://morepypy.blogspot.co.il/2014/04/numpy-on-pypy-status-update.html
-.. _`February`: http://morepypy.blogspot.co.il/2014/03/numpy-status-update-february.html
+.. _`latest status reports`: http://morepypy.blogspot.com/search/label/numpy
+.. _`coverage dashboard`: http://buildbot.pypy.org/numpy-status/latest.html
 
 Speed
 -----

From noreply at buildbot.pypy.org  Thu Nov 20 11:22:59 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 20 Nov 2014 11:22:59 +0100 (CET)
Subject: [pypy-commit] pypy default: Update the doc: one difference with
	CPython disappeared, but
Message-ID: <20141120102259.594AB1D2859@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r74607:dccc80a97cab
Date: 2014-11-20 11:22 +0100
http://bitbucket.org/pypy/pypy/changeset/dccc80a97cab/

Log:	Update the doc: one difference with CPython disappeared, but there
	are others more subtle ones.

diff --git a/pypy/doc/cpython_differences.rst b/pypy/doc/cpython_differences.rst
--- a/pypy/doc/cpython_differences.rst
+++ b/pypy/doc/cpython_differences.rst
@@ -205,23 +205,28 @@
 The above is true both in CPython and in PyPy.  Differences
 can occur about whether a built-in function or method will
 call an overridden method of *another* object than ``self``.
-In PyPy, they are generally always called, whereas not in
-CPython.  For example, in PyPy, ``dict1.update(dict2)``
-considers that ``dict2`` is just a general mapping object, and
-will thus call overridden ``keys()``  and ``__getitem__()``
-methods on it.  So the following code prints ``42`` on PyPy
-but ``foo`` on CPython::
+In PyPy, they are often called in cases where CPython would not.
+Two examples::
 
-    >>>> class D(dict):
-    ....     def __getitem__(self, key):
-    ....         return 42
-    ....
-    >>>>
-    >>>> d1 = {}
-    >>>> d2 = D(a='foo')
-    >>>> d1.update(d2)
-    >>>> print d1['a']
-    42
+    class D(dict):
+        def __getitem__(self, key):
+            return "%r from D" % (key,)
+
+    class A(object):
+        pass
+
+    a = A()
+    a.__dict__ = D()
+    a.foo = "a's own foo"
+    print a.foo
+    # CPython => a's own foo
+    # PyPy => 'foo' from D
+
+    glob = D(foo="base item")
+    loc = {}
+    exec "print foo" in glob, loc
+    # CPython => base item
+    # PyPy => 'foo' from D
 
 
 Mutating classes of objects which are already used as dictionary keys

From noreply at buildbot.pypy.org  Thu Nov 20 11:34:41 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 20 Nov 2014 11:34:41 +0100 (CET)
Subject: [pypy-commit] pypy default: Add a warning about comparing strings
	with ``is``.
Message-ID: <20141120103441.ECB611D288F@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r74608:677120906bff
Date: 2014-11-20 11:34 +0100
http://bitbucket.org/pypy/pypy/changeset/677120906bff/

Log:	Add a warning about comparing strings with ``is``.

diff --git a/pypy/doc/cpython_differences.rst b/pypy/doc/cpython_differences.rst
--- a/pypy/doc/cpython_differences.rst
+++ b/pypy/doc/cpython_differences.rst
@@ -297,6 +297,9 @@
 above types will return a value that is computed from the argument, and can
 thus be larger than ``sys.maxint`` (i.e. it can be an arbitrary long).
 
+Notably missing from the list above are ``str`` and ``unicode``.  If your
+code relies on comparing strings with ``is``, then it might break in PyPy.
+
 
 Miscellaneous
 -------------

From noreply at buildbot.pypy.org  Thu Nov 20 12:09:54 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 20 Nov 2014 12:09:54 +0100 (CET)
Subject: [pypy-commit] pypy intern-not-immortal: Make intern() return
 non-immortal strings, like it does in CPython.
Message-ID: <20141120110954.78C351C3140@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: intern-not-immortal
Changeset: r74609:be4cd7a1e1fb
Date: 2014-11-20 12:09 +0100
http://bitbucket.org/pypy/pypy/changeset/be4cd7a1e1fb/

Log:	Make intern() return non-immortal strings, like it does in CPython.

diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -14,7 +14,7 @@
     UserDelAction)
 from pypy.interpreter.error import OperationError, new_exception_class, oefmt
 from pypy.interpreter.argument import Arguments
-from pypy.interpreter.miscutils import ThreadLocals
+from pypy.interpreter.miscutils import ThreadLocals, make_weak_value_dictionary
 
 
 __all__ = ['ObjSpace', 'OperationError', 'W_Root']
@@ -384,7 +384,7 @@
         self.builtin_modules = {}
         self.reloading_modules = {}
 
-        self.interned_strings = {}
+        self.interned_strings = make_weak_value_dictionary(self, str, W_Root)
         self.actionflag = ActionFlag()    # changed by the signal module
         self.check_signal_action = None   # changed by the signal module
         self.user_del_action = UserDelAction(self)
@@ -777,25 +777,24 @@
             return self.w_False
 
     def new_interned_w_str(self, w_s):
+        assert isinstance(w_s, W_Root)   # and is not None
         s = self.str_w(w_s)
         if not we_are_translated():
             assert type(s) is str
-        try:
-            return self.interned_strings[s]
-        except KeyError:
-            pass
-        self.interned_strings[s] = w_s
-        return w_s
+        w_s1 = self.interned_strings.get(s)
+        if w_s1 is None:
+            w_s1 = w_s
+            self.interned_strings.set(s, w_s1)
+        return w_s1
 
     def new_interned_str(self, s):
         if not we_are_translated():
             assert type(s) is str
-        try:
-            return self.interned_strings[s]
-        except KeyError:
-            pass
-        w_s = self.interned_strings[s] = self.wrap(s)
-        return w_s
+        w_s1 = self.interned_strings.get(s)
+        if w_s1 is None:
+            w_s1 = self.wrap(s)
+            self.interned_strings.set(s, w_s1)
+        return w_s1
 
     def descr_self_interp_w(self, RequiredClass, w_obj):
         if not isinstance(w_obj, RequiredClass):
diff --git a/pypy/interpreter/miscutils.py b/pypy/interpreter/miscutils.py
--- a/pypy/interpreter/miscutils.py
+++ b/pypy/interpreter/miscutils.py
@@ -31,3 +31,19 @@
 
     def getallvalues(self):
         return {0: self._value}
+
+
+def make_weak_value_dictionary(space, keytype, valuetype):
+    "NOT_RPYTHON"
+    if space.config.translation.rweakref:
+        from rpython.rlib.rweakref import RWeakValueDictionary
+        return RWeakValueDictionary(keytype, valuetype)
+    else:
+        class FakeWeakValueDict(object):
+            def __init__(self):
+                self._dict = {}
+            def get(self, key):
+                return self._dict.get(key, None)
+            def set(self, key, value):
+                self._dict[key] = value
+        return FakeWeakValueDict()
diff --git a/pypy/interpreter/test/test_objspace.py b/pypy/interpreter/test/test_objspace.py
--- a/pypy/interpreter/test/test_objspace.py
+++ b/pypy/interpreter/test/test_objspace.py
@@ -378,3 +378,41 @@
         assert space.str_w(space.getattr(space.sys, w_executable)) == 'foobar'
         space.startup()
         assert space.str_w(space.getattr(space.sys, w_executable)) == 'foobar'
+
+    def test_interned_strings_are_weak(self):
+        import weakref, gc, random
+        space = self.space
+        assert space.config.translation.rweakref
+        w1 = space.new_interned_str("abcdef")
+        w2 = space.new_interned_str("abcdef")
+        assert w2 is w1
+        #
+        # check that 'w1' goes away if we don't hold a reference to it
+        rw1 = weakref.ref(w1)
+        del w1, w2
+        i = 10
+        while rw1() is not None:
+            i -= 1
+            assert i >= 0
+            gc.collect()
+        #
+        s = "foobar%r" % random.random()
+        w0 = space.wrap(s)
+        w1 = space.new_interned_w_str(w0)
+        assert w1 is w0
+        w2 = space.new_interned_w_str(w0)
+        assert w2 is w0
+        w3 = space.wrap(s)
+        assert w3 is not w0
+        w4 = space.new_interned_w_str(w3)
+        assert w4 is w0
+        #
+        # check that 'w0' goes away if we don't hold a reference to it
+        # (even if we hold a reference to 'w3')
+        rw0 = weakref.ref(w0)
+        del w0, w1, w2, w4
+        i = 10
+        while rw0() is not None:
+            i -= 1
+            assert i >= 0
+            gc.collect()

From noreply at buildbot.pypy.org  Thu Nov 20 13:16:43 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 20 Nov 2014 13:16:43 +0100 (CET)
Subject: [pypy-commit] pypy intern-not-immortal: Try to fix marshal's
 handling of interned strings
Message-ID: <20141120121643.97BEC1D266F@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: intern-not-immortal
Changeset: r74610:b866d4c8dbd1
Date: 2014-11-20 13:16 +0100
http://bitbucket.org/pypy/pypy/changeset/b866d4c8dbd1/

Log:	Try to fix marshal's handling of interned strings

diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -796,6 +796,12 @@
             self.interned_strings.set(s, w_s1)
         return w_s1
 
+    def is_interned_str(self, s):
+        # interface for marshal_impl
+        if not we_are_translated():
+            assert type(s) is str
+        return self.interned_strings.get(s) is not None
+
     def descr_self_interp_w(self, RequiredClass, w_obj):
         if not isinstance(w_obj, RequiredClass):
             raise DescrMismatch()
diff --git a/pypy/module/marshal/interp_marshal.py b/pypy/module/marshal/interp_marshal.py
--- a/pypy/module/marshal/interp_marshal.py
+++ b/pypy/module/marshal/interp_marshal.py
@@ -144,7 +144,6 @@
     atom_int(tc, int)           puts code and int
     atom_int64(tc, int64)       puts code and int64
     atom_str(tc, str)           puts code, len and string
-    atom_strlist(tc, strlist)   puts code, len and list of strings
 
     building blocks for compound types:
 
@@ -198,15 +197,6 @@
         self.atom_int(typecode, len(x))
         self.put(x)
 
-    def atom_strlist(self, typecode, tc2, x):
-        self.atom_int(typecode, len(x))
-        atom_str = self.atom_str
-        for item in x:
-            # type(str) seems to be forbidden
-            #if type(item) is not str:
-            #    self.raise_exc('object with wrong type in strlist')
-            atom_str(tc2, item)
-
     def start(self, typecode):
         # type(char) not supported
         self.put(typecode)
@@ -379,16 +369,6 @@
         self.start(typecode)
         return self.get_lng()
 
-    def atom_strlist(self, typecode, tc2):
-        self.start(typecode)
-        lng = self.get_lng()
-        res = [None] * lng
-        idx = 0
-        while idx < lng:
-            res[idx] = self.atom_str(tc2)
-            idx += 1
-        return res
-
     def start(self, typecode):
         tc = self.get1()
         if tc != typecode:
@@ -436,7 +416,6 @@
 
     def get_w_obj(self, allow_null=False):
         space = self.space
-        w_ret = space.w_None # something not None
         tc = self.get1()
         w_ret = self._dispatch[ord(tc)](space, self, tc)
         if w_ret is None and not allow_null:
diff --git a/pypy/objspace/std/marshal_impl.py b/pypy/objspace/std/marshal_impl.py
--- a/pypy/objspace/std/marshal_impl.py
+++ b/pypy/objspace/std/marshal_impl.py
@@ -244,26 +244,19 @@
     return space.newcomplex(real, imag)
 
 
-# XXX currently, intern() is at applevel,
-# and there is no interface to get at the
-# internal table.
-# Move intern to interplevel and add a flag
-# to strings.
-def PySTRING_CHECK_INTERNED(w_str):
-    return False
-
 @marshaller(W_BytesObject)
 def marshal_bytes(space, w_str, m):
     s = space.str_w(w_str)
-    if m.version >= 1 and PySTRING_CHECK_INTERNED(w_str):
+    if m.version >= 1 and space.is_interned_str(s):
         # we use a native rtyper stringdict for speed
-        idx = m.stringtable.get(s, -1)
-        if idx >= 0:
-            m.atom_int(TYPE_STRINGREF, idx)
-        else:
+        try:
+            idx = m.stringtable[s]
+        except KeyError:
             idx = len(m.stringtable)
             m.stringtable[s] = idx
             m.atom_str(TYPE_INTERNED, s)
+        else:
+            m.atom_int(TYPE_STRINGREF, idx)
     else:
         m.atom_str(TYPE_STRING, s)
 
@@ -273,10 +266,8 @@
 
 @unmarshaller(TYPE_INTERNED)
 def unmarshal_interned(space, u, tc):
-    w_ret = space.wrap(u.get_str())
+    w_ret = space.new_interned_str(u.get_str())
     u.stringtable_w.append(w_ret)
-    w_intern = space.builtin.get('intern')
-    space.call_function(w_intern, w_ret)
     return w_ret
 
 @unmarshaller(TYPE_STRINGREF)
@@ -338,6 +329,12 @@
     return None
 
 
+def _put_interned_str_list(space, m, strlist):
+    lst = [None] * len(strlist)
+    for i in range(len(strlist)):
+        lst[i] = space.new_interned_str(strlist[i])
+    m.put_tuple_w(TYPE_TUPLE, lst)
+
 @marshaller(PyCode)
 def marshal_pycode(space, w_pycode, m):
     m.start(TYPE_CODE)
@@ -348,19 +345,18 @@
     m.put_int(x.co_stacksize)
     m.put_int(x.co_flags)
     m.atom_str(TYPE_STRING, x.co_code)
-    m.put_tuple_w(TYPE_TUPLE, x.co_consts_w[:])
-    m.atom_strlist(TYPE_TUPLE, TYPE_INTERNED, [space.str_w(w_name) for w_name in x.co_names_w])
-    m.atom_strlist(TYPE_TUPLE, TYPE_INTERNED, x.co_varnames)
-    m.atom_strlist(TYPE_TUPLE, TYPE_INTERNED, x.co_freevars)
-    m.atom_strlist(TYPE_TUPLE, TYPE_INTERNED, x.co_cellvars)
-    m.atom_str(TYPE_INTERNED, x.co_filename)
-    m.atom_str(TYPE_INTERNED, x.co_name)
+    m.put_tuple_w(TYPE_TUPLE, x.co_consts_w)
+    m.put_tuple_w(TYPE_TUPLE, x.co_names_w)
+    _put_interned_str_list(space, m, x.co_varnames)
+    _put_interned_str_list(space, m, x.co_freevars)
+    _put_interned_str_list(space, m, x.co_cellvars)
+    m.put_w_obj(space.new_interned_str(x.co_filename))
+    m.put_w_obj(space.new_interned_str(x.co_name))
     m.put_int(x.co_firstlineno)
     m.atom_str(TYPE_STRING, x.co_lnotab)
 
-# helper for unmarshalling string lists of code objects.
-# unfortunately they now can be interned or referenced,
-# so we no longer can handle it in interp_marshal.atom_strlist
+# helper for unmarshalling "tuple of string" objects
+# into rpython-level lists of strings.  Only for code objects.
 
 def unmarshal_str(u):
     w_obj = u.get_w_obj()

From noreply at buildbot.pypy.org  Thu Nov 20 13:22:01 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 20 Nov 2014 13:22:01 +0100 (CET)
Subject: [pypy-commit] pypy intern-not-immortal: Test and fix
Message-ID: <20141120122201.DB0D11D283E@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: intern-not-immortal
Changeset: r74611:95fdb025f04d
Date: 2014-11-20 13:21 +0100
http://bitbucket.org/pypy/pypy/changeset/95fdb025f04d/

Log:	Test and fix

diff --git a/rpython/rlib/rweakref.py b/rpython/rlib/rweakref.py
--- a/rpython/rlib/rweakref.py
+++ b/rpython/rlib/rweakref.py
@@ -105,7 +105,7 @@
                                                rtyper.getrepr(self.s_key))
 
     def rtyper_makekey(self):
-        return self.__class__,
+        return self.__class__, self.s_key.rtyper_makekey(), self.valueclassdef
 
     def method_get(self, s_key):
         return annmodel.SomeInstance(self.valueclassdef, can_be_None=True)
@@ -165,7 +165,7 @@
         return _rweakkeydict.WeakKeyDictRepr(rtyper)
 
     def rtyper_makekey(self):
-        return self.__class__,
+        return self.__class__, self.keyclassdef, self.valueclassdef
 
     def method_get(self, s_key):
         assert isinstance(s_key, annmodel.SomeInstance)
diff --git a/rpython/rlib/test/test_rweakvaldict.py b/rpython/rlib/test/test_rweakvaldict.py
--- a/rpython/rlib/test/test_rweakvaldict.py
+++ b/rpython/rlib/test/test_rweakvaldict.py
@@ -144,3 +144,13 @@
             d = RWeakValueDictionary(str, Y)
         d.set("x", X())
     py.test.raises(Exception, interpret, g, [1])
+
+
+def test_bogus_makekey():
+    class X: pass
+    class Y: pass
+    def g():
+        X(); Y()
+        RWeakValueDictionary(str, X).get("foobar")
+        RWeakValueDictionary(int, Y).get(42)
+    interpret(g, [])

From noreply at buildbot.pypy.org  Thu Nov 20 13:52:55 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 20 Nov 2014 13:52:55 +0100 (CET)
Subject: [pypy-commit] pypy intern-not-immortal: Intern the identifier-like
 strings that appear in the compiled code.
Message-ID: <20141120125255.61FCB1C3288@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: intern-not-immortal
Changeset: r74612:e95ebc7224cd
Date: 2014-11-20 13:52 +0100
http://bitbucket.org/pypy/pypy/changeset/e95ebc7224cd/

Log:	Intern the identifier-like strings that appear in the compiled code.

diff --git a/pypy/interpreter/astcompiler/assemble.py b/pypy/interpreter/astcompiler/assemble.py
--- a/pypy/interpreter/astcompiler/assemble.py
+++ b/pypy/interpreter/astcompiler/assemble.py
@@ -2,7 +2,7 @@
 Python control flow graph generation and bytecode assembly.
 """
 
-from pypy.interpreter.astcompiler import ast, symtable
+from pypy.interpreter.astcompiler import ast, symtable, misc
 from pypy.interpreter import pycode
 from pypy.tool import stdlib_opcode as ops
 
@@ -365,7 +365,9 @@
                     raise
                 break
             w_index = space.getitem(w_consts, w_key)
-            consts_w[space.int_w(w_index)] = space.getitem(w_key, first)
+            w_constant = space.getitem(w_key, first)
+            w_constant = misc.intern_if_common_string(space, w_constant)
+            consts_w[space.int_w(w_index)] = w_constant
         return consts_w
 
     def _get_code_flags(self):
diff --git a/pypy/interpreter/astcompiler/misc.py b/pypy/interpreter/astcompiler/misc.py
--- a/pypy/interpreter/astcompiler/misc.py
+++ b/pypy/interpreter/astcompiler/misc.py
@@ -106,3 +106,13 @@
     except IndexError:
         return name
     return "_%s%s" % (klass[i:], name)
+
+
+def intern_if_common_string(space, w_const):
+    # only intern identifier-like strings
+    if not space.is_w(space.type(w_const), space.w_str):
+        return w_const
+    for c in space.str_w(w_const):
+        if not (c.isalnum() or c == '_'):
+            return w_const
+    return space.new_interned_w_str(w_const)
diff --git a/pypy/interpreter/astcompiler/optimize.py b/pypy/interpreter/astcompiler/optimize.py
--- a/pypy/interpreter/astcompiler/optimize.py
+++ b/pypy/interpreter/astcompiler/optimize.py
@@ -272,6 +272,11 @@
                 if w_const is None:
                     return tup
                 consts_w[i] = w_const
+            # intern the string constants packed into the tuple here,
+            # because assemble.py will see the result as just a tuple constant
+            for i in range(len(consts_w)):
+                consts_w[i] = misc.intern_if_common_string(
+                    self.space, consts_w[i])
         else:
             consts_w = []
         w_consts = self.space.newtuple(consts_w)
diff --git a/pypy/interpreter/test/test_compiler.py b/pypy/interpreter/test/test_compiler.py
--- a/pypy/interpreter/test/test_compiler.py
+++ b/pypy/interpreter/test/test_compiler.py
@@ -970,7 +970,12 @@
             sys.stdout = out
         output = s.getvalue()
         assert "CALL_METHOD" in output
-            
+
+    def test_interned_strings(self):
+        source = """x = ('foo_bar42', 5); y = 'foo_bar42'; z = x[0]"""
+        exec source
+        assert y is z
+
 
 class AppTestExceptions:
     def test_indentation_error(self):

From noreply at buildbot.pypy.org  Thu Nov 20 14:26:34 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 20 Nov 2014 14:26:34 +0100 (CET)
Subject: [pypy-commit] pypy default: Obscure one-time crash of the test:
 maybe the file descriptor 42 was in use?
Message-ID: <20141120132634.99A661C3140@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r74613:21763edebb42
Date: 2014-11-20 14:26 +0100
http://bitbucket.org/pypy/pypy/changeset/21763edebb42/

Log:	Obscure one-time crash of the test: maybe the file descriptor 42 was
	in use?

diff --git a/pypy/module/_file/test/test_file.py b/pypy/module/_file/test/test_file.py
--- a/pypy/module/_file/test/test_file.py
+++ b/pypy/module/_file/test/test_file.py
@@ -304,7 +304,7 @@
             py.test.skip("works with internals of _file impl on py.py")
         state = [0]
         def read(fd, n=None):
-            if fd != 42:
+            if fd != 424242:
                 return cls.old_read(fd, n)
             if state[0] == 0:
                 state[0] += 1
@@ -315,7 +315,7 @@
             return ''
         os.read = read
         stdin = W_File(cls.space)
-        stdin.file_fdopen(42, 'rb', 1)
+        stdin.file_fdopen(424242, 'rb', 1)
         stdin.name = ''
         cls.w_stream = stdin
 

From noreply at buildbot.pypy.org  Thu Nov 20 14:54:31 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 20 Nov 2014 14:54:31 +0100 (CET)
Subject: [pypy-commit] pypy intern-not-immortal: ready to merge
Message-ID: <20141120135431.7F14B1C3288@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: intern-not-immortal
Changeset: r74614:f8b55e922637
Date: 2014-11-20 14:51 +0100
http://bitbucket.org/pypy/pypy/changeset/f8b55e922637/

Log:	ready to merge


From noreply at buildbot.pypy.org  Thu Nov 20 14:54:34 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 20 Nov 2014 14:54:34 +0100 (CET)
Subject: [pypy-commit] pypy default: Merge intern-not-immortal: fix intern()
 to return mortal strings, 
Message-ID: <20141120135434.159411C3288@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r74615:2afdc25d04e2
Date: 2014-11-20 14:51 +0100
http://bitbucket.org/pypy/pypy/changeset/2afdc25d04e2/

Log:	Merge intern-not-immortal: fix intern() to return mortal strings,
	and try to fix things so that the AST compiler and the unmarshaller
	try to produce correctly-interned strings.

diff --git a/pypy/interpreter/astcompiler/assemble.py b/pypy/interpreter/astcompiler/assemble.py
--- a/pypy/interpreter/astcompiler/assemble.py
+++ b/pypy/interpreter/astcompiler/assemble.py
@@ -2,7 +2,7 @@
 Python control flow graph generation and bytecode assembly.
 """
 
-from pypy.interpreter.astcompiler import ast, symtable
+from pypy.interpreter.astcompiler import ast, symtable, misc
 from pypy.interpreter import pycode
 from pypy.tool import stdlib_opcode as ops
 
@@ -365,7 +365,9 @@
                     raise
                 break
             w_index = space.getitem(w_consts, w_key)
-            consts_w[space.int_w(w_index)] = space.getitem(w_key, first)
+            w_constant = space.getitem(w_key, first)
+            w_constant = misc.intern_if_common_string(space, w_constant)
+            consts_w[space.int_w(w_index)] = w_constant
         return consts_w
 
     def _get_code_flags(self):
diff --git a/pypy/interpreter/astcompiler/misc.py b/pypy/interpreter/astcompiler/misc.py
--- a/pypy/interpreter/astcompiler/misc.py
+++ b/pypy/interpreter/astcompiler/misc.py
@@ -106,3 +106,13 @@
     except IndexError:
         return name
     return "_%s%s" % (klass[i:], name)
+
+
+def intern_if_common_string(space, w_const):
+    # only intern identifier-like strings
+    if not space.is_w(space.type(w_const), space.w_str):
+        return w_const
+    for c in space.str_w(w_const):
+        if not (c.isalnum() or c == '_'):
+            return w_const
+    return space.new_interned_w_str(w_const)
diff --git a/pypy/interpreter/astcompiler/optimize.py b/pypy/interpreter/astcompiler/optimize.py
--- a/pypy/interpreter/astcompiler/optimize.py
+++ b/pypy/interpreter/astcompiler/optimize.py
@@ -272,6 +272,11 @@
                 if w_const is None:
                     return tup
                 consts_w[i] = w_const
+            # intern the string constants packed into the tuple here,
+            # because assemble.py will see the result as just a tuple constant
+            for i in range(len(consts_w)):
+                consts_w[i] = misc.intern_if_common_string(
+                    self.space, consts_w[i])
         else:
             consts_w = []
         w_consts = self.space.newtuple(consts_w)
diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -14,7 +14,7 @@
     UserDelAction)
 from pypy.interpreter.error import OperationError, new_exception_class, oefmt
 from pypy.interpreter.argument import Arguments
-from pypy.interpreter.miscutils import ThreadLocals
+from pypy.interpreter.miscutils import ThreadLocals, make_weak_value_dictionary
 
 
 __all__ = ['ObjSpace', 'OperationError', 'W_Root']
@@ -384,7 +384,7 @@
         self.builtin_modules = {}
         self.reloading_modules = {}
 
-        self.interned_strings = {}
+        self.interned_strings = make_weak_value_dictionary(self, str, W_Root)
         self.actionflag = ActionFlag()    # changed by the signal module
         self.check_signal_action = None   # changed by the signal module
         self.user_del_action = UserDelAction(self)
@@ -777,25 +777,30 @@
             return self.w_False
 
     def new_interned_w_str(self, w_s):
+        assert isinstance(w_s, W_Root)   # and is not None
         s = self.str_w(w_s)
         if not we_are_translated():
             assert type(s) is str
-        try:
-            return self.interned_strings[s]
-        except KeyError:
-            pass
-        self.interned_strings[s] = w_s
-        return w_s
+        w_s1 = self.interned_strings.get(s)
+        if w_s1 is None:
+            w_s1 = w_s
+            self.interned_strings.set(s, w_s1)
+        return w_s1
 
     def new_interned_str(self, s):
         if not we_are_translated():
             assert type(s) is str
-        try:
-            return self.interned_strings[s]
-        except KeyError:
-            pass
-        w_s = self.interned_strings[s] = self.wrap(s)
-        return w_s
+        w_s1 = self.interned_strings.get(s)
+        if w_s1 is None:
+            w_s1 = self.wrap(s)
+            self.interned_strings.set(s, w_s1)
+        return w_s1
+
+    def is_interned_str(self, s):
+        # interface for marshal_impl
+        if not we_are_translated():
+            assert type(s) is str
+        return self.interned_strings.get(s) is not None
 
     def descr_self_interp_w(self, RequiredClass, w_obj):
         if not isinstance(w_obj, RequiredClass):
diff --git a/pypy/interpreter/miscutils.py b/pypy/interpreter/miscutils.py
--- a/pypy/interpreter/miscutils.py
+++ b/pypy/interpreter/miscutils.py
@@ -31,3 +31,19 @@
 
     def getallvalues(self):
         return {0: self._value}
+
+
+def make_weak_value_dictionary(space, keytype, valuetype):
+    "NOT_RPYTHON"
+    if space.config.translation.rweakref:
+        from rpython.rlib.rweakref import RWeakValueDictionary
+        return RWeakValueDictionary(keytype, valuetype)
+    else:
+        class FakeWeakValueDict(object):
+            def __init__(self):
+                self._dict = {}
+            def get(self, key):
+                return self._dict.get(key, None)
+            def set(self, key, value):
+                self._dict[key] = value
+        return FakeWeakValueDict()
diff --git a/pypy/interpreter/test/test_compiler.py b/pypy/interpreter/test/test_compiler.py
--- a/pypy/interpreter/test/test_compiler.py
+++ b/pypy/interpreter/test/test_compiler.py
@@ -970,7 +970,12 @@
             sys.stdout = out
         output = s.getvalue()
         assert "CALL_METHOD" in output
-            
+
+    def test_interned_strings(self):
+        source = """x = ('foo_bar42', 5); y = 'foo_bar42'; z = x[0]"""
+        exec source
+        assert y is z
+
 
 class AppTestExceptions:
     def test_indentation_error(self):
diff --git a/pypy/interpreter/test/test_objspace.py b/pypy/interpreter/test/test_objspace.py
--- a/pypy/interpreter/test/test_objspace.py
+++ b/pypy/interpreter/test/test_objspace.py
@@ -378,3 +378,41 @@
         assert space.str_w(space.getattr(space.sys, w_executable)) == 'foobar'
         space.startup()
         assert space.str_w(space.getattr(space.sys, w_executable)) == 'foobar'
+
+    def test_interned_strings_are_weak(self):
+        import weakref, gc, random
+        space = self.space
+        assert space.config.translation.rweakref
+        w1 = space.new_interned_str("abcdef")
+        w2 = space.new_interned_str("abcdef")
+        assert w2 is w1
+        #
+        # check that 'w1' goes away if we don't hold a reference to it
+        rw1 = weakref.ref(w1)
+        del w1, w2
+        i = 10
+        while rw1() is not None:
+            i -= 1
+            assert i >= 0
+            gc.collect()
+        #
+        s = "foobar%r" % random.random()
+        w0 = space.wrap(s)
+        w1 = space.new_interned_w_str(w0)
+        assert w1 is w0
+        w2 = space.new_interned_w_str(w0)
+        assert w2 is w0
+        w3 = space.wrap(s)
+        assert w3 is not w0
+        w4 = space.new_interned_w_str(w3)
+        assert w4 is w0
+        #
+        # check that 'w0' goes away if we don't hold a reference to it
+        # (even if we hold a reference to 'w3')
+        rw0 = weakref.ref(w0)
+        del w0, w1, w2, w4
+        i = 10
+        while rw0() is not None:
+            i -= 1
+            assert i >= 0
+            gc.collect()
diff --git a/pypy/module/marshal/interp_marshal.py b/pypy/module/marshal/interp_marshal.py
--- a/pypy/module/marshal/interp_marshal.py
+++ b/pypy/module/marshal/interp_marshal.py
@@ -144,7 +144,6 @@
     atom_int(tc, int)           puts code and int
     atom_int64(tc, int64)       puts code and int64
     atom_str(tc, str)           puts code, len and string
-    atom_strlist(tc, strlist)   puts code, len and list of strings
 
     building blocks for compound types:
 
@@ -198,15 +197,6 @@
         self.atom_int(typecode, len(x))
         self.put(x)
 
-    def atom_strlist(self, typecode, tc2, x):
-        self.atom_int(typecode, len(x))
-        atom_str = self.atom_str
-        for item in x:
-            # type(str) seems to be forbidden
-            #if type(item) is not str:
-            #    self.raise_exc('object with wrong type in strlist')
-            atom_str(tc2, item)
-
     def start(self, typecode):
         # type(char) not supported
         self.put(typecode)
@@ -379,16 +369,6 @@
         self.start(typecode)
         return self.get_lng()
 
-    def atom_strlist(self, typecode, tc2):
-        self.start(typecode)
-        lng = self.get_lng()
-        res = [None] * lng
-        idx = 0
-        while idx < lng:
-            res[idx] = self.atom_str(tc2)
-            idx += 1
-        return res
-
     def start(self, typecode):
         tc = self.get1()
         if tc != typecode:
@@ -436,7 +416,6 @@
 
     def get_w_obj(self, allow_null=False):
         space = self.space
-        w_ret = space.w_None # something not None
         tc = self.get1()
         w_ret = self._dispatch[ord(tc)](space, self, tc)
         if w_ret is None and not allow_null:
diff --git a/pypy/objspace/std/marshal_impl.py b/pypy/objspace/std/marshal_impl.py
--- a/pypy/objspace/std/marshal_impl.py
+++ b/pypy/objspace/std/marshal_impl.py
@@ -244,26 +244,19 @@
     return space.newcomplex(real, imag)
 
 
-# XXX currently, intern() is at applevel,
-# and there is no interface to get at the
-# internal table.
-# Move intern to interplevel and add a flag
-# to strings.
-def PySTRING_CHECK_INTERNED(w_str):
-    return False
-
 @marshaller(W_BytesObject)
 def marshal_bytes(space, w_str, m):
     s = space.str_w(w_str)
-    if m.version >= 1 and PySTRING_CHECK_INTERNED(w_str):
+    if m.version >= 1 and space.is_interned_str(s):
         # we use a native rtyper stringdict for speed
-        idx = m.stringtable.get(s, -1)
-        if idx >= 0:
-            m.atom_int(TYPE_STRINGREF, idx)
-        else:
+        try:
+            idx = m.stringtable[s]
+        except KeyError:
             idx = len(m.stringtable)
             m.stringtable[s] = idx
             m.atom_str(TYPE_INTERNED, s)
+        else:
+            m.atom_int(TYPE_STRINGREF, idx)
     else:
         m.atom_str(TYPE_STRING, s)
 
@@ -273,10 +266,8 @@
 
 @unmarshaller(TYPE_INTERNED)
 def unmarshal_interned(space, u, tc):
-    w_ret = space.wrap(u.get_str())
+    w_ret = space.new_interned_str(u.get_str())
     u.stringtable_w.append(w_ret)
-    w_intern = space.builtin.get('intern')
-    space.call_function(w_intern, w_ret)
     return w_ret
 
 @unmarshaller(TYPE_STRINGREF)
@@ -338,6 +329,12 @@
     return None
 
 
+def _put_interned_str_list(space, m, strlist):
+    lst = [None] * len(strlist)
+    for i in range(len(strlist)):
+        lst[i] = space.new_interned_str(strlist[i])
+    m.put_tuple_w(TYPE_TUPLE, lst)
+
 @marshaller(PyCode)
 def marshal_pycode(space, w_pycode, m):
     m.start(TYPE_CODE)
@@ -348,19 +345,18 @@
     m.put_int(x.co_stacksize)
     m.put_int(x.co_flags)
     m.atom_str(TYPE_STRING, x.co_code)
-    m.put_tuple_w(TYPE_TUPLE, x.co_consts_w[:])
-    m.atom_strlist(TYPE_TUPLE, TYPE_INTERNED, [space.str_w(w_name) for w_name in x.co_names_w])
-    m.atom_strlist(TYPE_TUPLE, TYPE_INTERNED, x.co_varnames)
-    m.atom_strlist(TYPE_TUPLE, TYPE_INTERNED, x.co_freevars)
-    m.atom_strlist(TYPE_TUPLE, TYPE_INTERNED, x.co_cellvars)
-    m.atom_str(TYPE_INTERNED, x.co_filename)
-    m.atom_str(TYPE_INTERNED, x.co_name)
+    m.put_tuple_w(TYPE_TUPLE, x.co_consts_w)
+    m.put_tuple_w(TYPE_TUPLE, x.co_names_w)
+    _put_interned_str_list(space, m, x.co_varnames)
+    _put_interned_str_list(space, m, x.co_freevars)
+    _put_interned_str_list(space, m, x.co_cellvars)
+    m.put_w_obj(space.new_interned_str(x.co_filename))
+    m.put_w_obj(space.new_interned_str(x.co_name))
     m.put_int(x.co_firstlineno)
     m.atom_str(TYPE_STRING, x.co_lnotab)
 
-# helper for unmarshalling string lists of code objects.
-# unfortunately they now can be interned or referenced,
-# so we no longer can handle it in interp_marshal.atom_strlist
+# helper for unmarshalling "tuple of string" objects
+# into rpython-level lists of strings.  Only for code objects.
 
 def unmarshal_str(u):
     w_obj = u.get_w_obj()
diff --git a/rpython/rlib/rweakref.py b/rpython/rlib/rweakref.py
--- a/rpython/rlib/rweakref.py
+++ b/rpython/rlib/rweakref.py
@@ -105,7 +105,7 @@
                                                rtyper.getrepr(self.s_key))
 
     def rtyper_makekey(self):
-        return self.__class__,
+        return self.__class__, self.s_key.rtyper_makekey(), self.valueclassdef
 
     def method_get(self, s_key):
         return annmodel.SomeInstance(self.valueclassdef, can_be_None=True)
@@ -165,7 +165,7 @@
         return _rweakkeydict.WeakKeyDictRepr(rtyper)
 
     def rtyper_makekey(self):
-        return self.__class__,
+        return self.__class__, self.keyclassdef, self.valueclassdef
 
     def method_get(self, s_key):
         assert isinstance(s_key, annmodel.SomeInstance)
diff --git a/rpython/rlib/test/test_rweakvaldict.py b/rpython/rlib/test/test_rweakvaldict.py
--- a/rpython/rlib/test/test_rweakvaldict.py
+++ b/rpython/rlib/test/test_rweakvaldict.py
@@ -144,3 +144,13 @@
             d = RWeakValueDictionary(str, Y)
         d.set("x", X())
     py.test.raises(Exception, interpret, g, [1])
+
+
+def test_bogus_makekey():
+    class X: pass
+    class Y: pass
+    def g():
+        X(); Y()
+        RWeakValueDictionary(str, X).get("foobar")
+        RWeakValueDictionary(int, Y).get(42)
+    interpret(g, [])

From noreply at buildbot.pypy.org  Thu Nov 20 16:47:28 2014
From: noreply at buildbot.pypy.org (Raemi)
Date: Thu, 20 Nov 2014 16:47:28 +0100 (CET)
Subject: [pypy-commit] pypy stmgc-c7: fix plot to show all threads
Message-ID: <20141120154729.0096D1D283E@cobra.cs.uni-duesseldorf.de>

Author: Remi Meier 
Branch: stmgc-c7
Changeset: r74616:71d03ab2f30b
Date: 2014-11-20 16:47 +0100
http://bitbucket.org/pypy/pypy/changeset/71d03ab2f30b/

Log:	fix plot to show all threads

diff --git a/pypy/stm/plot_stm_log.py b/pypy/stm/plot_stm_log.py
--- a/pypy/stm/plot_stm_log.py
+++ b/pypy/stm/plot_stm_log.py
@@ -107,17 +107,25 @@
                 tr.pauses[-1] = (tr.pauses[-1][0], entry.timestamp)
 
 
-        # attach logentry as transaction info
+        # attach logentry as clickable transaction info
         tr = curr_trs.get(th_num)
+        if tr is None:
+            # no current transaction
+            # try to attach it to the last transaction
+            tr = finished_trs.get(th_num, [None])[-1]
         if tr is not None:
             tr.info.append(str(entry))
-        if entry.event in (psl.STM_ABORTING_OTHER_CONTENTION,):
+        # also affects other transaction:
+        if entry.marker2:
             tr2 = curr_trs.get(entry.otherthreadnum)
+            if tr2 is None:
+                tr2 = finished_trs.get(entry.otherthreadnum, [None])[-1]
             if tr2 is not None:
                 tr2.info.append(str(entry))
 
 
 
+
     # plt.ion()
     for th_num, trs in finished_trs.items():
         # plt.draw()
@@ -170,9 +178,9 @@
     thread_count = len(trs)
 
     axs[0].set_ylabel("Thread")
-    axs[0].set_ylim(0, thread_count)
-    axs[0].set_yticks([r+0.5 for r in range(thread_count)])
-    axs[0].set_yticklabels(range(thread_count))
+    axs[0].set_ylim(0, thread_count+2)
+    axs[0].set_yticks([r+0.5 for r in range(thread_count+1)])
+    axs[0].set_yticklabels(range(thread_count+1))
     #axs[0].set_xticks([])
     print "Drawn."
 

From noreply at buildbot.pypy.org  Thu Nov 20 21:34:45 2014
From: noreply at buildbot.pypy.org (rlamy)
Date: Thu, 20 Nov 2014 21:34:45 +0100 (CET)
Subject: [pypy-commit] pypy framestate: inline save_locals_stack() and
	restore_locals_stack()
Message-ID: <20141120203445.A8A7A1D285E@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy 
Branch: framestate
Changeset: r74617:0c9d4802f0ab
Date: 2014-11-20 03:44 +0000
http://bitbucket.org/pypy/pypy/changeset/0c9d4802f0ab/

Log:	inline save_locals_stack() and restore_locals_stack()

diff --git a/rpython/flowspace/flowcontext.py b/rpython/flowspace/flowcontext.py
--- a/rpython/flowspace/flowcontext.py
+++ b/rpython/flowspace/flowcontext.py
@@ -344,17 +344,9 @@
             self.stack[n] = None
         self.stackdepth = finaldepth
 
-    def save_locals_stack(self):
-        return self.locals_w + self.stack[:self.stackdepth]
-
-    def restore_locals_stack(self, items_w):
-        self.locals_w = items_w[:self.nlocals]
-        self.stack[:len(items_w) - self.nlocals] = items_w[self.nlocals:]
-        self.dropvaluesuntil(len(items_w) - self.nlocals)
-
     def getstate(self, next_offset):
-        # getfastscope() can return real None, for undefined locals
-        data = self.save_locals_stack()
+        data = self.locals_w[:]
+        data.extend(self.stack[:self.stackdepth])
         if self.last_exception is None:
             data.append(Constant(None))
             data.append(Constant(None))
@@ -368,7 +360,9 @@
         """ Reset the context to the given frame state. """
         data = state.mergeable[:]
         recursively_unflatten(data)
-        self.restore_locals_stack(data[:-2])  # Nones == undefined locals
+        self.locals_w = data[:self.nlocals]
+        self.stack[:len(data) - 2 - self.nlocals] = data[self.nlocals:-2]
+        self.dropvaluesuntil(len(data) - 2 - self.nlocals)
         if data[-2] == Constant(None):
             assert data[-1] == Constant(None)
             self.last_exception = None

From noreply at buildbot.pypy.org  Thu Nov 20 21:34:47 2014
From: noreply at buildbot.pypy.org (rlamy)
Date: Thu, 20 Nov 2014 21:34:47 +0100 (CET)
Subject: [pypy-commit] pypy framestate: Use a simple variable-sized list for
	the stack
Message-ID: <20141120203447.E36071D285E@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy 
Branch: framestate
Changeset: r74618:10aac48469bc
Date: 2014-11-20 04:10 +0000
http://bitbucket.org/pypy/pypy/changeset/10aac48469bc/

Log:	Use a simple variable-sized list for the stack

diff --git a/rpython/flowspace/flowcontext.py b/rpython/flowspace/flowcontext.py
--- a/rpython/flowspace/flowcontext.py
+++ b/rpython/flowspace/flowcontext.py
@@ -309,44 +309,41 @@
         The locals are ordered according to self.pycode.signature.
         """
         self.nlocals = code.co_nlocals
-        self.stackdepth = 0
         self.locals_w = [None] * code.co_nlocals
-        self.stack = [None] * code.co_stacksize
+        self.stack = []
+
+    @property
+    def stackdepth(self):
+        return len(self.stack)
 
     def pushvalue(self, w_object):
-        depth = self.stackdepth
-        self.stackdepth = depth + 1
-        self.stack[depth] = w_object
+        self.stack.append(w_object)
 
     def popvalue(self):
-        depth = self.stackdepth - 1
-        w_object = self.stack[depth]
-        self.stack[depth] = None
-        self.stackdepth = depth
-        return w_object
+        return self.stack.pop()
 
     def peekvalue(self, index_from_top=0):
         # NOTE: top of the stack is peekvalue(0).
-        index = self.stackdepth + ~index_from_top
+        index = ~index_from_top
         return self.stack[index]
 
     def settopvalue(self, w_object, index_from_top=0):
-        index = self.stackdepth + ~index_from_top
+        index = ~index_from_top
         self.stack[index] = w_object
 
     def popvalues(self, n):
-        values_w = [self.popvalue() for i in range(n)]
-        values_w.reverse()
+        if n == 0:
+            return []
+        values_w = self.stack[-n:]
+        del self.stack[-n:]
         return values_w
 
     def dropvaluesuntil(self, finaldepth):
-        for n in range(finaldepth, self.stackdepth):
-            self.stack[n] = None
-        self.stackdepth = finaldepth
+        del self.stack[finaldepth:]
 
     def getstate(self, next_offset):
         data = self.locals_w[:]
-        data.extend(self.stack[:self.stackdepth])
+        data.extend(self.stack)
         if self.last_exception is None:
             data.append(Constant(None))
             data.append(Constant(None))
@@ -361,8 +358,7 @@
         data = state.mergeable[:]
         recursively_unflatten(data)
         self.locals_w = data[:self.nlocals]
-        self.stack[:len(data) - 2 - self.nlocals] = data[self.nlocals:-2]
-        self.dropvaluesuntil(len(data) - 2 - self.nlocals)
+        self.stack = data[self.nlocals:-2]
         if data[-2] == Constant(None):
             assert data[-1] == Constant(None)
             self.last_exception = None

From noreply at buildbot.pypy.org  Thu Nov 20 21:34:49 2014
From: noreply at buildbot.pypy.org (rlamy)
Date: Thu, 20 Nov 2014 21:34:49 +0100 (CET)
Subject: [pypy-commit] pypy framestate: Make FlowSignal interface more
	convenient
Message-ID: <20141120203449.360951D285E@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy 
Branch: framestate
Changeset: r74619:092128f86b9d
Date: 2014-11-20 20:34 +0000
http://bitbucket.org/pypy/pypy/changeset/092128f86b9d/

Log:	Make FlowSignal interface more convenient

diff --git a/rpython/flowspace/flowcontext.py b/rpython/flowspace/flowcontext.py
--- a/rpython/flowspace/flowcontext.py
+++ b/rpython/flowspace/flowcontext.py
@@ -1206,22 +1206,26 @@
     def nomoreblocks(self):
         raise BytecodeCorruption("misplaced bytecode - should not return")
 
+    def __eq__(self, other):
+        return type(other) is type(self) and other.args == self.args
+
 
 class Return(FlowSignal):
     """Signals a 'return' statement.
-    Argument is the wrapped object to return."""
-
+    Argument is the wrapped object to return.
+    """
     def __init__(self, w_value):
         self.w_value = w_value
 
     def nomoreblocks(self):
         raise Return(self.w_value)
 
-    def state_unpack_variables(self):
+    @property
+    def args(self):
         return [self.w_value]
 
     @staticmethod
-    def state_pack_variables(w_value):
+    def rebuild(w_value):
         return Return(w_value)
 
 class Raise(FlowSignal):
@@ -1234,11 +1238,12 @@
     def nomoreblocks(self):
         raise self
 
-    def state_unpack_variables(self):
+    @property
+    def args(self):
         return [self.w_exc.w_type, self.w_exc.w_value]
 
     @staticmethod
-    def state_pack_variables(w_type, w_value):
+    def rebuild(w_type, w_value):
         return Raise(FSException(w_type, w_value))
 
 class RaiseImplicit(Raise):
@@ -1248,11 +1253,12 @@
 class Break(FlowSignal):
     """Signals a 'break' statement."""
 
-    def state_unpack_variables(self):
+    @property
+    def args(self):
         return []
 
     @staticmethod
-    def state_pack_variables():
+    def rebuild():
         return Break.singleton
 
 Break.singleton = Break()
@@ -1264,11 +1270,12 @@
     def __init__(self, jump_to):
         self.jump_to = jump_to
 
-    def state_unpack_variables(self):
+    @property
+    def args(self):
         return [const(self.jump_to)]
 
     @staticmethod
-    def state_pack_variables(w_jump_to):
+    def rebuild(w_jump_to):
         return Continue(w_jump_to.value)
 
 
diff --git a/rpython/flowspace/framestate.py b/rpython/flowspace/framestate.py
--- a/rpython/flowspace/framestate.py
+++ b/rpython/flowspace/framestate.py
@@ -109,7 +109,7 @@
         if not isinstance(unroller, FlowSignal):
             i += 1
         else:
-            vars = unroller.state_unpack_variables()
+            vars = unroller.args
             key = unroller.__class__, len(vars)
             try:
                 tag = PICKLE_TAGS[key]
@@ -126,5 +126,5 @@
             unrollerclass, argcount = UNPICKLE_TAGS[item]
             arguments = lst[i + 1:i + 1 + argcount]
             del lst[i + 1:i + 1 + argcount]
-            unroller = unrollerclass.state_pack_variables(*arguments)
+            unroller = unrollerclass.rebuild(*arguments)
             lst[i] = unroller
diff --git a/rpython/flowspace/test/test_flowcontext.py b/rpython/flowspace/test/test_flowcontext.py
new file mode 100644
--- /dev/null
+++ b/rpython/flowspace/test/test_flowcontext.py
@@ -0,0 +1,15 @@
+""" Unit tests for flowcontext.py """
+import pytest
+from rpython.flowspace.model import Variable, FSException
+from rpython.flowspace.flowcontext import (
+    Return, Raise, RaiseImplicit, Continue, Break)
+
+ at pytest.mark.parametrize('signal', [
+    Return(Variable()),
+    Raise(FSException(Variable(), Variable())),
+    #RaiseImplicit(FSException(Variable(), Variable())),
+    Break(),
+    Continue(42),
+])
+def test_signals(signal):
+    assert signal.rebuild(*signal.args) == signal

From noreply at buildbot.pypy.org  Fri Nov 21 00:46:41 2014
From: noreply at buildbot.pypy.org (bdkearns)
Date: Fri, 21 Nov 2014 00:46:41 +0100 (CET)
Subject: [pypy-commit] pypy default: fix some socket tests for python.org
	changes
Message-ID: <20141120234641.154B21D2848@cobra.cs.uni-duesseldorf.de>

Author: Brian Kearns 
Branch: 
Changeset: r74620:4c456cd88383
Date: 2014-11-20 18:46 -0500
http://bitbucket.org/pypy/pypy/changeset/4c456cd88383/

Log:	fix some socket tests for python.org changes

diff --git a/rpython/rlib/test/test_rsocket.py b/rpython/rlib/test/test_rsocket.py
--- a/rpython/rlib/test/test_rsocket.py
+++ b/rpython/rlib/test/test_rsocket.py
@@ -311,7 +311,7 @@
     assert isinstance(lst, list)
     found = False
     for family, socktype, protocol, canonname, addr in lst:
-        if addr.get_host() == '140.211.10.69':
+        if addr.get_host() == '104.130.43.121':
             found = True
     result[i] += found
 
diff --git a/rpython/translator/sandbox/test/test_sandlib.py b/rpython/translator/sandbox/test/test_sandlib.py
--- a/rpython/translator/sandbox/test/test_sandlib.py
+++ b/rpython/translator/sandbox/test/test_sandlib.py
@@ -117,7 +117,7 @@
 
     proc = SocketProc([exe])
     output, error = proc.communicate("")
-    assert output.startswith('HTTP/1.0 503 Service Unavailable')
+    assert output.startswith('HTTP/1.1 301 Moved Permanently')
 
 @supported
 def test_oserror():

From noreply at buildbot.pypy.org  Fri Nov 21 00:57:15 2014
From: noreply at buildbot.pypy.org (rlamy)
Date: Fri, 21 Nov 2014 00:57:15 +0100 (CET)
Subject: [pypy-commit] pypy framestate: get RaiseImplicit.rebuild to behave
	as it should
Message-ID: <20141120235715.CF4CB1D2859@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy 
Branch: framestate
Changeset: r74621:e0bf1637cf8b
Date: 2014-11-20 22:48 +0000
http://bitbucket.org/pypy/pypy/changeset/e0bf1637cf8b/

Log:	get RaiseImplicit.rebuild to behave as it should

diff --git a/rpython/flowspace/flowcontext.py b/rpython/flowspace/flowcontext.py
--- a/rpython/flowspace/flowcontext.py
+++ b/rpython/flowspace/flowcontext.py
@@ -365,6 +365,13 @@
         else:
             self.last_exception = FSException(data[-2], data[-1])
         self.blockstack = state.blocklist[:]
+        self._normalize_raise_signals()
+
+    def _normalize_raise_signals(self):
+        st = self.stack
+        for i in range(len(st)):
+            if isinstance(st[i], RaiseImplicit):
+                st[i] = Raise(st[i].w_exc)
 
     def guessbool(self, w_condition):
         if isinstance(w_condition, Constant):
@@ -1242,9 +1249,9 @@
     def args(self):
         return [self.w_exc.w_type, self.w_exc.w_value]
 
-    @staticmethod
-    def rebuild(w_type, w_value):
-        return Raise(FSException(w_type, w_value))
+    @classmethod
+    def rebuild(cls, w_type, w_value):
+        return cls(FSException(w_type, w_value))
 
 class RaiseImplicit(Raise):
     """Signals an exception raised implicitly"""
diff --git a/rpython/flowspace/test/test_flowcontext.py b/rpython/flowspace/test/test_flowcontext.py
--- a/rpython/flowspace/test/test_flowcontext.py
+++ b/rpython/flowspace/test/test_flowcontext.py
@@ -7,7 +7,7 @@
 @pytest.mark.parametrize('signal', [
     Return(Variable()),
     Raise(FSException(Variable(), Variable())),
-    #RaiseImplicit(FSException(Variable(), Variable())),
+    RaiseImplicit(FSException(Variable(), Variable())),
     Break(),
     Continue(42),
 ])

From noreply at buildbot.pypy.org  Fri Nov 21 00:57:17 2014
From: noreply at buildbot.pypy.org (rlamy)
Date: Fri, 21 Nov 2014 00:57:17 +0100 (CET)
Subject: [pypy-commit] pypy framestate: Refactor FrameState
Message-ID: <20141120235717.1982E1D2859@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy 
Branch: framestate
Changeset: r74622:06b064a032d8
Date: 2014-11-20 23:48 +0000
http://bitbucket.org/pypy/pypy/changeset/06b064a032d8/

Log:	Refactor FrameState

diff --git a/rpython/flowspace/flowcontext.py b/rpython/flowspace/flowcontext.py
--- a/rpython/flowspace/flowcontext.py
+++ b/rpython/flowspace/flowcontext.py
@@ -12,8 +12,7 @@
 from rpython.flowspace.argument import CallSpec
 from rpython.flowspace.model import (Constant, Variable, Block, Link,
     c_last_exception, const, FSException)
-from rpython.flowspace.framestate import (FrameState, recursively_unflatten,
-    recursively_flatten)
+from rpython.flowspace.framestate import FrameState
 from rpython.flowspace.specialcase import (rpython_print_item,
     rpython_print_newline)
 from rpython.flowspace.operation import op
@@ -342,28 +341,14 @@
         del self.stack[finaldepth:]
 
     def getstate(self, next_offset):
-        data = self.locals_w[:]
-        data.extend(self.stack)
-        if self.last_exception is None:
-            data.append(Constant(None))
-            data.append(Constant(None))
-        else:
-            data.append(self.last_exception.w_type)
-            data.append(self.last_exception.w_value)
-        recursively_flatten(data)
-        return FrameState(data, self.blockstack[:], next_offset)
+        return FrameState(self.locals_w[:], self.stack[:],
+                self.last_exception, self.blockstack[:], next_offset)
 
     def setstate(self, state):
         """ Reset the context to the given frame state. """
-        data = state.mergeable[:]
-        recursively_unflatten(data)
-        self.locals_w = data[:self.nlocals]
-        self.stack = data[self.nlocals:-2]
-        if data[-2] == Constant(None):
-            assert data[-1] == Constant(None)
-            self.last_exception = None
-        else:
-            self.last_exception = FSException(data[-2], data[-1])
+        self.locals_w = state.locals_w[:]
+        self.stack = state.stack[:]
+        self.last_exception = state.last_exception
         self.blockstack = state.blocklist[:]
         self._normalize_raise_signals()
 
diff --git a/rpython/flowspace/framestate.py b/rpython/flowspace/framestate.py
--- a/rpython/flowspace/framestate.py
+++ b/rpython/flowspace/framestate.py
@@ -1,21 +1,47 @@
-from rpython.flowspace.model import Variable, Constant
+from rpython.flowspace.model import Variable, Constant, FSException
 from rpython.rlib.unroll import SpecTag
 
+def _copy(v):
+    from rpython.flowspace.flowcontext import FlowSignal
+    if isinstance(v, Variable):
+        return Variable(v)
+    elif isinstance(v, FlowSignal):
+        vars = [_copy(var) for var in v.args]
+        return v.rebuild(*vars)
+    else:
+        return v
+
+def _union(seq1, seq2):
+    return [union(v1, v2) for v1, v2 in zip(seq1, seq2)]
+
 
 class FrameState(object):
-    def __init__(self, mergeable, blocklist, next_offset):
-        self.mergeable = mergeable
+    def __init__(self, locals_w, stack, last_exception, blocklist, next_offset):
+        self.locals_w = locals_w
+        self.stack = stack
+        self.last_exception = last_exception
         self.blocklist = blocklist
         self.next_offset = next_offset
 
+    @property
+    def mergeable(self):
+        data = self.locals_w + self.stack
+        if self.last_exception is None:
+            data.append(Constant(None))
+            data.append(Constant(None))
+        else:
+            data.append(self.last_exception.w_type)
+            data.append(self.last_exception.w_value)
+        recursively_flatten(data)
+        return data
+
     def copy(self):
         "Make a copy of this state in which all Variables are fresh."
-        newstate = []
-        for w in self.mergeable:
-            if isinstance(w, Variable):
-                w = Variable(w)
-            newstate.append(w)
-        return FrameState(newstate, self.blocklist, self.next_offset)
+        exc = self.last_exception
+        if exc is not None:
+            exc = FSException(_copy(exc.w_type), _copy(exc.w_value))
+        return FrameState(map(_copy, self.locals_w), map(_copy, self.stack),
+                exc, self.blocklist, self.next_offset)
 
     def getvariables(self):
         return [w for w in self.mergeable if isinstance(w, Variable)]
@@ -38,13 +64,21 @@
            A state 'a' is more general than a state 'b' if all Variables in 'b'
            are also Variables in 'a', but 'a' may have more Variables.
         """
-        newstate = []
         try:
-            for w1, w2 in zip(self.mergeable, other.mergeable):
-                newstate.append(union(w1, w2))
+            locals = _union(self.locals_w, other.locals_w)
+            stack = _union(self.stack, other.stack)
+            if self.last_exception is None:
+                exc = other.last_exception
+            elif other.last_exception is None:
+                exc = self.last_exception
+            else:
+                e1 = self.last_exception
+                e2 = other.last_exception
+                exc = FSException(union(e1.w_type, e2.w_type),
+                        union(e1.w_value, e2.w_value))
         except UnionError:
             return None
-        return FrameState(newstate, self.blocklist, self.next_offset)
+        return FrameState(locals, stack, exc, self.blocklist, self.next_offset)
 
     def getoutputargs(self, targetstate):
         "Return the output arguments needed to link self to targetstate."
@@ -61,6 +95,7 @@
 
 def union(w1, w2):
     "Union of two variables or constants."
+    from rpython.flowspace.flowcontext import FlowSignal
     if w1 == w2:
         return w1
     if w1 is None or w2 is None:
@@ -69,17 +104,17 @@
     if isinstance(w1, Variable) or isinstance(w2, Variable):
         return Variable()  # new fresh Variable
     if isinstance(w1, Constant) and isinstance(w2, Constant):
-        # FlowSignal represent stack unrollers in the stack.
-        # They should not be merged because they will be unwrapped.
-        # This is needed for try:except: and try:finally:, though
-        # it makes the control flow a bit larger by duplicating the
-        # handlers.
-        dont_merge_w1 = w1 in UNPICKLE_TAGS or isinstance(w1.value, SpecTag)
-        dont_merge_w2 = w2 in UNPICKLE_TAGS or isinstance(w2.value, SpecTag)
-        if dont_merge_w1 or dont_merge_w2:
+        if isinstance(w1.value, SpecTag) or isinstance(w2.value, SpecTag):
             raise UnionError
         else:
             return Variable()  # generalize different constants
+    if isinstance(w1, FlowSignal) and isinstance(w2, FlowSignal):
+        if type(w1) is not type(w2):
+            raise UnionError
+        vars = [union(v1, v2) for v1, v2 in zip(w1.args, w2.args)]
+        return w1.rebuild(*vars)
+    if isinstance(w1, FlowSignal) or isinstance(w2, FlowSignal):
+        raise UnionError
     raise TypeError('union of %r and %r' % (w1.__class__.__name__,
                                             w2.__class__.__name__))
 
diff --git a/rpython/flowspace/pygraph.py b/rpython/flowspace/pygraph.py
--- a/rpython/flowspace/pygraph.py
+++ b/rpython/flowspace/pygraph.py
@@ -11,10 +11,10 @@
 
     def __init__(self, func, code):
         from rpython.flowspace.flowcontext import SpamBlock
-        data = [None] * code.co_nlocals
+        locals = [None] * code.co_nlocals
         for i in range(code.formalargcount):
-            data[i] = Variable(code.co_varnames[i])
-        state = FrameState(data + [Constant(None), Constant(None)], [], 0)
+            locals[i] = Variable(code.co_varnames[i])
+        state = FrameState(locals, [], None, [], 0)
         initialblock = SpamBlock(state)
         super(PyGraph, self).__init__(self._sanitize_funcname(func), initialblock)
         self.func = func

From noreply at buildbot.pypy.org  Fri Nov 21 05:16:05 2014
From: noreply at buildbot.pypy.org (rlamy)
Date: Fri, 21 Nov 2014 05:16:05 +0100 (CET)
Subject: [pypy-commit] pypy framestate: fix exception merging
Message-ID: <20141121041605.8C9151D285E@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy 
Branch: framestate
Changeset: r74623:c7bd1fd3509e
Date: 2014-11-21 04:00 +0000
http://bitbucket.org/pypy/pypy/changeset/c7bd1fd3509e/

Log:	fix exception merging

diff --git a/rpython/flowspace/framestate.py b/rpython/flowspace/framestate.py
--- a/rpython/flowspace/framestate.py
+++ b/rpython/flowspace/framestate.py
@@ -59,6 +59,13 @@
                 return False
         return True
 
+    def _exc_args(self):
+        if self.last_exception is None:
+            return [Constant(None), Constant(None)]
+        else:
+            return [self.last_exception.w_type,
+                    self.last_exception.w_value]
+
     def union(self, other):
         """Compute a state that is at least as general as both self and other.
            A state 'a' is more general than a state 'b' if all Variables in 'b'
@@ -67,15 +74,13 @@
         try:
             locals = _union(self.locals_w, other.locals_w)
             stack = _union(self.stack, other.stack)
-            if self.last_exception is None:
-                exc = other.last_exception
-            elif other.last_exception is None:
-                exc = self.last_exception
+            if self.last_exception is None and other.last_exception is None:
+                exc = None
             else:
-                e1 = self.last_exception
-                e2 = other.last_exception
-                exc = FSException(union(e1.w_type, e2.w_type),
-                        union(e1.w_value, e2.w_value))
+                args1 = self._exc_args()
+                args2 = other._exc_args()
+                exc = FSException(union(args1[0], args2[0]),
+                        union(args1[1], args2[1]))
         except UnionError:
             return None
         return FrameState(locals, stack, exc, self.blocklist, self.next_offset)

From noreply at buildbot.pypy.org  Fri Nov 21 05:16:06 2014
From: noreply at buildbot.pypy.org (rlamy)
Date: Fri, 21 Nov 2014 05:16:06 +0100 (CET)
Subject: [pypy-commit] pypy framestate: Remove unnecessary complications in
	framestate.py
Message-ID: <20141121041606.BCB0F1D285E@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy 
Branch: framestate
Changeset: r74624:5336be0ae46c
Date: 2014-11-21 04:05 +0000
http://bitbucket.org/pypy/pypy/changeset/5336be0ae46c/

Log:	Remove unnecessary complications in framestate.py

diff --git a/rpython/flowspace/flowcontext.py b/rpython/flowspace/flowcontext.py
--- a/rpython/flowspace/flowcontext.py
+++ b/rpython/flowspace/flowcontext.py
@@ -468,7 +468,7 @@
 
         newblock = SpamBlock(newstate)
         varnames = self.pycode.co_varnames
-        for name, w_value in zip(varnames, newstate.mergeable):
+        for name, w_value in zip(varnames, newstate.locals_w):
             if isinstance(w_value, Variable):
                 w_value.rename(name)
         # unconditionally link the current block to the newblock
diff --git a/rpython/flowspace/framestate.py b/rpython/flowspace/framestate.py
--- a/rpython/flowspace/framestate.py
+++ b/rpython/flowspace/framestate.py
@@ -124,23 +124,6 @@
                                             w2.__class__.__name__))
 
 
-# ____________________________________________________________
-#
-# We have to flatten out the state of the frame into a list of
-# Variables and Constants.  This is done above by collecting the
-# locals and the items on the value stack, but the latter may contain
-# FlowSignal.  We have to handle these specially, because
-# some of them hide references to more Variables and Constants.
-# The trick is to flatten ("pickle") them into the list so that the
-# extra Variables show up directly in the list too.
-
-class PickleTag:
-    pass
-
-PICKLE_TAGS = {}
-UNPICKLE_TAGS = {}
-
-
 def recursively_flatten(lst):
     from rpython.flowspace.flowcontext import FlowSignal
     i = 0
@@ -149,22 +132,4 @@
         if not isinstance(unroller, FlowSignal):
             i += 1
         else:
-            vars = unroller.args
-            key = unroller.__class__, len(vars)
-            try:
-                tag = PICKLE_TAGS[key]
-            except KeyError:
-                tag = PICKLE_TAGS[key] = Constant(PickleTag())
-                UNPICKLE_TAGS[tag] = key
-            lst[i:i + 1] = [tag] + vars
-
-
-def recursively_unflatten(lst):
-    for i in xrange(len(lst) - 1, -1, -1):
-        item = lst[i]
-        if item in UNPICKLE_TAGS:
-            unrollerclass, argcount = UNPICKLE_TAGS[item]
-            arguments = lst[i + 1:i + 1 + argcount]
-            del lst[i + 1:i + 1 + argcount]
-            unroller = unrollerclass.rebuild(*arguments)
-            lst[i] = unroller
+            lst[i:i + 1] = unroller.args

From noreply at buildbot.pypy.org  Fri Nov 21 08:03:34 2014
From: noreply at buildbot.pypy.org (fijal)
Date: Fri, 21 Nov 2014 08:03:34 +0100 (CET)
Subject: [pypy-commit] pypy optresult: reinstantiate cloning
Message-ID: <20141121070334.103D41D281E@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: optresult
Changeset: r74625:486e9a88f4d6
Date: 2014-11-19 13:15 +0200
http://bitbucket.org/pypy/pypy/changeset/486e9a88f4d6/

Log:	reinstantiate cloning

diff --git a/rpython/jit/metainterp/compile.py b/rpython/jit/metainterp/compile.py
--- a/rpython/jit/metainterp/compile.py
+++ b/rpython/jit/metainterp/compile.py
@@ -130,7 +130,7 @@
     h_ops = history.operations
     # XXX why do we clone here?
     part.operations = [ResOperation(rop.LABEL, inputargs, descr=TargetToken(jitcell_token))] + \
-                      [h_ops[i] for i in range(start, len(h_ops))] + \
+                      [h_ops[i].clone() for i in range(start, len(h_ops))] + \
                       [ResOperation(rop.LABEL, jumpargs, descr=jitcell_token)]
 
     try:
@@ -206,10 +206,10 @@
     h_ops = history.operations
 
     part.operations = [partial_trace.operations[-1]] + \
-                      [h_ops[i] for i in range(start, len(h_ops))] + \
+                      [h_ops[i].clone() for i in range(start, len(h_ops))] + \
                       [ResOperation(rop.JUMP, jumpargs, None, descr=loop_jitcell_token)]
     label = part.operations[0]
-    orignial_label = label
+    orignial_label = label.clone()
     assert label.getopnum() == rop.LABEL
     try:
         optimize_trace(metainterp_sd, part, jitdriver_sd.warmstate.enable_opts)
@@ -784,7 +784,7 @@
     new_trace.inputargs = metainterp.history.inputargs[:]
     # clone ops, as optimize_bridge can mutate the ops
 
-    new_trace.operations = [op for op in metainterp.history.operations]
+    new_trace.operations = [op.clone() for op in metainterp.history.operations]
     metainterp_sd = metainterp.staticdata
     state = metainterp.jitdriver_sd.warmstate
     if isinstance(resumekey, ResumeAtPositionDescr):

From noreply at buildbot.pypy.org  Fri Nov 21 08:03:35 2014
From: noreply at buildbot.pypy.org (fijal)
Date: Fri, 21 Nov 2014 08:03:35 +0100 (CET)
Subject: [pypy-commit] pypy optresult: clone enough to pass the first test
Message-ID: <20141121070335.3FE951D281E@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: optresult
Changeset: r74626:5bd127267e31
Date: 2014-11-19 14:45 +0200
http://bitbucket.org/pypy/pypy/changeset/5bd127267e31/

Log:	clone enough to pass the first test

diff --git a/rpython/jit/metainterp/compile.py b/rpython/jit/metainterp/compile.py
--- a/rpython/jit/metainterp/compile.py
+++ b/rpython/jit/metainterp/compile.py
@@ -7,7 +7,6 @@
 from rpython.rlib import rstack
 from rpython.rlib.jit import JitDebugInfo, Counters, dont_look_inside
 from rpython.conftest import option
-from rpython.tool.sourcetools import func_with_new_name
 
 from rpython.jit.metainterp.resoperation import ResOperation, rop, get_deep_immutable_oplist
 from rpython.jit.metainterp.history import (TreeLoop, Const, JitCellToken,
@@ -104,6 +103,17 @@
 
 # ____________________________________________________________
 
+class Memo(object):
+    def __init__(self):
+        self.snapshots = {}
+        self.box_mapping = {}
+
+    def get(self, box, default):
+        return self.box_mapping.get(box, default)
+
+    def set(self, box, newbox):
+        self.box_mapping[box] = newbox
+
 def compile_loop(metainterp, greenkey, start,
                  inputargs, jumpargs,
                  full_preamble_needed=True,
@@ -128,10 +138,11 @@
     part = create_empty_loop(metainterp)
     part.inputargs = inputargs[:]
     h_ops = history.operations
-    # XXX why do we clone here?
+    memo = Memo()
     part.operations = [ResOperation(rop.LABEL, inputargs, descr=TargetToken(jitcell_token))] + \
-                      [h_ops[i].clone() for i in range(start, len(h_ops))] + \
-                      [ResOperation(rop.LABEL, jumpargs, descr=jitcell_token)]
+                      [h_ops[i].clone(memo) for i in range(start, len(h_ops))]
+    jumpargs = [memo.get(box, box) for box in jumpargs]
+    part.operations.append(ResOperation(rop.LABEL, jumpargs, descr=jitcell_token))
 
     try:
         optimize_trace(metainterp_sd, part, enable_opts)
@@ -191,6 +202,7 @@
     """Try to compile a new procedure by closing the current history back
     to the first operation.
     """
+    xxx
     from rpython.jit.metainterp.optimizeopt import optimize_trace
 
     history = metainterp.history
@@ -633,9 +645,9 @@
                                self, inputargs, new_loop.operations,
                                new_loop.original_jitcell_token)
 
-    def copy_all_attributes_into(self, res):
+    def copy_all_attributes_into(self, res, memo):
         # XXX a bit ugly to have to list them all here
-        res.rd_snapshot = self.rd_snapshot
+        res.rd_snapshot = self.rd_snapshot.copy(memo)
         res.rd_frame_info_list = self.rd_frame_info_list
         res.rd_numb = self.rd_numb
         res.rd_consts = self.rd_consts
@@ -643,21 +655,21 @@
         res.rd_pendingfields = self.rd_pendingfields
         res.rd_count = self.rd_count
 
-    def _clone_if_mutable(self):
+    def _clone_if_mutable(self, memo):
         res = ResumeGuardDescr()
-        self.copy_all_attributes_into(res)
+        self.copy_all_attributes_into(res, memo)
         return res
 
 class ResumeGuardNotInvalidated(ResumeGuardDescr):
-    def _clone_if_mutable(self):
+    def _clone_if_mutable(self, memo):
         res = ResumeGuardNotInvalidated()
-        self.copy_all_attributes_into(res)
+        self.copy_all_attributes_into(res, memo)
         return res
 
 class ResumeAtPositionDescr(ResumeGuardDescr):
-    def _clone_if_mutable(self):
+    def _clone_if_mutable(self, memo):
         res = ResumeAtPositionDescr()
-        self.copy_all_attributes_into(res)
+        self.copy_all_attributes_into(res, memo)
         return res
 
 class AllVirtuals:
@@ -741,10 +753,10 @@
         hidden_all_virtuals = obj.hide(metainterp_sd.cpu)
         metainterp_sd.cpu.set_savedata_ref(deadframe, hidden_all_virtuals)
 
-    def _clone_if_mutable(self):
+    def _clone_if_mutable(self, memo):
         res = ResumeGuardForcedDescr(self.metainterp_sd,
                                      self.jitdriver_sd)
-        self.copy_all_attributes_into(res)
+        self.copy_all_attributes_into(res, memo)
         return res
 
 
@@ -773,6 +785,7 @@
     """Try to compile a new bridge leading from the beginning of the history
     to some existing place.
     """
+    xxx
     from rpython.jit.metainterp.optimizeopt import optimize_trace
 
     # The history contains new operations to attach as the code for the
diff --git a/rpython/jit/metainterp/history.py b/rpython/jit/metainterp/history.py
--- a/rpython/jit/metainterp/history.py
+++ b/rpython/jit/metainterp/history.py
@@ -130,10 +130,10 @@
     def repr_of_descr(self):
         return '%r' % (self,)
 
-    def _clone_if_mutable(self):
+    def _clone_if_mutable(self, memo):
         return self
-    def clone_if_mutable(self):
-        clone = self._clone_if_mutable()
+    def clone_if_mutable(self, memo):
+        clone = self._clone_if_mutable(memo)
         if not we_are_translated():
             assert clone.__class__ is self.__class__
         return clone
diff --git a/rpython/jit/metainterp/resoperation.py b/rpython/jit/metainterp/resoperation.py
--- a/rpython/jit/metainterp/resoperation.py
+++ b/rpython/jit/metainterp/resoperation.py
@@ -109,16 +109,17 @@
             assert lltype.typeOf(value) == llmemory.GCREF
             self._resref = value
 
-
-    def clone(self):
-        args = self.getarglist()
+            
+    def clone(self, memo):
+        args = [memo.get(arg, arg) for arg in self.getarglist()]
         descr = self.getdescr()
         if descr is not None:
-            descr = descr.clone_if_mutable()
-        op = ResOperation(self.getopnum(), args[:], descr)
+            descr = descr.clone_if_mutable(memo)
+        op = ResOperation(self.getopnum(), args, descr)
         if not we_are_translated():
             op.name = self.name
             op.pc = self.pc
+        memo.set(self, op)
         return op
 
     def repr(self, memo, graytext=False):
@@ -275,9 +276,11 @@
         newop.setfailargs(self.getfailargs())
         return newop
 
-    def clone(self):
-        newop = AbstractResOp.clone(self)
-        newop.setfailargs(self.getfailargs())
+    def clone(self, memo):
+        newop = AbstractResOp.clone(self, memo)
+        failargs = self.getfailargs()
+        if failargs is not None:
+            newop.setfailargs([memo.get(arg, arg) for arg in failargs])
         return newop
 
 # ===========
diff --git a/rpython/jit/metainterp/resume.py b/rpython/jit/metainterp/resume.py
--- a/rpython/jit/metainterp/resume.py
+++ b/rpython/jit/metainterp/resume.py
@@ -29,6 +29,19 @@
         self.prev = prev
         self.boxes = boxes
 
+    def copy(self, memo):
+        try:
+            return memo.snapshots[self]
+        except KeyError:
+            if self.prev is not None:
+                prev = self.prev.copy(memo)
+            else:
+                prev = None
+            boxes = [memo.get(box, box) for box in self.boxes]
+            new_snapshot = Snapshot(prev, boxes)
+            memo.snapshots[self] = new_snapshot
+            return new_snapshot
+
 class FrameInfo(object):
     __slots__ = ('prev', 'jitcode', 'pc')
 

From noreply at buildbot.pypy.org  Fri Nov 21 08:03:36 2014
From: noreply at buildbot.pypy.org (fijal)
Date: Fri, 21 Nov 2014 08:03:36 +0100 (CET)
Subject: [pypy-commit] pypy default: Kill obmalloc. Shockingly everything
	nicely passes
Message-ID: <20141121070336.8276E1D281E@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: 
Changeset: r74627:7b11194ab1d4
Date: 2014-11-21 09:02 +0200
http://bitbucket.org/pypy/pypy/changeset/7b11194ab1d4/

Log:	Kill obmalloc. Shockingly everything nicely passes

diff --git a/rpython/translator/c/genc.py b/rpython/translator/c/genc.py
--- a/rpython/translator/c/genc.py
+++ b/rpython/translator/c/genc.py
@@ -405,8 +405,6 @@
             ('debug', '', '$(MAKE) CFLAGS="$(DEBUGFLAGS) -DRPY_ASSERT" debug_target'),
             ('debug_exc', '', '$(MAKE) CFLAGS="$(DEBUGFLAGS) -DRPY_ASSERT -DDO_LOG_EXC" debug_target'),
             ('debug_mem', '', '$(MAKE) CFLAGS="$(DEBUGFLAGS) -DRPY_ASSERT -DPYPY_USE_TRIVIAL_MALLOC" debug_target'),
-            ('no_obmalloc', '', '$(MAKE) CFLAGS="-g -O2 -DRPY_ASSERT -DPYPY_NO_OBMALLOC" $(TARGET)'),
-            ('linuxmemchk', '', '$(MAKE) CFLAGS="$(DEBUGFLAGS) -DRPY_ASSERT -DPYPY_USE_LINUXMEMCHK" debug_target'),
             ('llsafer', '', '$(MAKE) CFLAGS="-O2 -DRPY_LL_ASSERT" $(TARGET)'),
             ('lldebug', '', '$(MAKE) CFLAGS="$(DEBUGFLAGS) -DRPY_ASSERT -DRPY_LL_ASSERT" debug_target'),
             ('lldebug0','', '$(MAKE) CFLAGS="$(DEBUGFLAGS) -O0 -DRPY_ASSERT -DRPY_LL_ASSERT" debug_target'),
@@ -762,7 +760,6 @@
     srcdir = py.path.local(__file__).join('..', 'src')
     files = [
         srcdir / 'entrypoint.c',       # ifdef PYPY_STANDALONE
-        srcdir / 'allocator.c',        # ifdef PYPY_STANDALONE
         srcdir / 'mem.c',
         srcdir / 'exception.c',
         srcdir / 'rtyper.c',           # ifdef HAVE_RTYPER
diff --git a/rpython/translator/c/src/allocator.c b/rpython/translator/c/src/allocator.c
deleted file mode 100644
--- a/rpython/translator/c/src/allocator.c
+++ /dev/null
@@ -1,33 +0,0 @@
-/* allocation functions */
-#include "common_header.h"
-#ifdef PYPY_STANDALONE
-#include 
-
-#if defined(PYPY_USE_TRIVIAL_MALLOC)
-  void *PyObject_Malloc(size_t n) { return malloc(n); }
-  void *PyObject_Realloc(void *p, size_t n) { return realloc(p, n); }
-  void PyObject_Free(void *p) { if (p) { *((int*)p) = 0xDDDDDDDD; } free(p); }
-
-#elif defined(PYPY_USE_LINUXMEMCHK)
-#  include "linuxmemchk.c"
-
-#elif defined(PYPY_NO_OBMALLOC)
-  void *PyObject_Malloc(size_t n) { return malloc(n); }
-  void *PyObject_Realloc(void *p, size_t n) { return realloc(p, n); }
-  void PyObject_Free(void *p) { free(p); }
-
-#else
-#  ifndef WITH_PYMALLOC
-#    define WITH_PYMALLOC
-#  endif
-/* The same obmalloc as CPython */
-#  include "src/obmalloc.c"
-
-#endif
-#elif defined _MSC_VER
-/* link will fail without some kind of definition for the functions */  
-  void *PyObject_Malloc(size_t n) { return NULL; }
-  void *PyObject_Realloc(void *p, size_t n) { return NULL; }
-  void PyObject_Free(void *p) {  }
-
-#endif  /* PYPY_STANDALONE */
diff --git a/rpython/translator/c/src/allocator.h b/rpython/translator/c/src/allocator.h
deleted file mode 100644
--- a/rpython/translator/c/src/allocator.h
+++ /dev/null
@@ -1,4 +0,0 @@
-/* allocation functions prototypes */
-RPY_EXTERN void *PyObject_Malloc(size_t n);
-RPY_EXTERN void *PyObject_Realloc(void *p, size_t n);
-RPY_EXTERN void PyObject_Free(void *p);
diff --git a/rpython/translator/c/src/g_include.h b/rpython/translator/c/src/g_include.h
--- a/rpython/translator/c/src/g_include.h
+++ b/rpython/translator/c/src/g_include.h
@@ -33,7 +33,6 @@
 #  include "src/debug_traceback.h"
 #endif
 
-#  include "src/allocator.h"
 #ifdef PYPY_STANDALONE
 #  include "src/entrypoint.h"
 #endif
diff --git a/rpython/translator/c/src/linuxmemchk.c b/rpython/translator/c/src/linuxmemchk.c
deleted file mode 100644
--- a/rpython/translator/c/src/linuxmemchk.c
+++ /dev/null
@@ -1,101 +0,0 @@
-/* custom checking allocators a la Electric Fence */
-#include 
-#include 
-#include 
-#include 
-#include 
-#include 
-#define PAGESIZE 4096
-#ifndef MALLOC_BIGBUFFER
-# define MALLOC_BIGBUFFER   (PAGESIZE*32768)   /* 128MB */
-#endif
-
-
-struct _alloc_s {
-  void* ptr;
-  int npages;
-};
-static void* _na_start = NULL;
-static char* _na_cur;
-
-static void _na_assert(int x, char* msg)
-{
-  if (!x)
-    {
-      fprintf(stderr, "linuxmemchk: failed assertion: %s\n", msg);
-      abort();
-    }
-}
-
-static struct _alloc_s* _na_find(void* data)
-{
-  int err;
-  long data1;
-  struct _alloc_s* s;
-  _na_assert(_na_start+PAGESIZE <= data &&
-             data < _na_start+MALLOC_BIGBUFFER-PAGESIZE,
-             "corrupted na_start");
-  data1 = (long) data;
-  data1 &= ~(PAGESIZE-1);
-  data1 -= PAGESIZE;
-  err = mprotect((void*) data1, PAGESIZE, PROT_READ|PROT_WRITE);
-  _na_assert(!err, "mprotect[1] failed");
-  s = (struct _alloc_s*) data1;
-  _na_assert(s->npages > 0, "corrupted s->npages");
-  return s;
-}
-
-void* PyObject_Malloc(size_t size)
-{
-  int err, npages = (size + PAGESIZE-1) / PAGESIZE + 1;
-  struct _alloc_s* s;
-  char* data;
-  if (_na_start == NULL)
-    {
-      _na_start = mmap(NULL, MALLOC_BIGBUFFER, PROT_NONE,
-                       MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
-      _na_assert(_na_start != MAP_FAILED, "initial mmap failed");
-      _na_cur = (char*) _na_start;
-    }
-  s = (struct _alloc_s*) _na_cur;
-  _na_cur += npages * PAGESIZE;
-  if (_na_cur >= ((char*) _na_start) + MALLOC_BIGBUFFER)
-    {
-      fprintf(stderr, "linuxmemchk.c: Nothing wrong so far, but we are running out\nlinuxmemchk.c: of mmap'ed memory.  Increase MALLOC_BIGBUFFER.\n");
-      abort();
-    }
-  err = mprotect(s, npages * PAGESIZE, PROT_READ|PROT_WRITE|PROT_EXEC);
-  _na_assert(!err, "mprotect[2] failed");
-  s->ptr = data = _na_cur - /*((size+3)&~3)*/ size;
-  s->npages = npages;
-  err = mprotect(s, PAGESIZE, PROT_NONE);
-  _na_assert(!err, "mprotect[3] failed");
-  return data;
-}
-
-void PyObject_Free(void* data)
-{
-  int err, npages;
-  struct _alloc_s* s;
-  if (data == NULL)
-    return;
-  s = _na_find(data);
-  _na_assert(s->ptr == data, "free got a pointer not returned by malloc");
-  npages = s->npages;
-  s->npages = 0;
-  err = mprotect(s, npages * PAGESIZE, PROT_NONE);
-  _na_assert(!err, "mprotect[4] failed");
-}
-
-void* PyObject_Realloc(void* data, size_t nsize)
-{
-  size_t size;
-  struct _alloc_s* s = _na_find(data);
-  void* ndata = PyObject_Malloc(nsize);
-
-  _na_assert(s->ptr == data, "realloc got a pointer not returned by malloc");
-  size = ((char*)s) + s->npages * PAGESIZE - (char*)data;
-  memcpy(ndata, data, size
-#include 
-
-/* An object allocator for Python.
-
-   Here is an introduction to the layers of the Python memory architecture,
-   showing where the object allocator is actually used (layer +2), It is
-   called for every object allocation and deallocation (PyObject_New/Del),
-   unless the object-specific allocators implement a proprietary allocation
-   scheme (ex.: ints use a simple free list). This is also the place where
-   the cyclic garbage collector operates selectively on container objects.
-
-
-        Object-specific allocators
-    _____   ______   ______       ________
-   [ int ] [ dict ] [ list ] ... [ string ]       Python core         |
-+3 | <----- Object-specific memory -----> | <-- Non-object memory --> |
-    _______________________________       |                           |
-   [   Python's object allocator   ]      |                           |
-+2 | ####### Object memory ####### | <------ Internal buffers ------> |
-    ______________________________________________________________    |
-   [          Python's raw memory allocator (PyMem_ API)          ]   |
-+1 | <----- Python memory (under PyMem manager's control) ------> |   |
-    __________________________________________________________________
-   [    Underlying general-purpose allocator (ex: C library malloc)   ]
- 0 | <------ Virtual memory allocated for the python process -------> |
-
-   =========================================================================
-    _______________________________________________________________________
-   [                OS-specific Virtual Memory Manager (VMM)               ]
--1 | <--- Kernel dynamic storage allocation & management (page-based) ---> |
-    __________________________________   __________________________________
-   [                                  ] [                                  ]
--2 | <-- Physical memory: ROM/RAM --> | | <-- Secondary storage (swap) --> |
-
-*/
-/*==========================================================================*/
-
-/* A fast, special-purpose memory allocator for small blocks, to be used
-   on top of a general-purpose malloc -- heavily based on previous art. */
-
-/* Vladimir Marangozov -- August 2000 */
-
-/*
- * "Memory management is where the rubber meets the road -- if we do the wrong
- * thing at any level, the results will not be good. And if we don't make the
- * levels work well together, we are in serious trouble." (1)
- *
- * (1) Paul R. Wilson, Mark S. Johnstone, Michael Neely, and David Boles,
- *    "Dynamic Storage Allocation: A Survey and Critical Review",
- *    in Proc. 1995 Int'l. Workshop on Memory Management, September 1995.
- */
-
-/* #undef WITH_MEMORY_LIMITS */		/* disable mem limit checks  */
-
-/*==========================================================================*/
-
-/*
- * Allocation strategy abstract:
- *
- * For small requests, the allocator sub-allocates  blocks of memory.
- * Requests greater than 256 bytes are routed to the system's allocator.
- *
- * Small requests are grouped in size classes spaced 8 bytes apart, due
- * to the required valid alignment of the returned address. Requests of
- * a particular size are serviced from memory pools of 4K (one VMM page).
- * Pools are fragmented on demand and contain free lists of blocks of one
- * particular size class. In other words, there is a fixed-size allocator
- * for each size class. Free pools are shared by the different allocators
- * thus minimizing the space reserved for a particular size class.
- *
- * This allocation strategy is a variant of what is known as "simple
- * segregated storage based on array of free lists". The main drawback of
- * simple segregated storage is that we might end up with lot of reserved
- * memory for the different free lists, which degenerate in time. To avoid
- * this, we partition each free list in pools and we share dynamically the
- * reserved space between all free lists. This technique is quite efficient
- * for memory intensive programs which allocate mainly small-sized blocks.
- *
- * For small requests we have the following table:
- *
- * Request in bytes	Size of allocated block      Size class idx
- * ----------------------------------------------------------------
- *        1-8                     8                       0
- *	  9-16                   16                       1
- *	 17-24                   24                       2
- *	 25-32                   32                       3
- *	 33-40                   40                       4
- *	 41-48                   48                       5
- *	 49-56                   56                       6
- *	 57-64                   64                       7
- *	 65-72                   72                       8
- *	  ...                   ...                     ...
- *	241-248                 248                      30
- *	249-256                 256                      31
- *
- *	0, 257 and up: routed to the underlying allocator.
- */
-
-/*==========================================================================*/
-
-/*
- * -- Main tunable settings section --
- */
-
-/*
- * Alignment of addresses returned to the user. 8-bytes alignment works
- * on most current architectures (with 32-bit or 64-bit address busses).
- * The alignment value is also used for grouping small requests in size
- * classes spaced ALIGNMENT bytes apart.
- *
- * You shouldn't change this unless you know what you are doing.
- */
-#define ALIGNMENT		8		/* must be 2^N */
-#define ALIGNMENT_SHIFT		3
-#define ALIGNMENT_MASK		(ALIGNMENT - 1)
-
-/* Return the number of bytes in size class I, as a uint. */
-#define INDEX2SIZE(I) (((uint)(I) + 1) << ALIGNMENT_SHIFT)
-
-/*
- * Max size threshold below which malloc requests are considered to be
- * small enough in order to use preallocated memory pools. You can tune
- * this value according to your application behaviour and memory needs.
- *
- * The following invariants must hold:
- *	1) ALIGNMENT <= SMALL_REQUEST_THRESHOLD <= 256
- *	2) SMALL_REQUEST_THRESHOLD is evenly divisible by ALIGNMENT
- *
- * Although not required, for better performance and space efficiency,
- * it is recommended that SMALL_REQUEST_THRESHOLD is set to a power of 2.
- */
-#define SMALL_REQUEST_THRESHOLD	256
-#define NB_SMALL_SIZE_CLASSES	(SMALL_REQUEST_THRESHOLD / ALIGNMENT)
-
-/*
- * The system's VMM page size can be obtained on most unices with a
- * getpagesize() call or deduced from various header files. To make
- * things simpler, we assume that it is 4K, which is OK for most systems.
- * It is probably better if this is the native page size, but it doesn't
- * have to be.  In theory, if SYSTEM_PAGE_SIZE is larger than the native page
- * size, then `POOL_ADDR(p)->arenaindex' could rarely cause a segmentation
- * violation fault.  4K is apparently OK for all the platforms that python
- * currently targets.
- */
-#define SYSTEM_PAGE_SIZE	(4 * 1024)
-#define SYSTEM_PAGE_SIZE_MASK	(SYSTEM_PAGE_SIZE - 1)
-
-/*
- * Maximum amount of memory managed by the allocator for small requests.
- */
-#ifdef WITH_MEMORY_LIMITS
-#ifndef SMALL_MEMORY_LIMIT
-#define SMALL_MEMORY_LIMIT	(64 * 1024 * 1024)	/* 64 MB -- more? */
-#endif
-#endif
-
-/*
- * The allocator sub-allocates  blocks of memory (called arenas) aligned
- * on a page boundary. This is a reserved virtual address space for the
- * current process (obtained through a malloc call). In no way this means
- * that the memory arenas will be used entirely. A malloc() is usually
- * an address range reservation for  bytes, unless all pages within this
- * space are referenced subsequently. So malloc'ing big blocks and not using
- * them does not mean "wasting memory". It's an addressable range wastage...
- *
- * Therefore, allocating arenas with malloc is not optimal, because there is
- * some address space wastage, but this is the most portable way to request
- * memory from the system across various platforms.
- */
-#define ARENA_SIZE		(256 << 10)	/* 256KB */
-
-#ifdef WITH_MEMORY_LIMITS
-#define MAX_ARENAS		(SMALL_MEMORY_LIMIT / ARENA_SIZE)
-#endif
-
-/*
- * Size of the pools used for small blocks. Should be a power of 2,
- * between 1K and SYSTEM_PAGE_SIZE, that is: 1k, 2k, 4k.
- */
-#define POOL_SIZE		SYSTEM_PAGE_SIZE	/* must be 2^N */
-#define POOL_SIZE_MASK		SYSTEM_PAGE_SIZE_MASK
-
-/*
- * -- End of tunable settings section --
- */
-
-/*==========================================================================*/
-
-/*
- * Locking
- *
- * To reduce lock contention, it would probably be better to refine the
- * crude function locking with per size class locking. I'm not positive
- * however, whether it's worth switching to such locking policy because
- * of the performance penalty it might introduce.
- *
- * The following macros describe the simplest (should also be the fastest)
- * lock object on a particular platform and the init/fini/lock/unlock
- * operations on it. The locks defined here are not expected to be recursive
- * because it is assumed that they will always be called in the order:
- * INIT, [LOCK, UNLOCK]*, FINI.
- */
-
-/*
- * Python's threads are serialized, so object malloc locking is disabled.
- */
-#define SIMPLELOCK_DECL(lock)	/* simple lock declaration		*/
-#define SIMPLELOCK_INIT(lock)	/* allocate (if needed) and initialize	*/
-#define SIMPLELOCK_FINI(lock)	/* free/destroy an existing lock 	*/
-#define SIMPLELOCK_LOCK(lock)	/* acquire released lock */
-#define SIMPLELOCK_UNLOCK(lock)	/* release acquired lock */
-
-/*
- * Basic types
- * I don't care if these are defined in  or elsewhere. Axiom.
- */
-#undef  uchar
-#define uchar			unsigned char	/* assuming == 8 bits  */
-
-#undef  uint
-#define uint			unsigned int	/* assuming >= 16 bits */
-
-#undef  ulong
-#define ulong			Unsigned	/* assuming >= 32 bits */
-
-#undef uptr
-#define uptr			Unsigned
-
-/* When you say memory, my mind reasons in terms of (pointers to) blocks */
-typedef uchar block;
-
-/* Pool for small blocks. */
-struct pool_header {
-	union { block *_padding;
-		uint count; } ref;	/* number of allocated blocks    */
-	block *freeblock;		/* pool's free list head         */
-	struct pool_header *nextpool;	/* next pool of this size class  */
-	struct pool_header *prevpool;	/* previous pool       ""        */
-	uint arenaindex;		/* index into arenas of base adr */
-	uint szidx;			/* block size class index	 */
-	uint nextoffset;		/* bytes to virgin block	 */
-	uint maxnextoffset;		/* largest valid nextoffset	 */
-};
-
-typedef struct pool_header *poolp;
-
-#undef  ROUNDUP
-#define ROUNDUP(x)		(((x) + ALIGNMENT_MASK) & ~ALIGNMENT_MASK)
-#define POOL_OVERHEAD		ROUNDUP(sizeof(struct pool_header))
-
-#define DUMMY_SIZE_IDX		0xffff	/* size class of newly cached pools */
-
-/* Round pointer P down to the closest pool-aligned address <= P, as a poolp */
-#define POOL_ADDR(P) ((poolp)((uptr)(P) & ~(uptr)POOL_SIZE_MASK))
-
-/* Return total number of blocks in pool of size index I, as a uint. */
-#define NUMBLOCKS(I) ((uint)(POOL_SIZE - POOL_OVERHEAD) / INDEX2SIZE(I))
-
-/*==========================================================================*/
-
-/*
- * This malloc lock
- */
-SIMPLELOCK_DECL(_malloc_lock)
-#define LOCK()		SIMPLELOCK_LOCK(_malloc_lock)
-#define UNLOCK()	SIMPLELOCK_UNLOCK(_malloc_lock)
-#define LOCK_INIT()	SIMPLELOCK_INIT(_malloc_lock)
-#define LOCK_FINI()	SIMPLELOCK_FINI(_malloc_lock)
-
-/*
- * Pool table -- headed, circular, doubly-linked lists of partially used pools.
-
-This is involved.  For an index i, usedpools[i+i] is the header for a list of
-all partially used pools holding small blocks with "size class idx" i. So
-usedpools[0] corresponds to blocks of size 8, usedpools[2] to blocks of size
-16, and so on:  index 2*i <-> blocks of size (i+1)<freeblock points to
-the start of a singly-linked list of free blocks within the pool.  When a
-block is freed, it's inserted at the front of its pool's freeblock list.  Note
-that the available blocks in a pool are *not* linked all together when a pool
-is initialized.  Instead only "the first two" (lowest addresses) blocks are
-set up, returning the first such block, and setting pool->freeblock to a
-one-block list holding the second such block.  This is consistent with that
-pymalloc strives at all levels (arena, pool, and block) never to touch a piece
-of memory until it's actually needed.
-
-So long as a pool is in the used state, we're certain there *is* a block
-available for allocating, and pool->freeblock is not NULL.  If pool->freeblock
-points to the end of the free list before we've carved the entire pool into
-blocks, that means we simply haven't yet gotten to one of the higher-address
-blocks.  The offset from the pool_header to the start of "the next" virgin
-block is stored in the pool_header nextoffset member, and the largest value
-of nextoffset that makes sense is stored in the maxnextoffset member when a
-pool is initialized.  All the blocks in a pool have been passed out at least
-once when and only when nextoffset > maxnextoffset.
-
-
-Major obscurity:  While the usedpools vector is declared to have poolp
-entries, it doesn't really.  It really contains two pointers per (conceptual)
-poolp entry, the nextpool and prevpool members of a pool_header.  The
-excruciating initialization code below fools C so that
-
-    usedpool[i+i]
-
-"acts like" a genuine poolp, but only so long as you only reference its
-nextpool and prevpool members.  The "- 2*sizeof(block *)" gibberish is
-compensating for that a pool_header's nextpool and prevpool members
-immediately follow a pool_header's first two members:
-
-	union { block *_padding;
-		uint count; } ref;
-	block *freeblock;
-
-each of which consume sizeof(block *) bytes.  So what usedpools[i+i] really
-contains is a fudged-up pointer p such that *if* C believes it's a poolp
-pointer, then p->nextpool and p->prevpool are both p (meaning that the headed
-circular list is empty).
-
-It's unclear why the usedpools setup is so convoluted.  It could be to
-minimize the amount of cache required to hold this heavily-referenced table
-(which only *needs* the two interpool pointer members of a pool_header). OTOH,
-referencing code has to remember to "double the index" and doing so isn't
-free, usedpools[0] isn't a strictly legal pointer, and we're crucially relying
-on that C doesn't insert any padding anywhere in a pool_header at or before
-the prevpool member.
-**************************************************************************** */
-
-#define PTA(x)	((poolp )((uchar *)&(usedpools[2*(x)]) - 2*sizeof(block *)))
-#define PT(x)	PTA(x), PTA(x)
-
-static poolp usedpools[2 * ((NB_SMALL_SIZE_CLASSES + 7) / 8) * 8] = {
-	PT(0), PT(1), PT(2), PT(3), PT(4), PT(5), PT(6), PT(7)
-#if NB_SMALL_SIZE_CLASSES > 8
-	, PT(8), PT(9), PT(10), PT(11), PT(12), PT(13), PT(14), PT(15)
-#if NB_SMALL_SIZE_CLASSES > 16
-	, PT(16), PT(17), PT(18), PT(19), PT(20), PT(21), PT(22), PT(23)
-#if NB_SMALL_SIZE_CLASSES > 24
-	, PT(24), PT(25), PT(26), PT(27), PT(28), PT(29), PT(30), PT(31)
-#if NB_SMALL_SIZE_CLASSES > 32
-	, PT(32), PT(33), PT(34), PT(35), PT(36), PT(37), PT(38), PT(39)
-#if NB_SMALL_SIZE_CLASSES > 40
-	, PT(40), PT(41), PT(42), PT(43), PT(44), PT(45), PT(46), PT(47)
-#if NB_SMALL_SIZE_CLASSES > 48
-	, PT(48), PT(49), PT(50), PT(51), PT(52), PT(53), PT(54), PT(55)
-#if NB_SMALL_SIZE_CLASSES > 56
-	, PT(56), PT(57), PT(58), PT(59), PT(60), PT(61), PT(62), PT(63)
-#endif /* NB_SMALL_SIZE_CLASSES > 56 */
-#endif /* NB_SMALL_SIZE_CLASSES > 48 */
-#endif /* NB_SMALL_SIZE_CLASSES > 40 */
-#endif /* NB_SMALL_SIZE_CLASSES > 32 */
-#endif /* NB_SMALL_SIZE_CLASSES > 24 */
-#endif /* NB_SMALL_SIZE_CLASSES > 16 */
-#endif /* NB_SMALL_SIZE_CLASSES >  8 */
-};
-
-/*
- * Free (cached) pools
- */
-static poolp freepools = NULL;		/* free list for cached pools */
-
-/*==========================================================================*/
-/* Arena management. */
-
-/* arenas is a vector of arena base addresses, in order of allocation time.
- * arenas currently contains narenas entries, and has space allocated
- * for at most maxarenas entries.
- *
- * CAUTION:  See the long comment block about thread safety in new_arena():
- * the code currently relies in deep ways on that this vector only grows,
- * and only grows by appending at the end.  For now we never return an arena
- * to the OS.
- */
-static uptr *volatile arenas = NULL;	/* the pointer itself is volatile */
-static volatile uint narenas = 0;
-static uint maxarenas = 0;
-
-/* Number of pools still available to be allocated in the current arena. */
-static uint nfreepools = 0;
-
-/* Free space start address in current arena.  This is pool-aligned. */
-static block *arenabase = NULL;
-
-/* Allocate a new arena and return its base address.  If we run out of
- * memory, return NULL.
- */
-static block *
-new_arena(void)
-{
-	uint excess;	/* number of bytes above pool alignment */
-	block *bp = (block *)malloc(ARENA_SIZE);
-	if (bp == NULL)
-		return NULL;
-
-#if 0   /* XXX removed for PyPy - #ifdef PYMALLOC_DEBUG */
-	if (Py_GETENV("PYTHONMALLOCSTATS"))
-		_PyObject_DebugMallocStats();
-#endif
-
-	/* arenabase <- first pool-aligned address in the arena
-	   nfreepools <- number of whole pools that fit after alignment */
-	arenabase = bp;
-	nfreepools = ARENA_SIZE / POOL_SIZE;
-	assert(POOL_SIZE * nfreepools == ARENA_SIZE);
-	excess = (uint) ((long)bp & POOL_SIZE_MASK);
-	if (excess != 0) {
-		--nfreepools;
-		arenabase += POOL_SIZE - excess;
-	}
-
-	/* Make room for a new entry in the arenas vector. */
-	if (arenas == NULL) {
-		assert(narenas == 0 && maxarenas == 0);
-		arenas = (uptr *)malloc(16 * sizeof(*arenas));
-		if (arenas == NULL)
-			goto error;
-		maxarenas = 16;
-	}
-	else if (narenas == maxarenas) {
-		/* Grow arenas.
-		 *
-		 * Exceedingly subtle:  Someone may be calling the pymalloc
-		 * free via PyMem_{DEL, Del, FREE, Free} without holding the
-		 *.GIL.  Someone else may simultaneously be calling the
-		 * pymalloc malloc while holding the GIL via, e.g.,
-		 * PyObject_New.  Now the pymalloc free may index into arenas
-		 * for an address check, while the pymalloc malloc calls
-		 * new_arena and we end up here to grow a new arena *and*
-		 * grow the arenas vector.  If the value for arenas pymalloc
-		 * free picks up "vanishes" during this resize, anything may
-		 * happen, and it would be an incredibly rare bug.  Therefore
-		 * the code here takes great pains to make sure that, at every
-		 * moment, arenas always points to an intact vector of
-		 * addresses.  It doesn't matter whether arenas points to a
-		 * wholly up-to-date vector when pymalloc free checks it in
-		 * this case, because the only legal (and that even this is
-		 * legal is debatable) way to call PyMem_{Del, etc} while not
-		 * holding the GIL is if the memory being released is not
-		 * object memory, i.e. if the address check in pymalloc free
-		 * is supposed to fail.  Having an incomplete vector can't
-		 * make a supposed-to-fail case succeed by mistake (it could
-		 * only make a supposed-to-succeed case fail by mistake).
-		 *
-		 * In addition, without a lock we can't know for sure when
-		 * an old vector is no longer referenced, so we simply let
-		 * old vectors leak.
-		 *
-		 * And on top of that, since narenas and arenas can't be
-		 * changed as-a-pair atomically without a lock, we're also
-		 * careful to declare them volatile and ensure that we change
-		 * arenas first.  This prevents another thread from picking
-		 * up an narenas value too large for the arenas value it
-		 * reads up (arenas never shrinks).
-		 *
-		 * Read the above 50 times before changing anything in this
-		 * block.
-		 */
-		uptr *p;
-		uint newmax = maxarenas << 1;
-		if (newmax <= maxarenas)	/* overflow */
-			goto error;
-		p = (uptr *)malloc(newmax * sizeof(*arenas));
-		if (p == NULL)
-			goto error;
-		memcpy(p, arenas, narenas * sizeof(*arenas));
-		arenas = p;	/* old arenas deliberately leaked */
-		maxarenas = newmax;
-	}
-
-	/* Append the new arena address to arenas. */
-	assert(narenas < maxarenas);
-	arenas[narenas] = (uptr)bp;
-	++narenas;	/* can't overflow, since narenas < maxarenas before */
-	return bp;
-
-error:
-	free(bp);
-	nfreepools = 0;
-	return NULL;
-}
-
-/* Return true if and only if P is an address that was allocated by
- * pymalloc.  I must be the index into arenas that the address claims
- * to come from.
- *
- * Tricky:  Letting B be the arena base address in arenas[I], P belongs to the
- * arena if and only if
- *	B <= P < B + ARENA_SIZE
- * Subtracting B throughout, this is true iff
- *	0 <= P-B < ARENA_SIZE
- * By using unsigned arithmetic, the "0 <=" half of the test can be skipped.
- *
- * Obscure:  A PyMem "free memory" function can call the pymalloc free or
- * realloc before the first arena has been allocated.  arenas is still
- * NULL in that case.  We're relying on that narenas is also 0 in that case,
- * so the (I) < narenas must be false, saving us from trying to index into
- * a NULL arenas.
- */
-#define Py_ADDRESS_IN_RANGE(P, POOL)	\
-	((POOL)->arenaindex < narenas &&		\
-	 (uptr)(P) - arenas[(POOL)->arenaindex] < (uptr)ARENA_SIZE)
-
-/* This is only useful when running memory debuggers such as
- * Purify or Valgrind.  Uncomment to use.
- *
-#define Py_USING_MEMORY_DEBUGGER
- */
-
-#ifdef Py_USING_MEMORY_DEBUGGER
-
-/* Py_ADDRESS_IN_RANGE may access uninitialized memory by design
- * This leads to thousands of spurious warnings when using
- * Purify or Valgrind.  By making a function, we can easily
- * suppress the uninitialized memory reads in this one function.
- * So we won't ignore real errors elsewhere.
- *
- * Disable the macro and use a function.
- */
-
-#undef Py_ADDRESS_IN_RANGE
-
-/* Don't make static, to ensure this isn't inlined. */
-int Py_ADDRESS_IN_RANGE(void *P, poolp pool);
-#endif
-
-/*==========================================================================*/
-
-/* malloc.  Note that nbytes==0 tries to return a non-NULL pointer, distinct
- * from all other currently live pointers.  This may not be possible.
- */
-
-/*
- * The basic blocks are ordered by decreasing execution frequency,
- * which minimizes the number of jumps in the most common cases,
- * improves branching prediction and instruction scheduling (small
- * block allocations typically result in a couple of instructions).
- * Unless the optimizer reorders everything, being too smart...
- */
-
-#undef PyObject_Malloc
-void *
-PyObject_Malloc(size_t nbytes)
-{
-	block *bp;
-	poolp pool;
-	poolp next;
-	uint size;
-
-	/*
-	 * This implicitly redirects malloc(0).
-	 */
-	if ((nbytes - 1) < SMALL_REQUEST_THRESHOLD) {
-		LOCK();
-		/*
-		 * Most frequent paths first
-		 */
-		size = (uint )(nbytes - 1) >> ALIGNMENT_SHIFT;
-		pool = usedpools[size + size];
-		if (pool != pool->nextpool) {
-			/*
-			 * There is a used pool for this size class.
-			 * Pick up the head block of its free list.
-			 */
-			++pool->ref.count;
-			bp = pool->freeblock;
-			assert(bp != NULL);
-			if ((pool->freeblock = *(block **)bp) != NULL) {
-				UNLOCK();
-				return (void *)bp;
-			}
-			/*
-			 * Reached the end of the free list, try to extend it
-			 */
-			if (pool->nextoffset <= pool->maxnextoffset) {
-				/*
-				 * There is room for another block
-				 */
-				pool->freeblock = (block *)pool +
-						  pool->nextoffset;
-				pool->nextoffset += INDEX2SIZE(size);
-				*(block **)(pool->freeblock) = NULL;
-				UNLOCK();
-				return (void *)bp;
-			}
-			/*
-			 * Pool is full, unlink from used pools
-			 */
-			next = pool->nextpool;
-			pool = pool->prevpool;
-			next->prevpool = pool;
-			pool->nextpool = next;
-			UNLOCK();
-			return (void *)bp;
-		}
-		/*
-		 * Try to get a cached free pool
-		 */
-		pool = freepools;
-		if (pool != NULL) {
-			/*
-			 * Unlink from cached pools
-			 */
-			freepools = pool->nextpool;
-		init_pool:
-			/*
-			 * Frontlink to used pools
-			 */
-			next = usedpools[size + size]; /* == prev */
-			pool->nextpool = next;
-			pool->prevpool = next;
-			next->nextpool = pool;
-			next->prevpool = pool;
-			pool->ref.count = 1;
-			if (pool->szidx == size) {
-				/*
-				 * Luckily, this pool last contained blocks
-				 * of the same size class, so its header
-				 * and free list are already initialized.
-				 */
-				bp = pool->freeblock;
-				pool->freeblock = *(block **)bp;
-				UNLOCK();
-				return (void *)bp;
-			}
-			/*
-			 * Initialize the pool header, set up the free list to
-			 * contain just the second block, and return the first
-			 * block.
-			 */
-			pool->szidx = size;
-			size = INDEX2SIZE(size);
-			bp = (block *)pool + POOL_OVERHEAD;
-			pool->nextoffset = POOL_OVERHEAD + (size << 1);
-			pool->maxnextoffset = POOL_SIZE - size;
-			pool->freeblock = bp + size;
-			*(block **)(pool->freeblock) = NULL;
-			UNLOCK();
-			return (void *)bp;
-		}
-		/*
-		 * Allocate new pool
-		 */
-		if (nfreepools) {
-		commit_pool:
-			--nfreepools;
-			pool = (poolp)arenabase;
-			arenabase += POOL_SIZE;
-			pool->arenaindex = narenas - 1;
-			pool->szidx = DUMMY_SIZE_IDX;
-			goto init_pool;
-		}
-		/*
-		 * Allocate new arena
-		 */
-#ifdef WITH_MEMORY_LIMITS
-		if (!(narenas < MAX_ARENAS)) {
-			UNLOCK();
-			goto redirect;
-		}
-#endif
-		bp = new_arena();
-		if (bp != NULL)
-			goto commit_pool;
-		UNLOCK();
-		goto redirect;
-	}
-
-        /* The small block allocator ends here. */
-
-redirect:
-	/*
-	 * Redirect the original request to the underlying (libc) allocator.
-	 * We jump here on bigger requests, on error in the code above (as a
-	 * last chance to serve the request) or when the max memory limit
-	 * has been reached.
-	 */
-	if (nbytes == 0)
-		nbytes = 1;
-	return (void *)malloc(nbytes);
-}
-
-/* free */
-
-#undef PyObject_Free
-void
-PyObject_Free(void *p)
-{
-	poolp pool;
-	block *lastfree;
-	poolp next, prev;
-	uint size;
-
-	if (p == NULL)	/* free(NULL) has no effect */
-		return;
-
-	pool = POOL_ADDR(p);
-	if (Py_ADDRESS_IN_RANGE(p, pool)) {
-		/* We allocated this address. */
-		LOCK();
-		/*
-		 * Link p to the start of the pool's freeblock list.  Since
-		 * the pool had at least the p block outstanding, the pool
-		 * wasn't empty (so it's already in a usedpools[] list, or
-		 * was full and is in no list -- it's not in the freeblocks
-		 * list in any case).
-		 */
-		assert(pool->ref.count > 0);	/* else it was empty */
-		*(block **)p = lastfree = pool->freeblock;
-		pool->freeblock = (block *)p;
-		if (lastfree) {
-			/*
-			 * freeblock wasn't NULL, so the pool wasn't full,
-			 * and the pool is in a usedpools[] list.
-			 */
-			if (--pool->ref.count != 0) {
-				/* pool isn't empty:  leave it in usedpools */
-				UNLOCK();
-				return;
-			}
-			/*
-			 * Pool is now empty:  unlink from usedpools, and
-			 * link to the front of freepools.  This ensures that
-			 * previously freed pools will be allocated later
-			 * (being not referenced, they are perhaps paged out).
-			 */
-			next = pool->nextpool;
-			prev = pool->prevpool;
-			next->prevpool = prev;
-			prev->nextpool = next;
-			/* Link to freepools.  This is a singly-linked list,
-			 * and pool->prevpool isn't used there.
-			 */
-			pool->nextpool = freepools;
-			freepools = pool;
-			UNLOCK();
-			return;
-		}
-		/*
-		 * Pool was full, so doesn't currently live in any list:
-		 * link it to the front of the appropriate usedpools[] list.
-		 * This mimics LRU pool usage for new allocations and
-		 * targets optimal filling when several pools contain
-		 * blocks of the same size class.
-		 */
-		--pool->ref.count;
-		assert(pool->ref.count > 0);	/* else the pool is empty */
-		size = pool->szidx;
-		next = usedpools[size + size];
-		prev = next->prevpool;
-		/* insert pool before next:   prev <-> pool <-> next */
-		pool->nextpool = next;
-		pool->prevpool = prev;
-		next->prevpool = pool;
-		prev->nextpool = pool;
-		UNLOCK();
-		return;
-	}
-
-	/* We didn't allocate this address. */
-	free(p);
-}
-
-/* realloc.  If p is NULL, this acts like malloc(nbytes).  Else if nbytes==0,
- * then as the Python docs promise, we do not treat this like free(p), and
- * return a non-NULL result.
- */
-
-#undef PyObject_Realloc
-void *
-PyObject_Realloc(void *p, size_t nbytes)
-{
-	void *bp;
-	poolp pool;
-	uint size;
-
-	if (p == NULL)
-		return PyObject_Malloc(nbytes);
-
-	pool = POOL_ADDR(p);
-	if (Py_ADDRESS_IN_RANGE(p, pool)) {
-		/* We're in charge of this block */
-		size = INDEX2SIZE(pool->szidx);
-		if (nbytes <= size) {
-			/* The block is staying the same or shrinking.  If
-			 * it's shrinking, there's a tradeoff:  it costs
-			 * cycles to copy the block to a smaller size class,
-			 * but it wastes memory not to copy it.  The
-			 * compromise here is to copy on shrink only if at
-			 * least 25% of size can be shaved off.
-			 */
-			if (4 * nbytes > 3 * size) {
-				/* It's the same,
-				 * or shrinking and new/old > 3/4.
-				 */
-				return p;
-			}
-			size = nbytes;
-		}
-		bp = PyObject_Malloc(nbytes);
-		if (bp != NULL) {
-			memcpy(bp, p, size);
-			PyObject_Free(p);
-		}
-		return bp;
-	}
-	/* We're not managing this block.  If nbytes <=
-	 * SMALL_REQUEST_THRESHOLD, it's tempting to try to take over this
-	 * block.  However, if we do, we need to copy the valid data from
-	 * the C-managed block to one of our blocks, and there's no portable
-	 * way to know how much of the memory space starting at p is valid.
-	 * As bug 1185883 pointed out the hard way, it's possible that the
-	 * C-managed block is "at the end" of allocated VM space, so that
-	 * a memory fault can occur if we try to copy nbytes bytes starting
-	 * at p.  Instead we punt:  let C continue to manage this block.
-         */
-	if (nbytes)
-		return realloc(p, nbytes);
-	/* C doesn't define the result of realloc(p, 0) (it may or may not
-	 * return NULL then), but Python's docs promise that nbytes==0 never
-	 * returns NULL.  We don't pass 0 to realloc(), to avoid that endcase
-	 * to begin with.  Even then, we can't be sure that realloc() won't
-	 * return NULL.
-	 */
-	bp = realloc(p, 1);
-   	return bp ? bp : p;
-}
-
-#else	/* ! WITH_PYMALLOC */
-
-/*==========================================================================*/
-/* pymalloc not enabled:  Redirect the entry points to malloc.  These will
- * only be used by extensions that are compiled with pymalloc enabled. */
-
-void *
-PyObject_Malloc(size_t n)
-{
-	return PyMem_MALLOC(n);
-}
-
-void *
-PyObject_Realloc(void *p, size_t n)
-{
-	return PyMem_REALLOC(p, n);
-}
-
-void
-PyObject_Free(void *p)
-{
-	PyMem_FREE(p);
-}
-#endif /* WITH_PYMALLOC */
-
-#ifdef PYMALLOC_DEBUG
-/*==========================================================================*/
-/* A x-platform debugging allocator.  This doesn't manage memory directly,
- * it wraps a real allocator, adding extra debugging info to the memory blocks.
- */
-
-/* XXX added for PyPy for stand-alone usage */
-void Py_FatalError(const char *msg)
-{
-  fprintf(stderr, "Py_FatalError() called in obmalloc!\n%s\n", msg);
-  abort();
-}
-#define PyOS_snprintf snprintf
-/* end of XXX */
-
-
-/* Special bytes broadcast into debug memory blocks at appropriate times.
- * Strings of these are unlikely to be valid addresses, floats, ints or
- * 7-bit ASCII.
- */
-#undef CLEANBYTE
-#undef DEADBYTE
-#undef FORBIDDENBYTE
-#define CLEANBYTE      0xCB    /* clean (newly allocated) memory */
-#define DEADBYTE       0xDB    /* dead (newly freed) memory */
-#define FORBIDDENBYTE  0xFB    /* untouchable bytes at each end of a block */
-
-static ulong serialno = 0;	/* incremented on each debug {m,re}alloc */
-
-/* serialno is always incremented via calling this routine.  The point is
-   to supply a single place to set a breakpoint.
-*/
-static void
-bumpserialno(void)
-{
-	++serialno;
-}
-
-
-/* Read 4 bytes at p as a big-endian ulong. */
-static ulong
-read4(const void *p)
-{
-	const uchar *q = (const uchar *)p;
-	return ((ulong)q[0] << 24) |
-	       ((ulong)q[1] << 16) |
-	       ((ulong)q[2] <<  8) |
-	        (ulong)q[3];
-}
-
-/* Write the 4 least-significant bytes of n as a big-endian unsigned int,
-   MSB at address p, LSB at p+3. */
-static void
-write4(void *p, ulong n)
-{
-	uchar *q = (uchar *)p;
-	q[0] = (uchar)((n >> 24) & 0xff);
-	q[1] = (uchar)((n >> 16) & 0xff);
-	q[2] = (uchar)((n >>  8) & 0xff);
-	q[3] = (uchar)( n        & 0xff);
-}
-
-#ifdef Py_DEBUG
-/* Is target in the list?  The list is traversed via the nextpool pointers.
- * The list may be NULL-terminated, or circular.  Return 1 if target is in
- * list, else 0.
- */
-static int
-pool_is_in_list(const poolp target, poolp list)
-{
-	poolp origlist = list;
-	assert(target != NULL);
-	if (list == NULL)
-		return 0;
-	do {
-		if (target == list)
-			return 1;
-		list = list->nextpool;
-	} while (list != NULL && list != origlist);
-	return 0;
-}
-
-#else
-#define pool_is_in_list(X, Y) 1
-
-#endif	/* Py_DEBUG */
-
-/* The debug malloc asks for 16 extra bytes and fills them with useful stuff,
-   here calling the underlying malloc's result p:
-
-p[0:4]
-    Number of bytes originally asked for.  4-byte unsigned integer,
-    big-endian (easier to read in a memory dump).
-p[4:8]
-    Copies of FORBIDDENBYTE.  Used to catch under- writes and reads.
-p[8:8+n]
-    The requested memory, filled with copies of CLEANBYTE.
-    Used to catch reference to uninitialized memory.
-    &p[8] is returned.  Note that this is 8-byte aligned if pymalloc
-    handled the request itself.
-p[8+n:8+n+4]
-    Copies of FORBIDDENBYTE.  Used to catch over- writes and reads.
-p[8+n+4:8+n+8]
-    A serial number, incremented by 1 on each call to _PyObject_DebugMalloc
-    and _PyObject_DebugRealloc.
-    4-byte unsigned integer, big-endian.
-    If "bad memory" is detected later, the serial number gives an
-    excellent way to set a breakpoint on the next run, to capture the
-    instant at which this block was passed out.
-*/
-
-void *
-_PyObject_DebugMalloc(size_t nbytes)
-{
-	uchar *p;	/* base address of malloc'ed block */
-	uchar *tail;	/* p + 8 + nbytes == pointer to tail pad bytes */
-	size_t total;	/* nbytes + 16 */
-
-	bumpserialno();
-	total = nbytes + 16;
-	if (total < nbytes || (total >> 31) > 1) {
-		/* overflow, or we can't represent it in 4 bytes */
-		/* Obscure:  can't do (total >> 32) != 0 instead, because
-		   C doesn't define what happens for a right-shift of 32
-		   when size_t is a 32-bit type.  At least C guarantees
-		   size_t is an unsigned type. */
-		return NULL;
-	}
-
-	p = (uchar *)PyObject_Malloc(total);
-	if (p == NULL)
-		return NULL;
-
-	write4(p, nbytes);
-	p[4] = p[5] = p[6] = p[7] = FORBIDDENBYTE;
-
-	if (nbytes > 0)
-		memset(p+8, CLEANBYTE, nbytes);
-
-	tail = p + 8 + nbytes;
-	tail[0] = tail[1] = tail[2] = tail[3] = FORBIDDENBYTE;
-	write4(tail + 4, serialno);
-
-	return p+8;
-}
-
-/* The debug free first checks the 8 bytes on each end for sanity (in
-   particular, that the FORBIDDENBYTEs are still intact).
-   Then fills the original bytes with DEADBYTE.
-   Then calls the underlying free.
-*/
-void
-_PyObject_DebugFree(void *p)
-{
-	uchar *q = (uchar *)p;
-	size_t nbytes;
-
-	if (p == NULL)
-		return;
-	_PyObject_DebugCheckAddress(p);
-	nbytes = read4(q-8);
-	if (nbytes > 0)
-		memset(q, DEADBYTE, nbytes);
-	PyObject_Free(q-8);
-}
-
-void *
-_PyObject_DebugRealloc(void *p, size_t nbytes)
-{
-	uchar *q = (uchar *)p;
-	uchar *tail;
-	size_t total;	/* nbytes + 16 */
-	size_t original_nbytes;
-
-	if (p == NULL)
-		return _PyObject_DebugMalloc(nbytes);
-
-	_PyObject_DebugCheckAddress(p);
-	bumpserialno();
-	original_nbytes = read4(q-8);
-	total = nbytes + 16;
-	if (total < nbytes || (total >> 31) > 1) {
-		/* overflow, or we can't represent it in 4 bytes */
-		return NULL;
-	}
-
-	if (nbytes < original_nbytes) {
-		/* shrinking:  mark old extra memory dead */
-		memset(q + nbytes, DEADBYTE, original_nbytes - nbytes);
-	}
-
-	/* Resize and add decorations. */
-	q = (uchar *)PyObject_Realloc(q-8, total);
-	if (q == NULL)
-		return NULL;
-
-	write4(q, nbytes);
-	assert(q[4] == FORBIDDENBYTE &&
-	       q[5] == FORBIDDENBYTE &&
-	       q[6] == FORBIDDENBYTE &&
-	       q[7] == FORBIDDENBYTE);
-	q += 8;
-	tail = q + nbytes;
-	tail[0] = tail[1] = tail[2] = tail[3] = FORBIDDENBYTE;
-	write4(tail + 4, serialno);
-
-	if (nbytes > original_nbytes) {
-		/* growing:  mark new extra memory clean */
-		memset(q + original_nbytes, CLEANBYTE,
-			nbytes - original_nbytes);
-	}
-
-	return q;
-}
-
-/* Check the forbidden bytes on both ends of the memory allocated for p.
- * If anything is wrong, print info to stderr via _PyObject_DebugDumpAddress,
- * and call Py_FatalError to kill the program.
- */
- void
-_PyObject_DebugCheckAddress(const void *p)
-{
-	const uchar *q = (const uchar *)p;
-	char *msg;
-	ulong nbytes;
-	const uchar *tail;
-	int i;
-
-	if (p == NULL) {
-		msg = "didn't expect a NULL pointer";
-		goto error;
-	}
-
-	/* Check the stuff at the start of p first:  if there's underwrite
-	 * corruption, the number-of-bytes field may be nuts, and checking
-	 * the tail could lead to a segfault then.
-	 */
-	for (i = 4; i >= 1; --i) {
-		if (*(q-i) != FORBIDDENBYTE) {
-			msg = "bad leading pad byte";
-			goto error;
-		}
-	}
-
-	nbytes = read4(q-8);
-	tail = q + nbytes;
-	for (i = 0; i < 4; ++i) {
-		if (tail[i] != FORBIDDENBYTE) {
-			msg = "bad trailing pad byte";
-			goto error;
-		}
-	}
-
-	return;
-
-error:
-	_PyObject_DebugDumpAddress(p);
-	Py_FatalError(msg);
-}
-
-/* Display info to stderr about the memory block at p. */
-void
-_PyObject_DebugDumpAddress(const void *p)
-{
-	const uchar *q = (const uchar *)p;
-	const uchar *tail;
-	ulong nbytes, serial;
-	int i;
-
-	fprintf(stderr, "Debug memory block at address p=%p:\n", p);
-	if (p == NULL)
-		return;
-
-	nbytes = read4(q-8);
-	fprintf(stderr, "    %lu bytes originally requested\n", nbytes);
-
-	/* In case this is nuts, check the leading pad bytes first. */
-	fputs("    The 4 pad bytes at p-4 are ", stderr);
-	if (*(q-4) == FORBIDDENBYTE &&
-	    *(q-3) == FORBIDDENBYTE &&
-	    *(q-2) == FORBIDDENBYTE &&
-	    *(q-1) == FORBIDDENBYTE) {
-		fputs("FORBIDDENBYTE, as expected.\n", stderr);
-	}
-	else {
-		fprintf(stderr, "not all FORBIDDENBYTE (0x%02x):\n",
-			FORBIDDENBYTE);
-		for (i = 4; i >= 1; --i) {
-			const uchar byte = *(q-i);
-			fprintf(stderr, "        at p-%d: 0x%02x", i, byte);
-			if (byte != FORBIDDENBYTE)
-				fputs(" *** OUCH", stderr);
-			fputc('\n', stderr);
-		}
-
-		fputs("    Because memory is corrupted at the start, the "
-		      "count of bytes requested\n"
-		      "       may be bogus, and checking the trailing pad "
-		      "bytes may segfault.\n", stderr);
-	}
-
-	tail = q + nbytes;
-	fprintf(stderr, "    The 4 pad bytes at tail=%p are ", tail);
-	if (tail[0] == FORBIDDENBYTE &&
-	    tail[1] == FORBIDDENBYTE &&
-	    tail[2] == FORBIDDENBYTE &&
-	    tail[3] == FORBIDDENBYTE) {
-		fputs("FORBIDDENBYTE, as expected.\n", stderr);
-	}
-	else {
-		fprintf(stderr, "not all FORBIDDENBYTE (0x%02x):\n",
-			FORBIDDENBYTE);
-		for (i = 0; i < 4; ++i) {
-			const uchar byte = tail[i];
-			fprintf(stderr, "        at tail+%d: 0x%02x",
-				i, byte);
-			if (byte != FORBIDDENBYTE)
-				fputs(" *** OUCH", stderr);
-			fputc('\n', stderr);
-		}
-	}
-
-	serial = read4(tail+4);
-	fprintf(stderr, "    The block was made by call #%lu to "
-	                "debug malloc/realloc.\n", serial);
-
-	if (nbytes > 0) {
-		int i = 0;
-		fputs("    Data at p:", stderr);
-		/* print up to 8 bytes at the start */
-		while (q < tail && i < 8) {
-			fprintf(stderr, " %02x", *q);
-			++i;
-			++q;
-		}
-		/* and up to 8 at the end */
-		if (q < tail) {
-			if (tail - q > 8) {
-				fputs(" ...", stderr);
-				q = tail - 8;
-			}
-			while (q < tail) {
-				fprintf(stderr, " %02x", *q);
-				++q;
-			}
-		}
-		fputc('\n', stderr);
-	}
-}
-
-static ulong
-printone(const char* msg, ulong value)
-{
-	int i, k;
-	char buf[100];
-	ulong origvalue = value;
-
-	fputs(msg, stderr);
-	for (i = (int)strlen(msg); i < 35; ++i)
-		fputc(' ', stderr);
-	fputc('=', stderr);
-
-	/* Write the value with commas. */
-	i = 22;
-	buf[i--] = '\0';
-	buf[i--] = '\n';
-	k = 3;
-	do {
-		ulong nextvalue = value / 10UL;
-		uint digit = value - nextvalue * 10UL;
-		value = nextvalue;
-		buf[i--] = (char)(digit + '0');
-		--k;
-		if (k == 0 && value && i >= 0) {
-			k = 3;
-			buf[i--] = ',';
-		}
-	} while (value && i >= 0);
-
-	while (i >= 0)
-		buf[i--] = ' ';
-	fputs(buf, stderr);
-
-	return origvalue;
-}
-
-/* Print summary info to stderr about the state of pymalloc's structures.
- * In Py_DEBUG mode, also perform some expensive internal consistency
- * checks.
- */
-void
-_PyObject_DebugMallocStats(void)
-{
-	uint i;
-	const uint numclasses = SMALL_REQUEST_THRESHOLD >> ALIGNMENT_SHIFT;
-	/* # of pools, allocated blocks, and free blocks per class index */
-	ulong numpools[SMALL_REQUEST_THRESHOLD >> ALIGNMENT_SHIFT];
-	ulong numblocks[SMALL_REQUEST_THRESHOLD >> ALIGNMENT_SHIFT];
-	ulong numfreeblocks[SMALL_REQUEST_THRESHOLD >> ALIGNMENT_SHIFT];
-	/* total # of allocated bytes in used and full pools */
-	ulong allocated_bytes = 0;
-	/* total # of available bytes in used pools */
-	ulong available_bytes = 0;
-	/* # of free pools + pools not yet carved out of current arena */
-	uint numfreepools = 0;
-	/* # of bytes for arena alignment padding */
-	ulong arena_alignment = 0;
-	/* # of bytes in used and full pools used for pool_headers */
-	ulong pool_header_bytes = 0;
-	/* # of bytes in used and full pools wasted due to quantization,
-	 * i.e. the necessarily leftover space at the ends of used and
-	 * full pools.
-	 */
-	ulong quantization = 0;
-	/* running total -- should equal narenas * ARENA_SIZE */
-	ulong total;
-	char buf[128];
-
-	fprintf(stderr, "Small block threshold = %d, in %u size classes.\n",
-		SMALL_REQUEST_THRESHOLD, numclasses);
-
-	for (i = 0; i < numclasses; ++i)
-		numpools[i] = numblocks[i] = numfreeblocks[i] = 0;
-
-	/* Because full pools aren't linked to from anything, it's easiest
-	 * to march over all the arenas.  If we're lucky, most of the memory
-	 * will be living in full pools -- would be a shame to miss them.
-	 */
-	for (i = 0; i < narenas; ++i) {
-		uint poolsinarena;
-		uint j;
-		uptr base = arenas[i];
-
-		/* round up to pool alignment */
-		poolsinarena = ARENA_SIZE / POOL_SIZE;
-		if (base & (uptr)POOL_SIZE_MASK) {
-			--poolsinarena;
-			arena_alignment += POOL_SIZE;
-			base &= ~(uptr)POOL_SIZE_MASK;
-			base += POOL_SIZE;
-		}
-
-		if (i == narenas - 1) {
-			/* current arena may have raw memory at the end */
-			numfreepools += nfreepools;
-			poolsinarena -= nfreepools;
-		}
-
-		/* visit every pool in the arena */
-		for (j = 0; j < poolsinarena; ++j, base += POOL_SIZE) {
-			poolp p = (poolp)base;
-			const uint sz = p->szidx;
-			uint freeblocks;
-
-			if (p->ref.count == 0) {
-				/* currently unused */
-				++numfreepools;
-				assert(pool_is_in_list(p, freepools));
-				continue;
-			}
-			++numpools[sz];
-			numblocks[sz] += p->ref.count;
-			freeblocks = NUMBLOCKS(sz) - p->ref.count;
-			numfreeblocks[sz] += freeblocks;
-#ifdef Py_DEBUG
-			if (freeblocks > 0)
-				assert(pool_is_in_list(p, usedpools[sz + sz]));
-#endif
-		}
-	}
-
-	fputc('\n', stderr);
-	fputs("class   size   num pools   blocks in use  avail blocks\n"
-	      "-----   ----   ---------   -------------  ------------\n",
-		stderr);
-
-	for (i = 0; i < numclasses; ++i) {
-		ulong p = numpools[i];
-		ulong b = numblocks[i];
-		ulong f = numfreeblocks[i];
-		uint size = INDEX2SIZE(i);
-		if (p == 0) {
-			assert(b == 0 && f == 0);
-			continue;
-		}
-		fprintf(stderr, "%5u %6u %11lu %15lu %13lu\n",
-			i, size, p, b, f);
-		allocated_bytes += b * size;
-		available_bytes += f * size;
-		pool_header_bytes += p * POOL_OVERHEAD;
-		quantization += p * ((POOL_SIZE - POOL_OVERHEAD) % size);
-	}
-	fputc('\n', stderr);
-	(void)printone("# times object malloc called", serialno);
-
-	PyOS_snprintf(buf, sizeof(buf),
-		"%u arenas * %d bytes/arena", narenas, ARENA_SIZE);
-	(void)printone(buf, (ulong)narenas * ARENA_SIZE);
-
-	fputc('\n', stderr);
-
-	total = printone("# bytes in allocated blocks", allocated_bytes);
-	total += printone("# bytes in available blocks", available_bytes);
-
-	PyOS_snprintf(buf, sizeof(buf),
-		"%u unused pools * %d bytes", numfreepools, POOL_SIZE);
-	total += printone(buf, (ulong)numfreepools * POOL_SIZE);
-
-	total += printone("# bytes lost to pool headers", pool_header_bytes);
-	total += printone("# bytes lost to quantization", quantization);
-	total += printone("# bytes lost to arena alignment", arena_alignment);
-	(void)printone("Total", total);
-}
-
-#endif	/* PYMALLOC_DEBUG */
-
-#ifdef Py_USING_MEMORY_DEBUGGER
-/* Make this function last so gcc won't inline it
-   since the definition is after the reference. */
-int
-Py_ADDRESS_IN_RANGE(void *P, poolp pool)
-{
-	return ((pool->arenaindex) < narenas &&
-		(uptr)(P) - arenas[pool->arenaindex] < (uptr)ARENA_SIZE);
-}
-#endif

From noreply at buildbot.pypy.org  Fri Nov 21 08:15:56 2014
From: noreply at buildbot.pypy.org (fijal)
Date: Fri, 21 Nov 2014 08:15:56 +0100 (CET)
Subject: [pypy-commit] pypy default: kill mentions of obmalloc here
Message-ID: <20141121071556.B979C1C134B@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: 
Changeset: r74628:95b519f3bfe2
Date: 2014-11-21 09:15 +0200
http://bitbucket.org/pypy/pypy/changeset/95b519f3bfe2/

Log:	kill mentions of obmalloc here

diff --git a/rpython/memory/gc/incminimark.py b/rpython/memory/gc/incminimark.py
--- a/rpython/memory/gc/incminimark.py
+++ b/rpython/memory/gc/incminimark.py
@@ -216,8 +216,8 @@
         # fall-back number.
         "nursery_size": 896*1024,
 
-        # The system page size.  Like obmalloc.c, we assume that it is 4K
-        # for 32-bit systems; unlike obmalloc.c, we assume that it is 8K
+        # The system page size.  Like malloc, we assume that it is 4K
+        # for 32-bit systems; unlike malloc, we assume that it is 8K
         # for 64-bit systems, for consistent results.
         "page_size": 1024*WORD,
 
diff --git a/rpython/memory/gc/minimark.py b/rpython/memory/gc/minimark.py
--- a/rpython/memory/gc/minimark.py
+++ b/rpython/memory/gc/minimark.py
@@ -174,8 +174,8 @@
         # fall-back number.
         "nursery_size": 896*1024,
 
-        # The system page size.  Like obmalloc.c, we assume that it is 4K
-        # for 32-bit systems; unlike obmalloc.c, we assume that it is 8K
+        # The system page size.  Like malloc, we assume that it is 4K
+        # for 32-bit systems; unlike malloc, we assume that it is 8K
         # for 64-bit systems, for consistent results.
         "page_size": 1024*WORD,
 
diff --git a/rpython/memory/gc/minimarktest.py b/rpython/memory/gc/minimarktest.py
--- a/rpython/memory/gc/minimarktest.py
+++ b/rpython/memory/gc/minimarktest.py
@@ -5,7 +5,7 @@
 from rpython.rlib.rarithmetic import LONG_BIT
 
 # For testing, a simple implementation of ArenaCollection.
-# This version could be used together with obmalloc.c, but
+# This version could be used together with malloc, but
 # it requires an extra word per object in the 'all_objects'
 # list.
 

From noreply at buildbot.pypy.org  Fri Nov 21 09:11:16 2014
From: noreply at buildbot.pypy.org (fijal)
Date: Fri, 21 Nov 2014 09:11:16 +0100 (CET)
Subject: [pypy-commit] pypy optresult: hack hack hack until we start passing
 more tests from test_loop
Message-ID: <20141121081116.8ECA21D2868@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: optresult
Changeset: r74629:e2f4be5fe8dc
Date: 2014-11-21 10:11 +0200
http://bitbucket.org/pypy/pypy/changeset/e2f4be5fe8dc/

Log:	hack hack hack until we start passing more tests from test_loop

diff --git a/rpython/jit/backend/llgraph/runner.py b/rpython/jit/backend/llgraph/runner.py
--- a/rpython/jit/backend/llgraph/runner.py
+++ b/rpython/jit/backend/llgraph/runner.py
@@ -912,7 +912,7 @@
         # cond_call can't have a return value
         self.execute_call(calldescr, func, *args)
 
-    def execute_call(self, calldescr, func, *args):
+    def _execute_call(self, calldescr, func, *args):
         effectinfo = calldescr.get_extra_info()
         if effectinfo is not None and hasattr(effectinfo, 'oopspecindex'):
             oopspecindex = effectinfo.oopspecindex
@@ -928,6 +928,11 @@
             res = _example_res[getkind(TP.RESULT)[0]]
         return res
 
+    execute_call_i = _execute_call
+    execute_call_r = _execute_call
+    execute_call_f = _execute_call
+    execute_call_n = _execute_call
+
     def execute_call_may_force(self, calldescr, func, *args):
         call_op = self.lltrace.operations[self.current_index]
         guard_op = self.lltrace.operations[self.current_index + 1]
diff --git a/rpython/jit/metainterp/compile.py b/rpython/jit/metainterp/compile.py
--- a/rpython/jit/metainterp/compile.py
+++ b/rpython/jit/metainterp/compile.py
@@ -785,7 +785,6 @@
     """Try to compile a new bridge leading from the beginning of the history
     to some existing place.
     """
-    xxx
     from rpython.jit.metainterp.optimizeopt import optimize_trace
 
     # The history contains new operations to attach as the code for the
@@ -797,7 +796,9 @@
     new_trace.inputargs = metainterp.history.inputargs[:]
     # clone ops, as optimize_bridge can mutate the ops
 
-    new_trace.operations = [op.clone() for op in metainterp.history.operations]
+    memo = Memo()
+    new_trace.operations = [op.clone(memo) for op in
+                            metainterp.history.operations]
     metainterp_sd = metainterp.staticdata
     state = metainterp.jitdriver_sd.warmstate
     if isinstance(resumekey, ResumeAtPositionDescr):
@@ -805,7 +806,8 @@
     else:
         inline_short_preamble = True
     try:
-        optimize_trace(metainterp_sd, new_trace, state.enable_opts, inline_short_preamble)
+        optimize_trace(metainterp_sd, new_trace, state.enable_opts,
+                       inline_short_preamble)
     except InvalidLoop:
         debug_print("compile_new_bridge: got an InvalidLoop")
         # XXX I am fairly convinced that optimize_bridge cannot actually raise
diff --git a/rpython/jit/metainterp/executor.py b/rpython/jit/metainterp/executor.py
--- a/rpython/jit/metainterp/executor.py
+++ b/rpython/jit/metainterp/executor.py
@@ -1,7 +1,7 @@
 """This implements pyjitpl's execution of operations.
 """
 
-from rpython.rtyper.lltypesystem import lltype, rstr
+from rpython.rtyper.lltypesystem import lltype, rstr, llmemory
 from rpython.rlib.rarithmetic import ovfcheck, r_longlong, is_valid_int
 from rpython.rlib.unroll import unrolling_iterable
 from rpython.rlib.objectmodel import specialize
@@ -15,81 +15,81 @@
 
 # ____________________________________________________________
 
-def do_call(cpu, metainterp, argboxes, descr):
-    xxx
-    assert metainterp is not None
-    # count the number of arguments of the different types
-    count_i = count_r = count_f = 0
-    for i in range(1, len(argboxes)):
-        type = argboxes[i].type
-        if   type == INT:   count_i += 1
-        elif type == REF:   count_r += 1
-        elif type == FLOAT: count_f += 1
-    # allocate lists for each type that has at least one argument
-    if count_i: args_i = [0] * count_i
-    else:       args_i = None
-    if count_r: args_r = [NULL] * count_r
-    else:       args_r = None
-    if count_f: args_f = [longlong.ZEROF] * count_f
-    else:       args_f = None
-    # fill in the lists
-    count_i = count_r = count_f = 0
-    for i in range(1, len(argboxes)):
-        box = argboxes[i]
-        if   box.type == INT:
-            args_i[count_i] = box.getint()
-            count_i += 1
-        elif box.type == REF:
-            args_r[count_r] = box.getref_base()
-            count_r += 1
-        elif box.type == FLOAT:
-            args_f[count_f] = box.getfloatstorage()
-            count_f += 1
-    # get the function address as an integer
-    func = argboxes[0].getint()
-    # do the call using the correct function from the cpu
-    rettype = descr.get_result_type()
-    if rettype == INT or rettype == 'S':       # *S*ingle float
-        try:
-            result = cpu.bh_call_i(func, args_i, args_r, args_f, descr)
-        except Exception, e:
-            metainterp.execute_raised(e)
-            result = 0
-        return BoxInt(result)
-    if rettype == REF:
-        try:
-            result = cpu.bh_call_r(func, args_i, args_r, args_f, descr)
-        except Exception, e:
-            metainterp.execute_raised(e)
-            result = NULL
-        return BoxPtr(result)
-    if rettype == FLOAT or rettype == 'L':     # *L*ong long
-        try:
-            result = cpu.bh_call_f(func, args_i, args_r, args_f, descr)
-        except Exception, e:
-            metainterp.execute_raised(e)
-            result = longlong.ZEROF
-        return BoxFloat(result)
-    if rettype == VOID:
-        try:
-            cpu.bh_call_v(func, args_i, args_r, args_f, descr)
-        except Exception, e:
-            metainterp.execute_raised(e)
-        return None
-    raise AssertionError("bad rettype")
+def new_do_call(rettype):
+    def do_call(cpu, metainterp, argboxes, descr):
+        assert metainterp is not None
+        # count the number of arguments of the different types
+        count_i = count_r = count_f = 0
+        for i in range(1, len(argboxes)):
+            type = argboxes[i].type
+            if   type == INT:   count_i += 1
+            elif type == REF:   count_r += 1
+            elif type == FLOAT: count_f += 1
+        # allocate lists for each type that has at least one argument
+        if count_i: args_i = [0] * count_i
+        else:       args_i = None
+        if count_r: args_r = [NULL] * count_r
+        else:       args_r = None
+        if count_f: args_f = [longlong.ZEROF] * count_f
+        else:       args_f = None
+        # fill in the lists
+        count_i = count_r = count_f = 0
+        for i in range(1, len(argboxes)):
+            box = argboxes[i]
+            if   box.type == INT:
+                args_i[count_i] = box.getint()
+                count_i += 1
+            elif box.type == REF:
+                args_r[count_r] = box.getref_base()
+                count_r += 1
+            elif box.type == FLOAT:
+                args_f[count_f] = box.getfloatstorage()
+                count_f += 1
+        # get the function address as an integer
+        func = argboxes[0].getint()
+        # do the call using the correct function from the cpu
+        if rettype == INT or rettype == 'S':       # *S*ingle float
+            try:
+                result = cpu.bh_call_i(func, args_i, args_r, args_f, descr)
+            except Exception, e:
+                metainterp.execute_raised(e)
+                result = 0
+            return result
+        if rettype == REF:
+            try:
+                result = cpu.bh_call_r(func, args_i, args_r, args_f, descr)
+            except Exception, e:
+                metainterp.execute_raised(e)
+                result = NULL
+            return result
+        if rettype == FLOAT:
+            try:
+                result = cpu.bh_call_f(func, args_i, args_r, args_f, descr)
+            except Exception, e:
+                metainterp.execute_raised(e)
+                result = longlong.ZEROF
+            return result
+        if rettype == VOID:
+            try:
+                cpu.bh_call_v(func, args_i, args_r, args_f, descr)
+            except Exception, e:
+                metainterp.execute_raised(e)
+            return None
+        raise AssertionError("bad rettype")
+    return do_call
 
-do_call_r = do_call
-do_call_i = do_call
-do_call_f = do_call
-do_call_n = do_call
-do_call_loopinvariant_r = do_call
-do_call_loopinvariant_i = do_call
-do_call_loopinvariant_f = do_call
-do_call_loopinvariant_n = do_call
-do_call_may_force_r = do_call
-do_call_may_force_i = do_call
-do_call_may_force_f = do_call
-do_call_may_force_n = do_call
+do_call_r = new_do_call("r")
+do_call_i = new_do_call("i")
+do_call_f = new_do_call("f")
+do_call_n = new_do_call("v")
+do_call_loopinvariant_r = do_call_r
+do_call_loopinvariant_i = do_call_i
+do_call_loopinvariant_f = do_call_f
+do_call_loopinvariant_n = do_call_n
+do_call_may_force_r = do_call_r
+do_call_may_force_i = do_call_i
+do_call_may_force_f = do_call_f
+do_call_may_force_n = do_call_n
 
 def do_cond_call(cpu, metainterp, argboxes, descr):
     condbox = argboxes[0]
@@ -451,27 +451,26 @@
 execute._annspecialcase_ = 'specialize:arg(2)'
 
 def execute_varargs(cpu, metainterp, opnum, argboxes, descr):
-    xxxx
     # only for opnums with a variable arity (calls, typically)
     check_descr(descr)
     func = get_execute_function(opnum, -1, True)
     return func(cpu, metainterp, argboxes, descr)
 execute_varargs._annspecialcase_ = 'specialize:arg(2)'
 
-
+ at specialize.argtype(0)
+def wrap_constant(value):
+    if isinstance(value, int):
+        return ConstInt(value)
+    elif isinstance(value, float):
+        return ConstFloat(value)
+    else:
+        assert lltype.typeOf(value) == llmemory.GCREF
+        return ConstPtr(value)
+    
 def execute_nonspec_const(cpu, metainterp, opnum, argboxes, descr=None,
                           type='i'):
-    if type == 'i':
-        return ConstInt(_execute_nonspec(cpu, metainterp, opnum, argboxes,
-                                         descr, 'i'))
-    elif type == 'f':
-        return ConstFloat(_execute_nonspec(cpu, metainterp, opnum, argboxes,
-                                         descr, 'f'))
-    elif type == 'r':
-        return ConstPtr(_execute_nonspec(cpu, metainterp, opnum, argboxes,
-                                         descr, 'r'))
-    else:
-        assert False
+    return wrap_constant(_execute_nonspec(cpu, metainterp, opnum, argboxes,
+                                          descr, type))
 
 @specialize.arg(5)
 def _execute_nonspec(cpu, metainterp, opnum, argboxes, descr=None, type='i'):
diff --git a/rpython/jit/metainterp/heapcache.py b/rpython/jit/metainterp/heapcache.py
--- a/rpython/jit/metainterp/heapcache.py
+++ b/rpython/jit/metainterp/heapcache.py
@@ -141,7 +141,7 @@
             rop._NOSIDEEFFECT_FIRST <= opnum <= rop._NOSIDEEFFECT_LAST or
             rop._GUARD_FIRST <= opnum <= rop._GUARD_LAST):
             return
-        if opnum == rop.CALL or opnum == rop.CALL_LOOPINVARIANT or opnum == rop.COND_CALL:
+        if rop._CALL_FIRST <= opnum <= rop._CALL_LAST:
             effectinfo = descr.get_extra_info()
             ef = effectinfo.extraeffect
             if (ef == effectinfo.EF_LOOPINVARIANT or
diff --git a/rpython/jit/metainterp/optimizeopt/optimizer.py b/rpython/jit/metainterp/optimizeopt/optimizer.py
--- a/rpython/jit/metainterp/optimizeopt/optimizer.py
+++ b/rpython/jit/metainterp/optimizeopt/optimizer.py
@@ -354,17 +354,6 @@
             return rop.GETARRAYITEM_GC_PURE_F
         return rop.GETARRAYITEM_GC_PURE_I
 
-    def call_for_descr(self, descr):
-        tp = descr.get_result_type()
-        if tp == 'i':
-            return rop.CALL_I
-        elif tp == 'r':
-            return rop.CALL_R
-        elif tp == 'f':
-            return rop.CALL_F
-        assert tp == 'v'
-        return rop.CALL_N
-
     def setup(self):
         pass
 
diff --git a/rpython/jit/metainterp/pyjitpl.py b/rpython/jit/metainterp/pyjitpl.py
--- a/rpython/jit/metainterp/pyjitpl.py
+++ b/rpython/jit/metainterp/pyjitpl.py
@@ -13,7 +13,7 @@
 from rpython.jit.metainterp.logger import Logger
 from rpython.jit.metainterp.optimizeopt.util import args_dict
 from rpython.jit.metainterp.resoperation import rop, InputArgInt,\
-     InputArgFloat, InputArgRef
+     InputArgFloat, InputArgRef, OpHelpers
 from rpython.rlib import nonconst, rstack
 from rpython.rlib.debug import debug_start, debug_stop, debug_print
 from rpython.rlib.debug import have_debug_prints, make_sure_not_resized
@@ -602,12 +602,17 @@
         return indexbox
 
     @arguments("box", "descr")
-    def _opimpl_getfield_gc_any(self, box, fielddescr):
+    def opimpl_getfield_gc_i(self, box, fielddescr):
         return self._opimpl_getfield_gc_any_pureornot(
-                rop.GETFIELD_GC, box, fielddescr)
-    opimpl_getfield_gc_i = _opimpl_getfield_gc_any
-    opimpl_getfield_gc_r = _opimpl_getfield_gc_any
-    opimpl_getfield_gc_f = _opimpl_getfield_gc_any
+                rop.GETFIELD_GC_I, box, fielddescr, 'i')
+    @arguments("box", "descr")
+    def opimpl_getfield_gc_r(self, box, fielddescr):
+        return self._opimpl_getfield_gc_any_pureornot(
+                rop.GETFIELD_GC_R, box, fielddescr, 'r')
+    @arguments("box", "descr")
+    def opimpl_getfield_gc_f(self, box, fielddescr):
+        return self._opimpl_getfield_gc_any_pureornot(
+                rop.GETFIELD_GC_F, box, fielddescr, 'f')
 
     @arguments("box", "descr")
     def _opimpl_getfield_gc_pure_any(self, box, fielddescr):
@@ -630,15 +635,21 @@
     opimpl_getinteriorfield_gc_f = _opimpl_getinteriorfield_gc_any
     opimpl_getinteriorfield_gc_r = _opimpl_getinteriorfield_gc_any
 
-    @specialize.arg(1)
-    def _opimpl_getfield_gc_any_pureornot(self, opnum, box, fielddescr):
+    @specialize.arg(1, 4)
+    def _opimpl_getfield_gc_any_pureornot(self, opnum, box, fielddescr, type):
         tobox = self.metainterp.heapcache.getfield(box, fielddescr)
         if tobox is not None:
             # sanity check: see whether the current struct value
             # corresponds to what the cache thinks the value is
-            resbox = executor.execute(self.metainterp.cpu, self.metainterp,
-                                      rop.GETFIELD_GC, fielddescr, box)
-            assert resbox.constbox().same_constant(tobox.constbox())
+            resvalue = executor.execute(self.metainterp.cpu, self.metainterp,
+                                        opnum, fielddescr, box)
+            if type == 'i':
+                assert resvalue == tobox.getint()
+            elif type == 'r':
+                assert resvalue == tobox.getref_base()
+            else:
+                assert type == 'f'
+                assert resvalue == tobox.getfloatstorage()
             return tobox
         resbox = self.execute_with_descr(opnum, fielddescr, box)
         self.metainterp.heapcache.getfield_now_known(box, fielddescr, resbox)
@@ -1460,7 +1471,8 @@
                                                 descr, False, False)
                 exc = effectinfo.check_can_raise()
                 pure = effectinfo.check_is_elidable()
-                return self.execute_varargs(rop.CALL, allboxes, descr, exc, pure)
+                opnum = OpHelpers.call_for_descr(descr)
+                return self.execute_varargs(opnum, allboxes, descr, exc, pure)
         finally:
             debug_stop("jit-residual-call")
 
@@ -1896,16 +1908,15 @@
         # execute the operation
         profiler = self.staticdata.profiler
         profiler.count_ops(opnum)
-        xxx
-        resbox = executor.execute_varargs(self.cpu, self,
-                                          opnum, argboxes, descr)
+        resvalue = executor.execute_varargs(self.cpu, self,
+                                            opnum, argboxes, descr)
         # check if the operation can be constant-folded away
         argboxes = list(argboxes)
         if rop._ALWAYS_PURE_FIRST <= opnum <= rop._ALWAYS_PURE_LAST:
-            resbox = self._record_helper_pure_varargs(opnum, resbox, descr, argboxes)
-        else:
-            resbox = self._record_helper_nonpure_varargs(opnum, resbox, descr, argboxes)
-        return resbox
+            return self._record_helper_pure_varargs(opnum, resvalue, descr,
+                                                    argboxes)
+        return self._record_helper_nonpure_varargs(opnum, resvalue, descr,
+                                                   argboxes)
 
     @specialize.argtype(2)
     def _record_helper_pure(self, opnum, resvalue, descr, *argboxes):
@@ -1917,15 +1928,13 @@
                                                        list(argboxes))
 
     @specialize.argtype(2)
-    def _record_helper_pure_varargs(self, opnum, resbox, descr, argboxes):
-        xxx
+    def _record_helper_pure_varargs(self, opnum, resvalue, descr, argboxes):
         canfold = self._all_constants_varargs(argboxes)
         if canfold:
-            resbox = resbox.constbox()       # ensure it is a Const
-            return resbox
+            return executor.wrap_constant(resvalue)
         else:
-            resbox = resbox.nonconstbox()    # ensure it is a Box
-            return self._record_helper_nonpure_varargs(opnum, resbox, descr, argboxes)
+            return self._record_helper_nonpure_varargs(opnum, resvalue, descr,
+                                                       argboxes)
 
     @specialize.argtype(2)
     def _record_helper_nonpure_varargs(self, opnum, resvalue, descr, argboxes):
@@ -1939,7 +1948,8 @@
         self.heapcache.invalidate_caches(opnum, descr, argboxes)
         op = self.history.record(opnum, argboxes, resvalue, descr)
         self.attach_debug_info(op)
-        return op
+        if op.type != 'v':
+            return op
 
     def execute_new_with_vtable(self, known_class):
         resbox = self.execute_and_record(rop.NEW_WITH_VTABLE, None,
diff --git a/rpython/jit/metainterp/resoperation.py b/rpython/jit/metainterp/resoperation.py
--- a/rpython/jit/metainterp/resoperation.py
+++ b/rpython/jit/metainterp/resoperation.py
@@ -867,3 +867,16 @@
         op.setarg = setarg
         op.setdescr = setdescr
     return newops
+
+class OpHelpers(object):
+    @staticmethod
+    def call_for_descr(descr):
+        tp = descr.get_result_type()
+        if tp == 'i':
+            return rop.CALL_I
+        elif tp == 'r':
+            return rop.CALL_R
+        elif tp == 'f':
+            return rop.CALL_F
+        assert tp == 'v'
+        return rop.CALL_N
diff --git a/rpython/jit/metainterp/resume.py b/rpython/jit/metainterp/resume.py
--- a/rpython/jit/metainterp/resume.py
+++ b/rpython/jit/metainterp/resume.py
@@ -1,13 +1,13 @@
 from rpython.jit.codewriter.effectinfo import EffectInfo
 from rpython.jit.metainterp import jitprof
-from rpython.jit.metainterp.history import (Box, Const, ConstInt, getkind,
-    BoxInt, BoxPtr, BoxFloat, INT, REF, FLOAT, AbstractDescr)
-from rpython.jit.metainterp.resoperation import rop
+from rpython.jit.metainterp.history import (Const, ConstInt, getkind,
+    INT, REF, FLOAT, AbstractDescr)
+from rpython.jit.metainterp.resoperation import rop, InputArgInt,\
+     InputArgFloat, InputArgRef
 from rpython.rlib import rarithmetic, rstack
 from rpython.rlib.objectmodel import (we_are_translated, specialize,
-        compute_unique_id, import_from_mixin)
-from rpython.rlib.debug import (have_debug_prints, ll_assert, debug_start,
-    debug_stop, debug_print)
+        compute_unique_id)
+from rpython.rlib.debug import ll_assert, debug_print
 from rpython.rtyper import annlowlevel
 from rpython.rtyper.lltypesystem import lltype, llmemory, rffi, rstr
 from rpython.rtyper.rclass import OBJECTPTR
@@ -1160,11 +1160,11 @@
             num += len(self.liveboxes)
             assert num >= 0
         if kind == INT:
-            box = BoxInt(self.cpu.get_int_value(self.deadframe, num))
+            box = InputArgInt(self.cpu.get_int_value(self.deadframe, num))
         elif kind == REF:
-            box = BoxPtr(self.cpu.get_ref_value(self.deadframe, num))
+            box = InputArgRef(self.cpu.get_ref_value(self.deadframe, num))
         elif kind == FLOAT:
-            box = BoxFloat(self.cpu.get_float_value(self.deadframe, num))
+            box = InputArgFloat(self.cpu.get_float_value(self.deadframe, num))
         else:
             assert 0, "bad kind: %d" % ord(kind)
         self.liveboxes[num] = box

From noreply at buildbot.pypy.org  Fri Nov 21 09:51:30 2014
From: noreply at buildbot.pypy.org (hakanardo)
Date: Fri, 21 Nov 2014 09:51:30 +0100 (CET)
Subject: [pypy-commit] extradoc extradoc: numpy versions of these benchmarks
Message-ID: <20141121085130.0F4711C012A@cobra.cs.uni-duesseldorf.de>

Author: Hakan Ardo 
Branch: extradoc
Changeset: r5464:0fbd61901330
Date: 2014-11-21 09:50 +0100
http://bitbucket.org/pypy/extradoc/changeset/0fbd61901330/

Log:	numpy versions of these benchmarks

diff --git a/talk/dls2012/benchmarks/benchmark.sh b/talk/dls2012/benchmarks/benchmark.sh
--- a/talk/dls2012/benchmarks/benchmark.sh
+++ b/talk/dls2012/benchmarks/benchmark.sh
@@ -59,13 +59,21 @@
     #$* ./runner.py $EXTRA_OPTS convolution/convolution.py conv3 1
     #$* ./runner.py $EXTRA_OPTS convolution/convolution.py conv5 1
     $* ./runner.py $EXTRA_OPTS convolution/convolution.py conv3 100
+    $* ./runner.py $EXTRA_OPTS convolution/convolution.py conv3_numpy 100
     $* ./runner.py $EXTRA_OPTS convolution/convolution.py conv5 100
+    $* ./runner.py $EXTRA_OPTS convolution/convolution.py conv5_numpy 100
     $* ./runner.py $EXTRA_OPTS convolution/convolution.py conv3 1000
+    $* ./runner.py $EXTRA_OPTS convolution/convolution.py conv3_numpy 1000
     $* ./runner.py $EXTRA_OPTS convolution/convolution.py conv5 1000
+    $* ./runner.py $EXTRA_OPTS convolution/convolution.py conv5_numpy 1000
     $* ./runner.py $EXTRA_OPTS convolution/convolution.py conv3x3 1000000 3
+    $* ./runner.py $EXTRA_OPTS convolution/convolution.py conv3x3_numpy 1000000 3
     $* ./runner.py $EXTRA_OPTS convolution/convolution.py conv3x3 1000 1000
+    $* ./runner.py $EXTRA_OPTS convolution/convolution.py conv3x3_numpy 1000 1000
     $* ./runner.py $EXTRA_OPTS convolution/convolution.py dilate3x3 1000 1000
+    $* ./runner.py $EXTRA_OPTS convolution/convolution.py dilate3x3_numpy 1000 1000
     $* ./runner.py $EXTRA_OPTS convolution/convolution.py sobel_magnitude 1000 1000
+    $* ./runner.py $EXTRA_OPTS convolution/convolution.py sobel_magnitude_numpy 1000 1000
     #$* ./runner.py $EXTRA_OPTS image/noborder.py main NoBorderImagePadded
     #$* ./runner.py $EXTRA_OPTS image/noborder.py main NoBorderImagePadded iter
     #$* ./runner.py $EXTRA_OPTS image/noborder.py main NoBorderImagePadded range
diff --git a/talk/dls2012/benchmarks/convolution/convolution.py b/talk/dls2012/benchmarks/convolution/convolution.py
--- a/talk/dls2012/benchmarks/convolution/convolution.py
+++ b/talk/dls2012/benchmarks/convolution/convolution.py
@@ -1,5 +1,13 @@
 from array import array
 from math import log10, sqrt
+try:
+    import numpy as np
+except ImportError:
+    try:
+        import numpypy as np
+    except ImportError:
+        print "Cant find nympy"
+
 
 def _conv3(a, k, n=1):
     assert len(k)==3
@@ -14,7 +22,22 @@
     n = int(args[0])
     _conv3(array('d', [1]) * (100000000/n),
            array('d', [-1, 0, 1]), n)
-    return 'conv3(array(1e%d))' % log10(100000000/n)
+    return 'conv3(array.array(1e%d))' % log10(100000000/n)
+
+def _conv3_numpy(a, k, n=1):
+    assert len(k)==3
+    b = np.zeros(len(a) - 2, a.dtype)
+    while n:
+        n -= 1
+        for i in xrange(len(b)):
+            b[i] = k[2]*a[i] + k[1]*a[i+1] + k[0]*a[i+2]
+    return b
+
+def conv3_numpy(args):
+    n = int(args[0])
+    _conv3_numpy(np.ones(100000000/n, 'd'),
+           np.array([-1, 0, 1], 'd'), n)
+    return 'conv3(numpy.array(1e%d))' % log10(100000000/n)
 
 def _conv5(a, k, n=1):
     assert len(k)==5
@@ -29,7 +52,22 @@
     n = int(args[0])
     _conv5(array('d', [1]) * (100000000/n),
            array('d', [1, 4, 6, 4, 1]), n)
-    return 'conv5(array(1e%d))' % log10(100000000/n)
+    return 'conv5(array.array(1e%d))' % log10(100000000/n)
+
+def _conv5_numpy(a, k, n=1):
+    assert len(k)==5
+    b = np.zeros(len(a) - 4, a.dtype)
+    while n:
+        n -= 1
+        for i in xrange(len(b)):
+            b[i] = k[4]*a[i] + k[3]*a[i+1] + k[2]*a[i+2] + k[1]*a[i+3] + k[0]*a[i+4]
+    return b
+
+def conv5_numpy(args):
+    n = int(args[0])
+    _conv5_numpy(np.ones(100000000/n, 'd'),
+           np.array([1, 4, 6, 4, 1], 'd'), n)
+    return 'conv5(numpy.array(1e%d))' % log10(100000000/n)
 
 class Array2D(object):
     def __init__(self, w, h, data=None):
@@ -71,13 +109,16 @@
     def __init__(self, w, h):
         self.width = w
         self.height = h
-        import numpypy
-        self.data = numpypy.zeros([h, w], 'd')
+        self.data = np.zeros([h, w], 'd')
 
     def __getitem__(self, (x, y)):
+        if x < 0 or y < 0:
+            raise IndexError
         return self.data[y, x]
 
     def __setitem__(self, (x, y), val):
+        if x < 0 or y < 0:
+            raise IndexError
         self.data[y, x] = val
 
 def _conv3x3(a, b, k):
@@ -125,6 +166,13 @@
         _dilate3x3(a, b, Array2D(3,3))
     return 'dilate3x3(Array2D(%sx%s))' % tuple(args)
 
+def dilate3x3_numpy(args):
+    a = NumpyArray(int(args[0]), int(args[1]))
+    b = NumpyArray(a.width, a.height)
+    for i in range(10):
+        _dilate3x3(a, b, NumpyArray(3,3))
+    return 'dilate3x3(NumpyArray(%sx%s))' % tuple(args)
+
 def _sobel_magnitude(a):
     b = Array2D(a.width, a.height)    
     for y in xrange(1, a.height-1):
@@ -141,3 +189,8 @@
     for i in range(10):
         _sobel_magnitude(Array2D(int(args[0]), int(args[1])))
     return 'sobel(Array2D(%sx%s))' % tuple(args)
+
+def sobel_magnitude_numpy(args):
+    for i in range(10):
+        _sobel_magnitude(NumpyArray(int(args[0]), int(args[1])))
+    return 'sobel(NumpyArray(%sx%s))' % tuple(args)
diff --git a/talk/dls2012/benchmarks/iter/ndindex.py b/talk/dls2012/benchmarks/iter/ndindex.py
new file mode 100644
--- /dev/null
+++ b/talk/dls2012/benchmarks/iter/ndindex.py
@@ -0,0 +1,145 @@
+from numpy import ndindex, array, ones, tile
+
+range1 = range2 = ndindex
+
+def _sum1d(a):
+    sa = 0
+    for i, in range1(len(a)):
+        sa += a[i]
+
+def _xsum1d(a):
+    sa = 0
+    for i, in range1(len(a)):
+        sa += a[i] + i
+
+def _wsum1d(a):
+    sa = 0
+    for i, in range1(len(a)):
+        sa += a[i] + len(a)
+
+def _sum2d(a, w, h):
+    sa = 0
+    for x, y in range2(w, h):
+        sa += a[y, x]
+
+def _wsum2d(a, w, h):
+    sa = 0
+    for x, y in range2(w, h):
+        sa += a[y, x] + w
+
+def _xsum2d(a, w, h):
+    sa = 0
+    for x, y in range2(w, h):
+        sa += a[y, x] + x
+
+def _whsum2d(a, w, h):
+    sa = 0
+    for x, y in range2(w, h):
+        sa += a[y, x] + w + h
+
+def _xysum2d(a, w, h):
+    sa = 0
+    for x, y in range2(w, h):
+        sa += a[y, x] + x + y
+
+def _mean1d(a):
+    sa = 0
+    for i, in range1(len(a)):
+        sa = (i*sa + a[i])/(i + 1.0);
+        
+def _median1d(a):
+    sa = 0
+    for i, in range1(len(a)):
+        if sa > a[i]:
+            sa -= 1.0/(i + 1.0)
+        elif sa < a[i]:
+            sa += 1.0/(i + 1.0)
+
+def _ripple1d(a):
+    sa = 0
+    for i, in range1(len(a)):
+        if sa > a[i]:
+            sa -= 0.1
+        elif sa < a[i]:
+            sa += 0.1
+
+def _ripple2d(a, w, h):
+    sa = 0
+    for x, y in range2(w, h):
+        if sa > a[y, x]:
+            sa -= 0.1
+        elif sa < a[y, x]:
+            sa += 0.1
+
+def sum1d(args):
+    run1d(args, _sum1d)
+    return "sum1d"
+
+def xsum1d(args):
+    run1d(args, _xsum1d)
+    return "xsum1d"
+
+def wsum1d(args):
+    run1d(args, _wsum1d)
+    return "wsum1d"
+
+def sum2d(args):
+    run2d(args, _sum2d)
+    return "sum2d"
+
+def wsum2d(args):
+    run2d(args, _wsum2d)
+    return "wsum2d"
+
+def xsum2d(args):
+    run2d(args, _xsum2d)
+    return "xsum2d"
+
+def whsum2d(args):
+    run2d(args, _whsum2d)
+    return "whsum2d"
+
+def xysum2d(args):
+    run2d(args, _xysum2d)
+    return "xysum2d"
+
+def mean1d(args):
+    run1d(args, _mean1d, [1, -1])
+    return "mean1d"
+
+def median1d(args):
+    run1d(args, _median1d, [1, -1])
+    return "median1d"
+
+def ripple1d(args):
+    run1d(args, _ripple1d, [1, -1])
+    return "ripple1d"
+
+def ripple2d(args):
+    run2d(args, _ripple2d, [1, -1])
+    return "ripple2d"
+
+def run1d(args, f, data=None):
+    if data:
+        a = tile(array(data), 100000000/len(data))
+    else:
+        a = ones(100000000)
+    n = int(args[0])
+    for i in xrange(n):
+        f(a)
+    return "sum1d"
+
+def run2d(args, f, data=None):
+    if data:
+        a = tile(array(data), 100000000/len(data)).reshape((10000, 10000))
+    else:
+        a = ones(100000000).reshape((10000, 10000))
+    n = int(args[0])
+    for i in xrange(n):
+        f(a, 10000, 10000)
+    return "sum1d"
+
+if __name__ == '__main__':
+    import sys
+    eval(sys.argv[1])(sys.argv[2:])
+    
diff --git a/talk/dls2012/benchmarks/iter/nditer.py b/talk/dls2012/benchmarks/iter/nditer.py
new file mode 100644
--- /dev/null
+++ b/talk/dls2012/benchmarks/iter/nditer.py
@@ -0,0 +1,166 @@
+from numpy import nditer, array, ones, tile
+
+def _sum1d(a):
+    sa = 0
+    it = nditer(a, flags=['f_index'])
+    for v in it:
+        i = it.index    
+        sa += a[i]
+
+def _xsum1d(a):
+    sa = 0
+    it = nditer(a, flags=['f_index'])
+    for v in it:
+        i = it.index    
+        sa += a[i] + i
+
+def _wsum1d(a):
+    sa = 0
+    it = nditer(a, flags=['f_index'])
+    for v in it:
+        i = it.index    
+        sa += a[i] + len(a)
+
+def _sum2d(a, w, h):
+    sa = 0
+    it = nditer(a, flags=['multi_index'])
+    for v in it:
+        y, x = it.multi_index
+        sa += a[y, x]
+
+def _wsum2d(a, w, h):
+    sa = 0
+    it = nditer(a, flags=['multi_index'])
+    for v in it:
+        y, x = it.multi_index
+        sa += a[y, x] + w
+
+def _xsum2d(a, w, h):
+    sa = 0
+    it = nditer(a, flags=['multi_index'])
+    for v in it:
+        y, x = it.multi_index
+        sa += a[y, x] + x
+
+def _whsum2d(a, w, h):
+    sa = 0
+    it = nditer(a, flags=['multi_index'])
+    for v in it:
+        y, x = it.multi_index
+        sa += a[y, x] + w + h
+
+def _xysum2d(a, w, h):
+    sa = 0
+    it = nditer(a, flags=['multi_index'])
+    for v in it:
+        y, x = it.multi_index
+        sa += a[y, x] + x + y
+
+def _mean1d(a):
+    sa = 0
+    it = nditer(a, flags=['f_index'])
+    for v in it:
+        i = it.index    
+        sa = (i*sa + a[i])/(i + 1.0);
+        
+def _median1d(a):
+    sa = 0
+    it = nditer(a, flags=['f_index'])
+    for v in it:
+        i = it.index    
+        if sa > a[i]:
+            sa -= 1.0/(i + 1.0)
+        elif sa < a[i]:
+            sa += 1.0/(i + 1.0)
+
+def _ripple1d(a):
+    sa = 0
+    it = nditer(a, flags=['f_index'])
+    for v in it:
+        i = it.index    
+        if sa > a[i]:
+            sa -= 0.1
+        elif sa < a[i]:
+            sa += 0.1
+
+def _ripple2d(a, w, h):
+    sa = 0
+    it = nditer(a, flags=['multi_index'])
+    for v in it:
+        y, x = it.multi_index
+        if sa > a[y, x]:
+            sa -= 0.1
+        elif sa < a[y, x]:
+            sa += 0.1
+
+def sum1d(args):
+    run1d(args, _sum1d)
+    return "sum1d"
+
+def xsum1d(args):
+    run1d(args, _xsum1d)
+    return "xsum1d"
+
+def wsum1d(args):
+    run1d(args, _wsum1d)
+    return "wsum1d"
+
+def sum2d(args):
+    run2d(args, _sum2d)
+    return "sum2d"
+
+def wsum2d(args):
+    run2d(args, _wsum2d)
+    return "wsum2d"
+
+def xsum2d(args):
+    run2d(args, _xsum2d)
+    return "xsum2d"
+
+def whsum2d(args):
+    run2d(args, _whsum2d)
+    return "whsum2d"
+
+def xysum2d(args):
+    run2d(args, _xysum2d)
+    return "xysum2d"
+
+def mean1d(args):
+    run1d(args, _mean1d, [1, -1])
+    return "mean1d"
+
+def median1d(args):
+    run1d(args, _median1d, [1, -1])
+    return "median1d"
+
+def ripple1d(args):
+    run1d(args, _ripple1d, [1, -1])
+    return "ripple1d"
+
+def ripple2d(args):
+    run2d(args, _ripple2d, [1, -1])
+    return "ripple2d"
+
+def run1d(args, f, data=None):
+    if data:
+        a = tile(array(data), 100000000/len(data))
+    else:
+        a = ones(100000000)
+    n = int(args[0])
+    for i in xrange(n):
+        f(a)
+    return "sum1d"
+
+def run2d(args, f, data=None):
+    if data:
+        a = tile(array(data), 100000000/len(data)).reshape((10000, 10000))
+    else:
+        a = ones(100000000).reshape((10000, 10000))
+    n = int(args[0])
+    for i in xrange(n):
+        f(a, 10000, 10000)
+    return "sum1d"
+
+if __name__ == '__main__':
+    import sys
+    eval(sys.argv[1])(sys.argv[2:])
diff --git a/talk/dls2012/benchmarks/iter/result-2.4.0.txt b/talk/dls2012/benchmarks/iter/result-2.4.0.txt
new file mode 100644
--- /dev/null
+++ b/talk/dls2012/benchmarks/iter/result-2.4.0.txt
@@ -0,0 +1,90 @@
+gcc -O3
+sum1d: 0.83 +- 1.24126707662e-16
+sum2d: 0.83 +- 1.24126707662e-16
+whsum2d: 0.842 +- 0.004472135955
+3wsum1d: 0.836 +- 0.00894427191
+wsum2d: 0.85 +- 0.0308220700148
+xsum1d: 0.842 +- 0.004472135955
+xsum2d: 0.842 +- 0.004472135955
+xysum2d: 1.12 +- 0.0346410161514
+mean1d: 7.428 +- 0.0294957624075
+median1d: 3.818 +- 0.004472135955
+ripple1d: 1.342 +- 0.0109544511501
+ripple2d: 1.336 +- 0.00894427191
+
+pypy iter/generator.py
+sum1d: 5.53084101677 +- 0.00651376226379
+sum2d: 5.7555460453 +- 0.00951369332241
+whsum2d: 5.8612534523 +- 0.0505271222339
+wsum1d: 5.13269457817 +- 0.0823542879822
+wsum2d: 5.99619159698 +- 0.0487867098222
+xsum1d: 5.04685320854 +- 0.0555180883435
+xsum2d: 6.07883496284 +- 0.0389639282491
+xysum2d: 5.83931522369 +- 0.0576320488093
+mean1d: 8.94375824928 +- 0.0108197222492
+median1d: 10.4045877457 +- 0.0285781258496
+ripple1d: 10.2467153549 +- 0.00567696790862
+ripple2d: 11.1841029644 +- 0.139083424095
+
+pypy iter/generator2.py
+sum1d: 5.9797270298 +- 0.0755165051781
+sum2d: 5.80511965752 +- 0.380443555753
+whsum2d: 6.19872779846 +- 0.0262446391517
+wsum1d: 5.0686296463 +- 0.03220581952
+wsum2d: 6.22603621483 +- 0.0765020459155
+xsum1d: 5.03696541786 +- 0.0313417312818
+xsum2d: 6.64942345619 +- 0.0634006175674
+xysum2d: 5.85069346428 +- 0.024646797031
+mean1d: 9.20232362747 +- 0.27107580199
+median1d: 10.5072529793 +- 0.0780365503085
+ripple1d: 10.3291498184 +- 0.0457349492366
+ripple2d: 12.1278275967 +- 0.0184532784891
+
+pypy iter/iterator.py
+sum1d: 2.89783701897 +- 0.0338818402654
+sum2d: 6.21735162735 +- 0.0305362100956
+whsum2d: 5.78359918594 +- 0.0418847806897
+wsum1d: 2.90417222977 +- 0.00550225146282
+wsum2d: 6.1562063694 +- 0.0248465945318
+xsum1d: 3.15220880508 +- 0.0238542345497
+xsum2d: 6.17962999344 +- 0.00522105603458
+xysum2d: 6.54959263802 +- 0.0204275708962
+mean1d: 8.9222530365 +- 0.0569358104413
+median1d: 9.32725701332 +- 0.0118218937952
+ripple1d: 7.84541239738 +- 0.0128802667437
+ripple2d: 11.0337635994 +- 0.0381211395066
+
+pypy iter/range.py
+sum1d: 1.5154399395 +- 0.00218717543831
+sum2d: 1.78169260025 +- 0.0031213150465
+whsum2d: 1.78300223351 +- 0.00343094840578
+wsum1d: 1.51814541817 +- 0.00407496519924
+wsum2d: 1.78870997429 +- 0.0102636422345
+xsum1d: 1.78627576828 +- 0.0141348129086
+xsum2d: 2.04741225243 +- 0.00739649010433
+xysum2d: 2.04407844543 +- 0.00160511612186
+mean1d: 8.38623380661 +- 0.00678559642762
+median1d: 4.52713260651 +- 0.0215238656358
+ripple1d: 3.50191493034 +- 0.0344265982872
+ripple2d: 4.24880318642 +- 0.00425382282478
+
+pypy iter/while.py
+sum1d: 0.987529802322 +- 0.00129671178469
+sum2d: 1.75787782669 +- 0.00201957281403
+whsum2d: 1.75991163254 +- 0.000927917277915
+wsum1d: 0.989894151688 +- 0.00303482111217
+wsum2d: 1.76429200172 +- 0.00496186597436
+xsum1d: 1.23868374825 +- 0.00168004472652
+xsum2d: 2.0280834198 +- 0.00263820277497
+xysum2d: 2.02464160919 +- 0.00174334572372
+mean1d: 8.34133095741 +- 0.0121082272197
+median1d: 4.27610282898 +- 0.0313133386645
+ripple1d: 2.98665003777 +- 0.0134154060239
+ripple2d: 4.04827785492 +- 0.055327379039
+
+pypy iter/ndindex.py
+NotImplementedError: unable to create dtype from objects, "DummyArray" instance not supported
+
+pypy iter/nditer.py
+sum1d: 61.2064362049 +- 0.578041254203
+sum2d: 71.1426748753 +- 1.09482960038
diff --git a/talk/dls2012/benchmarks/runall.sh b/talk/dls2012/benchmarks/runall.sh
--- a/talk/dls2012/benchmarks/runall.sh
+++ b/talk/dls2012/benchmarks/runall.sh
@@ -1,6 +1,8 @@
 #!/bin/bash
 
 #./benchmark.sh pypy
+./benchmark.sh pypy-2.4.0
+./benchmark.sh pypy-2.0
 #./benchmark.sh pypy --jit enable_opts=intbounds:rewrite:virtualize:string:earlyforce:pure:heap:ffi
 #./benchmark.sh pypy-1.5
 #./benchmark.sh pypy-1.5 --jit enable_opts=intbounds:rewrite:virtualize:heap:unroll
@@ -8,10 +10,11 @@
 #./benchmark.sh gcc
 #./benchmark.sh gcc -O2
 #./benchmark.sh gcc -O3 -march=native -fno-tree-vectorize
+./benchmark.sh gcc -O3 -march=native
 #./benchmark.sh python2.7
 #./benchmark.sh python2.6 psyco-wrapper.py
 #./benchmark.sh luajit-2.0.0-beta10
 #./benchmark.sh luajit-2.0.0-beta10 -O-loop
 ./benchmark.sh luajit
-./benchmark.sh luajit -O-loop
+#./benchmark.sh luajit -O-loop
 #./benchmark.sh luajit
diff --git a/talk/dls2012/benchmarks/runiter.sh b/talk/dls2012/benchmarks/runiter.sh
--- a/talk/dls2012/benchmarks/runiter.sh
+++ b/talk/dls2012/benchmarks/runiter.sh
@@ -10,7 +10,7 @@
 for p in iter/*.py; do
     echo pypy $p
     for b in $BENCHMARKS; do
-	/tmp/pypy-trunk ./runner.py -n 5 $p $b 10
+	pypy ./runner.py -n 5 $p $b 10
     done
     echo
 done
\ No newline at end of file

From noreply at buildbot.pypy.org  Fri Nov 21 09:51:31 2014
From: noreply at buildbot.pypy.org (hakanardo)
Date: Fri, 21 Nov 2014 09:51:31 +0100 (CET)
Subject: [pypy-commit] extradoc extradoc: merge
Message-ID: <20141121085131.BDBA41C012A@cobra.cs.uni-duesseldorf.de>

Author: Hakan Ardo 
Branch: extradoc
Changeset: r5465:206c058dcb38
Date: 2014-11-21 09:50 +0100
http://bitbucket.org/pypy/extradoc/changeset/206c058dcb38/

Log:	merge

diff --git a/blog/draft/io-improvements.rst b/blog/draft/io-improvements.rst
new file mode 100644
--- /dev/null
+++ b/blog/draft/io-improvements.rst
@@ -0,0 +1,88 @@
+
+Hello everyone!
+
+We've wrapped up the Warsaw sprint, so I would like to describe some
+branches which have been recently merged and which improved the I/O and the
+GC: `gc_no_cleanup_nursery`_ and `gc-incminimark-pinning`_.
+
+.. _`gc_no_cleanup_nursery`: https://bitbucket.org/pypy/pypy/commits/9e2f7a37c1e2
+.. _`gc-incminimark-pinning`: https://bitbucket.org/pypy/pypy/commits/64017d818038
+
+The first branch was started by Wenzhu Man for her Google Summer of Code
+and finished by Maciej Fijałkowski and Armin Rigo.
+The PyPy GC works by allocating new objects in the young object
+area (the nursery), simply by incrementing a pointer. After each minor
+collection, the nursery has to be cleaned up. For simplicity, the GC used 
+to do it by zeroing the whole nursery.
+
+This approach has bad effects on the cache, since you zero a large piece of
+memory at once and do unnecessary work for things that don't require zeroing
+like large strings. We mitigated the first problem somewhat with incremental
+nursery zeroing, but this branch removes the zeroing completely, thus
+improving the string handling and recursive code (since jitframes don't
+requires zeroed memory either). I measured the effect on two examples: 
+a recursive implementation of  `fibonacci`_ and `gcbench`_,
+to measure GC performance.
+
+.. _`fibonacci`: https://bitbucket.org/pypy/benchmarks/src/69152c2aee7766051aab15735b0b82a46b82b802/own/fib.py?at=default
+.. _`gcbench`: https://bitbucket.org/pypy/benchmarks/src/69152c2aee7766051aab15735b0b82a46b82b802/own/gcbench.py?at=default
+
+The results for fibonacci and gcbench are below (normalized to cpython
+2.7). Benchmarks were run 50 times each (note that the big standard
+deviation comes mostly from the warmup at the beginning, true figures
+are smaller):
+
++----------------+----- ------------+-------------------------+--------------------+
+| benchmark      | CPython          | PyPy 2.4                | PyPy non-zero      |
++----------------+------------------+-------------------------+--------------------+
+| fibonacci      | 4.8+-0.15 (1.0x) | 0.59+-0.07 (8.1x)       | 0.45+-0.07 (10.6x) |
++----------------+------------------+-------------------------+--------------------+
+| gcbench        | 22+-0.36 (1.0x)  | 1.34+-0.28 (16.4x)      | 1.02+-0.15 (21.6x) |
++----------------+------------------+-------------------------+--------------------+
+
+The second branch was done by Gregor Wegberg for his master thesis and finished
+by Maciej Fijałkowski and Armin Rigo. Because of the way it works, the PyPy GC from
+time to time moves the objects in memory, meaning that their address can change.
+Therefore, if you want to pass pointers to some external C function (for
+example, write(2) or read(2)), you need to ensure that the objects they are
+pointing to will not be moved by the GC (e.g. when running a different thread).
+PyPy up to 2.4 solves the problem by copying the data into or from a non-movable buffer, which
+is obviously inefficient.
+The branch introduce the concept of "pinning", which allows us to inform the
+GC that it is not allowed to move a certain object for a short period of time.
+This introduces a bit of extra complexity
+in the garbage collector, but improves the I/O performance quite drastically,
+because we no longer need the extra copy to and from the non-movable buffers.
+
+In `this benchmark`_, which does I/O in a loop,
+we either write a number of bytes from a freshly allocated string into
+/dev/null or read a number of bytes from /dev/full. I'm showing the results
+for PyPy 2.4, PyPy with non-zero-nursery and PyPy with non-zero-nursery and
+object pinning. Those are wall times for cases using ``os.read/os.write``
+and ``file.read/file.write``, normalized against CPython 2.7.
+
+Benchmarks were done using PyPy 2.4 and revisions ``85646d1d07fb`` for
+non-zero-nursery and ``3d8fe96dc4d9`` for non-zero-nursery and pinning.
+The benchmarks were run once, since the standard deviation was small.
+
+XXXX
+
+What we can see is that ``os.read`` and ``os.write`` both improved greatly
+and outperforms CPython now for each combination. ``file`` operations are
+a little more tricky, and while those branches improved the situation a bit,
+the improvement is not as drastic as in ``os`` versions.  It really should not
+be the case and it showcases how our ``file`` buffering is inferior to CPython.
+We plan on removing our own buffering and using ``FILE*`` in C in the near future,
+so we should outperform CPython on those too (since our allocations are cheaper).
+If you look carefully in the benchmark, the write function is copied three times.
+This hack is intended to avoid JIT overspecializing the assembler code, which happens
+because the buffering code was written way before the JIT was done. In fact, our buffering
+is hilariously bad, but if stars align correctly it can be JIT-compiled to something
+that's not half bad. Try removing the hack and seeing how the performance of the last
+benchmark drops :-) Again, this hack should be absolutely unnecessary once we remove
+our own buffering, stay tuned for more.
+
+Cheers,
+fijal
+
+.. _`this benchmark`: https://bitbucket.org/pypy/benchmarks/src/69152c2aee7766051aab15735b0b82a46b82b802/io/iobasic.py?at=default
diff --git a/blog/draft/iobase.png b/blog/draft/iobase.png
new file mode 100644
index 0000000000000000000000000000000000000000..0b8bedc12421ce39c24931d68f8993da74bb2808
GIT binary patch

[cut]

diff --git a/blog/draft/tornado-stm.rst b/blog/draft/tornado-stm.rst
new file mode 100644
--- /dev/null
+++ b/blog/draft/tornado-stm.rst
@@ -0,0 +1,252 @@
+Tornado without a GIL on PyPy STM
+=================================
+
+Python has a GIL, right? Not quite - PyPy STM is a python implementation
+without a GIL, so it can scale CPU-bound work to several cores.
+PyPy STM is developed by Armin Rigo and Remi Meier,
+and supported by community `donations `_.
+You can read more about it in the
+`docs `_.
+
+Although PyPy STM is still a work in progress, in many cases it can already
+run CPU-bound code faster than regular PyPy, when using multiple cores.
+Here we will see how to slightly modify Tornado IO loop to use
+`transaction `_
+module.
+This module is `described `_
+in the docs and is really simple to use - please see an example there.
+An event loop of Tornado, or any other asynchronous
+web server, looks like this (with some simplifications)::
+
+    while True:
+        for callback in list(self._callbacks):
+            self._run_callback(callback)
+        event_pairs = self._impl.poll()
+        self._events.update(event_pairs)
+        while self._events:
+            fd, events = self._events.popitem()
+            handler = self._handlers[fd]
+            self._handle_event(fd, handler, events)
+
+We get IO events, and run handlers for all of them, these handlers can
+also register new callbacks, which we run too. When using such a framework,
+it is very nice to have a guaranty that all handlers are run serially,
+so you do not have to put any locks. This is an ideal case for the
+transaction module - it gives us guaranties that things appear
+to be run serially, so in user code we do not need any locks. We just
+need to change the code above to something like::
+
+    while True:
+        for callback in list(self._callbacks):
+            transaction.add(
+            self._run_callback, callback)   # added
+        transaction.run()                   # added
+        event_pairs = self._impl.poll()
+        self._events.update(event_pairs)
+        while self._events:
+            fd, events = self._events.popitem()
+            handler = self._handlers[fd]
+            transaction.add(                # added
+                self._handle_event, fd, handler, events)
+        transaction.run()                   # added
+
+The actual commit is
+`here `_,
+- we had to extract a little function to run the callback.
+
+Part 1: a simple benchmark: primes
+----------------------------------
+
+Now we need a simple benchmark, lets start with
+`this `_
+- just calculate a list of primes up to the given number, and return it
+as JSON::
+
+    def is_prime(n):
+        for i in xrange(2, n):
+            if n % i == 0:
+                return False
+        return True
+
+    class MainHandler(tornado.web.RequestHandler):
+        def get(self, num):
+            num = int(num)
+            primes = [n for n in xrange(2, num + 1) if is_prime(n)]
+            self.write({'primes': primes})
+
+
+We can benchmark it with ``siege``::
+
+    siege -c 50 -t 20s http://localhost:8888/10000
+
+But this does not scale. The CPU load is at 101-104 %, and we handle 30 %
+less request per second. The reason for the slowdown is STM overhead,
+which needs to keep track of all writes and reads in order to detect conflicts.
+And the reason for using only one core is, obviously, conflicts!
+Fortunately, we can see what this conflicts are, if we run code like this
+(here 4 is the number of cores to use)::
+
+    PYPYSTM=stm.log ./primes.py 4
+
+Then we can use `print_stm_log.py `_
+to analyse this log. It lists the most expensive conflicts::
+
+    14.793s lost in aborts, 0.000s paused (1258x STM_CONTENTION_INEVITABLE)
+    File "/home/ubuntu/tornado-stm/tornado/tornado/httpserver.py", line 455, in __init__
+        self._start_time = time.time()
+    File "/home/ubuntu/tornado-stm/tornado/tornado/httpserver.py", line 455, in __init__
+        self._start_time = time.time()
+    ...
+
+There are only three kinds of conflicts, they are described in
+`stm source `_,
+Here we see that two threads call into external function to get current time,
+and we can not rollback any of them, so one of them must wait till the other
+transaction finishes.
+For now we can hack around this by disabling this timing - this is only
+needed for internal profiling in tornado.
+
+If we do it, we get the following results (but see caveats below):
+
+============  =========
+Impl.           req/s
+============  =========
+PyPy 2.4        14.4
+------------  ---------
+CPython 2.7      3.2
+------------  ---------
+PyPy-STM 1       9.3
+------------  ---------
+PyPy-STM 2      16.4
+------------  ---------
+PyPy-STM 3      20.4
+------------  ---------
+PyPy STM 4      24.2
+============  =========
+
+.. image:: results-1.png
+
+As we can see, in this benchmark PyPy STM using just two cores
+can beat regular PyPy!
+This is not linear scaling, there are still conflicts left, and this
+is a very simple example but still, it works!
+
+But its not that simple yet :)
+
+First, these are best-case numbers after long (much longer than for regular
+PyPy) warmup. Second, it can sometimes crash (although removing old pyc files
+fixes it). Third, benchmark meta-parameters are also tuned.
+
+Here we get relatively good results only when there are a lot of concurrent
+clients - as a results, a lot of requests pile up, the server is not keeping
+with the load, and transaction module is busy with work running this piled up
+requests. If we decrease the number of concurrent clients, results get slightly worse.
+Another thing we can tune is how heavy is each request - again, if we ask
+primes up to a lower number, then less time is spent doing calculations,
+more time is spent in tornado, and results get much worse.
+
+Besides the ``time.time()`` conflict described above, there are a lot of others.
+The bulk of time is lost in these two conflicts::
+
+    14.153s lost in aborts, 0.000s paused (270x STM_CONTENTION_INEVITABLE)
+    File "/home/ubuntu/tornado-stm/tornado/tornado/web.py", line 1082, in compute_etag
+        hasher = hashlib.sha1()
+    File "/home/ubuntu/tornado-stm/tornado/tornado/web.py", line 1082, in compute_etag
+        hasher = hashlib.sha1()
+
+    13.484s lost in aborts, 0.000s paused (130x STM_CONTENTION_WRITE_READ)
+    File "/home/ubuntu/pypy/lib_pypy/transaction.py", line 164, in _run_thread
+        got_exception)
+
+The first one is presumably calling into some C function from stdlib, and we get
+the same conflict as for ``time.time()`` above, but is can be fixed on PyPy
+side, as we can be sure that computing sha1 is pure.
+
+It is easy to hack around this one too, just removing etag support, but if
+we do it, performance is much worse, only slightly faster than regular PyPy,
+with the top conflict being::
+
+    83.066s lost in aborts, 0.000s paused (459x STM_CONTENTION_WRITE_WRITE)
+    File "/home/arigo/hg/pypy/stmgc-c7/lib-python/2.7/_weakrefset.py", line 70, in __contains__
+    File "/home/arigo/hg/pypy/stmgc-c7/lib-python/2.7/_weakrefset.py", line 70, in __contains__
+
+**FIXME** why does it happen?
+
+The second conflict (without etag tweaks) originates
+in the transaction module, from this piece of code::
+
+    while True:
+        self._do_it(self._grab_next_thing_to_do(tloc_pending),
+                    got_exception)
+        counter[0] += 1
+
+**FIXME** why does it happen?
+
+Tornado modification used in this blog post is based on 3.2.dev2.
+As of now, the latest version is 4.0.2, and if we
+`apply `_
+the same changes to this version, then we no longer get any scaling on this benchmark,
+and there are no conflicts that take any substantial time.
+
+
+Part 2: a more interesting benchmark: A-star
+--------------------------------------------
+
+Although we have seen that PyPy STM is not all moonlight and roses,
+it is interesting to see how it works on a more realistic application.
+
+`astar.py `_
+is a simple game where several players move on a map
+(represented as a list of lists of integers),
+build and destroy walls, and ask server to give them
+shortest paths between two points
+using A-star search, adopted from `ActiveState recipie `_.
+
+The benchmark `bench_astar.py `_
+is simulating players, and tries to put the main load on A-star search,
+but also does some wall building and destruction. There are no locks
+around map modifications, as normal tornado is executing all callbacks
+serially, and we can keep this guaranty with atomic blocks of PyPy STM.
+This is also an example of a program that is not trivial
+to scale to multiple cores with separate processes (assuming
+more interesting shared state and logic).
+
+This benchmark is very noisy due to randomness of client interactions
+(also it could be not linear), so just lower and upper bounds for
+number of requests are reported
+
+============  ==========
+Impl.           req/s
+============  ==========
+PyPy 2.4        5 .. 7
+------------  ----------
+CPython 2.7   0.5 .. 0.9
+------------  ----------
+PyPy-STM 1      2 .. 4
+------------  ----------
+PyPy STM 4      2 .. 6
+============  ==========
+
+Clearly this is a very benchmark, but still we can see that scaling is worse
+and STM overhead is sometimes higher.
+The bulk of conflicts come from the transaction module (we have seen it
+above)::
+
+    91.655s lost in aborts, 0.000s paused (249x STM_CONTENTION_WRITE_READ)
+    File "/home/ubuntu/pypy/lib_pypy/transaction.py", line 164, in _run_thread
+        got_exception)
+
+
+Although it is definitely not ready for production use, you can already try
+to run things, report bugs, and see what is missing in user-facing tools
+and libraries.
+
+
+Benchmarks setup:
+
+* Amazon c3.xlarge (4 cores) running Ubuntu 14.04
+* pypy-c-r74011-stm-jit for the primes benchmark (but it has more bugs
+  than more recent versions), and
+  `pypy-c-r74378-74379-stm-jit `_
+  for astar benchmark (put it inside pypy source checkout at 38c9afbd253c)
+* http://bitbucket.org/kostialopuhin/tornado-stm-bench at 65144cda7a1f
diff --git a/sprintinfo/warsaw-2014/announcement.txt b/sprintinfo/warsaw-2014/announcement.txt
--- a/sprintinfo/warsaw-2014/announcement.txt
+++ b/sprintinfo/warsaw-2014/announcement.txt
@@ -38,8 +38,8 @@
 ------------
 
 The sprint will happen within a room of Warsaw University.  The
-address is Pasteura 5 (which is a form of "Pasteur street"), room 550.
-The person of contact is Maciej Fijalkowski.
+address is Pasteura 5 (which is a form of "Pasteur street"), dept. of
+Physics, room 450.  The person of contact is Maciej Fijalkowski.
 
 
 --------------
diff --git a/sprintinfo/warsaw-2014/people.txt b/sprintinfo/warsaw-2014/people.txt
--- a/sprintinfo/warsaw-2014/people.txt
+++ b/sprintinfo/warsaw-2014/people.txt
@@ -9,5 +9,12 @@
 ==================== ============== =======================
     Name              Arrive/Depart     Accomodation 
 ==================== ============== =======================
-Armin Rigo           20/10-2X/10    with fijal?
+Armin Rigo           20/10-28/10    with fijal
+Maciej Fijalkowski   20/10-30/10    private
+Romain Guillebert    19/10-26-10    ibis Reduta with mjacob
+Manuel Jacob         20/10-26/10    ibis Reduta with rguillebert
+Kostia Lopuhin
+Antonio Cuni         20/10-26/10    ibis Reduta http://www.ibis.com/gb/hotel-7125-ibis-warszawa-reduta/index.shtml
+Matti Picus          20/10-20/10    just a long layover between flights
+Ronan Lamy           19/10-26/10    ibis Reduta
 ==================== ============== =======================
diff --git a/sprintinfo/warsaw-2014/planning.txt b/sprintinfo/warsaw-2014/planning.txt
new file mode 100644
--- /dev/null
+++ b/sprintinfo/warsaw-2014/planning.txt
@@ -0,0 +1,41 @@
+Topics
+======
+
+* cffi.verify dlopen flag - TO BE MERGED
+
+* PyPy/CPython Bridge (Romain, kostia) - MORE PROGRESS
+
+* Profiler (Antonio, Armin) - IN PROGRESS
+
+* Merge improve-docs (Manuel, Ronan) - IN PROGRESS
+
+* Merge kill-multimethod remove-remaining-smm  (Manuel, Antonio, fijal) - MERGED remove-remaining-smm, kill-multimethod WAITING FOR REVIEW
+
+* STM presentation (Everybody) - DONE
+
+* Refactor annotator/rtyper (Ronan?) - LOOKING FOR PAIRING
+
+* Python 3.3 - IN PROGRESS
+
+* look into merging gc pinning (fijal, arigo) - ALMOST READY, more debugging needed
+
+* investigate -fPIC slowdown (fijal, arigo) - IN PROGRESS, complete mess
+
+* NumPyPy discussion (everybody) DONE
+
+* Trying stuff on PyPy-STM (Antonio, Kostia)
+
+* convincing anto why resume refactoring is a good idea
+
+* switchify chains of guard_value (Armin, Romain...)
+
+People
+=====
+
+Antonio
+Armin
+Kostia
+Ronan
+Romain
+Manuel
+Maciej
diff --git a/talk/img/baroquesoftware.png b/talk/img/baroquesoftware.png
new file mode 100644
index 0000000000000000000000000000000000000000..038e80b25722d7917a1fbecb581ce25b54707ab2
GIT binary patch

[cut]

diff --git a/talk/pyconie2014/Makefile b/talk/pyconie2014/Makefile
new file mode 100644
--- /dev/null
+++ b/talk/pyconie2014/Makefile
@@ -0,0 +1,18 @@
+# you can find rst2beamer.py here:
+# https://bitbucket.org/antocuni/env/raw/default/bin/rst2beamer.py
+
+# WARNING: to work, it needs this patch for docutils
+# https://sourceforge.net/tracker/?func=detail&atid=422032&aid=1459707&group_id=38414
+
+talk.pdf: talk.rst author.latex stylesheet.latex
+	python `which rst2beamer.py` --stylesheet=stylesheet.latex --documentoptions=14pt talk.rst talk.latex || exit
+	#/home/antocuni/.virtualenvs/rst2beamer/bin/python `which rst2beamer.py` --stylesheet=stylesheet.latex --documentoptions=14pt talk.rst talk.latex || exit
+	sed 's/\\date{}/\\input{author.latex}/' -i talk.latex || exit
+	#sed 's/\\maketitle/\\input{title.latex}/' -i talk.latex || exit
+	pdflatex talk.latex  || exit
+
+view: talk.pdf
+	evince talk.pdf &
+
+xpdf: talk.pdf
+	xpdf talk.pdf &
diff --git a/talk/pyconie2014/author.latex b/talk/pyconie2014/author.latex
new file mode 100644
--- /dev/null
+++ b/talk/pyconie2014/author.latex
@@ -0,0 +1,9 @@
+\definecolor{rrblitbackground}{rgb}{0.0, 0.0, 0.0}
+
+\title[PyPy : A fast Python Virtual Machine]{PyPy : A fast Python Virtual Machine}
+\author[rguillebert]
+{Romain Guillebert\\
+\includegraphics[width=80px]{../img/py-web-new.png}}
+
+\institute{Pycon IE}
+\date{October 12th, 2014}
diff --git a/talk/pyconie2014/beamerdefs.txt b/talk/pyconie2014/beamerdefs.txt
new file mode 100644
--- /dev/null
+++ b/talk/pyconie2014/beamerdefs.txt
@@ -0,0 +1,108 @@
+.. colors
+.. ===========================
+
+.. role:: green
+.. role:: red
+
+
+.. general useful commands
+.. ===========================
+
+.. |pause| raw:: latex
+
+   \pause
+
+.. |small| raw:: latex
+
+   {\small
+
+.. |end_small| raw:: latex
+
+   }
+
+.. |scriptsize| raw:: latex
+
+   {\scriptsize
+
+.. |end_scriptsize| raw:: latex
+
+   }
+
+.. |strike<| raw:: latex
+
+   \sout{
+
+.. closed bracket
+.. ===========================
+
+.. |>| raw:: latex
+
+   }
+
+
+.. example block
+.. ===========================
+
+.. |example<| raw:: latex
+
+   \begin{exampleblock}{
+
+
+.. |end_example| raw:: latex
+
+   \end{exampleblock}
+
+
+
+.. alert block
+.. ===========================
+
+.. |alert<| raw:: latex
+
+   \begin{alertblock}{
+
+
+.. |end_alert| raw:: latex
+
+   \end{alertblock}
+
+
+
+.. columns
+.. ===========================
+
+.. |column1| raw:: latex
+
+   \begin{columns}
+      \begin{column}{0.45\textwidth}
+
+.. |column2| raw:: latex
+
+      \end{column}
+      \begin{column}{0.45\textwidth}
+
+
+.. |end_columns| raw:: latex
+
+      \end{column}
+   \end{columns}
+
+
+
+.. |snake| image:: ../../img/py-web-new.png
+           :scale: 15%
+           
+
+
+.. nested blocks
+.. ===========================
+
+.. |nested| raw:: latex
+
+   \begin{columns}
+      \begin{column}{0.85\textwidth}
+
+.. |end_nested| raw:: latex
+
+      \end{column}
+   \end{columns}
diff --git a/talk/pyconie2014/speed.png b/talk/pyconie2014/speed.png
new file mode 100644
index 0000000000000000000000000000000000000000..4640c76f8a665af1c414dc4c4ca22be3bd8ff360
GIT binary patch

[cut]

diff --git a/talk/pyconie2014/stylesheet.latex b/talk/pyconie2014/stylesheet.latex
new file mode 100644
--- /dev/null
+++ b/talk/pyconie2014/stylesheet.latex
@@ -0,0 +1,9 @@
+\setbeamercovered{transparent}
+\setbeamertemplate{navigation symbols}{}
+
+\definecolor{darkgreen}{rgb}{0, 0.5, 0.0}
+\newcommand{\docutilsrolegreen}[1]{\color{darkgreen}#1\normalcolor}
+\newcommand{\docutilsrolered}[1]{\color{red}#1\normalcolor}
+
+\newcommand{\green}[1]{\color{darkgreen}#1\normalcolor}
+\newcommand{\red}[1]{\color{red}#1\normalcolor}
diff --git a/talk/pyconie2014/talk.pdf b/talk/pyconie2014/talk.pdf
new file mode 100644
index 0000000000000000000000000000000000000000..748494720b5c056a3734955b2bbb4ad2f93692e4
GIT binary patch

[cut]

diff --git a/talk/pyconie2014/talk.rst b/talk/pyconie2014/talk.rst
new file mode 100644
--- /dev/null
+++ b/talk/pyconie2014/talk.rst
@@ -0,0 +1,170 @@
+.. include:: beamerdefs.txt
+
+PyPy : A fast Python Virtual Machine
+====================================
+
+Me
+--
+
+- rguillebert on twitter and irc
+
+- PyPy contributor since 2011
+
+- NumPyPy contributor
+
+- Software consultant (hire me !)
+
+Introduction
+------------
+
+- "PyPy is a fast, compliant alternative implementation of the Python language"
+
+- Aims to reach the best performance possible without changing the syntax or semantics
+
+- Supports x86, x86_64, ARM
+
+- Production ready
+
+- MIT Licensed
+
+Speed
+-----
+
+.. image:: speed.png
+   :scale: 37%
+
+Speed
+-----
+
+- Automatically generated tracing just-in-time compiler
+
+- Generates linear traces from loops
+
+- Generates efficient machine code based on runtime observations
+
+- Removes overhead when unnecessary
+
+- But Python features which require overhead remain available (frame introspection, pdb)
+
+Performance ?
+-------------
+
+- Things get done faster
+
+- Serve more requests per second
+
+- Lower latency
+
+- Less servers for the same performance
+
+Demo
+----
+
+- Real-time edge detection
+
+
+Compatibility
+-------------
+
+- Fully compatible with CPython 2.7 & 3.2 (minus implementation details)
+
+- Partial and slow support of the C-API
+
+- Alternatives might exist
+
+Ecosystem
+---------
+
+- We should (slowly, incrementally) move away from the C extension API
+
+  * Makes assumptions on refcounting, object layout, the GIL
+
+  * The future of Python is bound to the future of CPython (a more than 20 years old interpreter)
+
+  * It's hard for a new Python VM without C extension support to get traction (not only PyPy)
+
+- This doesn't mean we should lose Python's ability to interface with C easily
+
+- CFFI is the PyPy team's attempt at solving this
+
+CFFI (1/2)
+----------
+
+- Where do we go from here ?
+
+- CFFI is a fairly new way of interacting with C in an implementation independant way
+
+- Very fast on PyPy
+
+- Decently fast on CPython
+
+- The Jython project is working on support
+
+CFFI (2/2)
+----------
+
+- More convenient, safer, faster than ctypes
+
+- Can call C functions easily, API and ABI mode
+
+- Python functions can be exposed to C
+
+- Already used by pyopenssl, psycopg2cffi, pygame_cffi, lxml_cffi
+
+- Other tools could be built on top of it (Cython cffi backend ?)
+
+Success stories
+---------------
+
+    Magnetic is the leader in online search retargeting, with a large, high volume, performance-critical platform written in Python. [...] 
+
+    The Magnetic bidders were ported from CPython to PyPy, yielding an overall 30% performance gain.
+
+- Julian Berman
+
+  magnetic.com
+
+Success stories
+---------------
+
+    Currently we have improvements in raw performance (read: response times) that span from 8% to a pretty interesting 40%, but we have a peak of an astonishing 100-120% and even more.
+
+    Take into the account that most of our apps are simple "blocking-on-db" ones, so a 2x increase is literally money.
+
+- Roberto De Ioris
+
+  Unbit
+
+Success stories
+---------------
+
+    In addition to this our main (almost secret) objective was reducing resource usage of the application servers, which directly translates to being able to host more customers on the same server.
+
+- Roberto De Ioris
+
+  Unbit
+
+Success stories
+---------------
+
+    PyPy is an excellent choice for every pure Python project that depends on speed of execution of readable and maintainable large source code.
+    [...]
+    We had roughly a 2x speedup with PyPy over CPython.
+
+- Marko Tasic (Web and Data processing)
+
+Future
+------
+
+- Python 3.3
+
+- NumPyPy
+
+- STM
+
+- You can donate to help the progress of these features : pypy.org
+
+Questions
+---------
+
+- Questions ?
diff --git a/talk/pyconpl-2014/Makefile b/talk/pyconpl-2014/Makefile
new file mode 100644
--- /dev/null
+++ b/talk/pyconpl-2014/Makefile
@@ -0,0 +1,18 @@
+# you can find rst2beamer.py here:
+# https://bitbucket.org/antocuni/env/raw/default/bin/rst2beamer.py
+
+# WARNING: to work, it needs this patch for docutils
+# https://sourceforge.net/tracker/?func=detail&atid=422032&aid=1459707&group_id=38414
+
+talk.pdf: talk.rst author.latex stylesheet.latex
+	rst2beamer --stylesheet=stylesheet.latex --documentoptions=14pt --output-encoding=utf8 --overlaybullets=False talk.rst talk.latex || exit
+	#/home/antocuni/.virtualenvs/rst2beamer/bin/python `which rst2beamer.py` --stylesheet=stylesheet.latex --documentoptions=14pt talk.rst talk.latex || exit
+	sed 's/\\date{}/\\input{author.latex}/' -i talk.latex || exit
+	#sed 's/\\maketitle/\\input{title.latex}/' -i talk.latex || exit
+	pdflatex talk.latex  || exit
+
+view: talk.pdf
+	evince talk.pdf &
+
+xpdf: talk.pdf
+	xpdf talk.pdf &
diff --git a/talk/pyconpl-2014/author.latex b/talk/pyconpl-2014/author.latex
new file mode 100644
--- /dev/null
+++ b/talk/pyconpl-2014/author.latex
@@ -0,0 +1,10 @@
+\definecolor{rrblitbackground}{rgb}{0.0, 0.0, 0.0}
+
+\title[PyPy]{PyPy}
+\author[arigo, fijal]
+{Armin Rigo, Maciej Fijałkowski\\
+\includegraphics[width=80px]{../img/py-web-new.png}
+\hspace{1em}
+\includegraphics[width=80px]{../img/baroquesoftware.png}}
+\institute{PyCon PL}
+\date{October 2014}
diff --git a/talk/pyconpl-2014/beamerdefs.txt b/talk/pyconpl-2014/beamerdefs.txt
new file mode 100644
--- /dev/null
+++ b/talk/pyconpl-2014/beamerdefs.txt
@@ -0,0 +1,108 @@
+.. colors
+.. ===========================
+
+.. role:: green
+.. role:: red
+
+
+.. general useful commands
+.. ===========================
+
+.. |pause| raw:: latex
+
+   \pause
+
+.. |small| raw:: latex
+
+   {\small
+
+.. |end_small| raw:: latex
+
+   }
+
+.. |scriptsize| raw:: latex
+
+   {\scriptsize
+
+.. |end_scriptsize| raw:: latex
+
+   }
+
+.. |strike<| raw:: latex
+
+   \sout{
+
+.. closed bracket
+.. ===========================
+
+.. |>| raw:: latex
+
+   }
+
+
+.. example block
+.. ===========================
+
+.. |example<| raw:: latex
+
+   \begin{exampleblock}{
+
+
+.. |end_example| raw:: latex
+
+   \end{exampleblock}
+
+
+
+.. alert block
+.. ===========================
+
+.. |alert<| raw:: latex
+
+   \begin{alertblock}{
+
+
+.. |end_alert| raw:: latex
+
+   \end{alertblock}
+
+
+
+.. columns
+.. ===========================
+
+.. |column1| raw:: latex
+
+   \begin{columns}
+      \begin{column}{0.45\textwidth}
+
+.. |column2| raw:: latex
+
+      \end{column}
+      \begin{column}{0.45\textwidth}
+
+
+.. |end_columns| raw:: latex
+
+      \end{column}
+   \end{columns}
+
+
+
+.. |snake| image:: ../../img/py-web-new.png
+           :scale: 15%
+           
+
+
+.. nested blocks
+.. ===========================
+
+.. |nested| raw:: latex
+
+   \begin{columns}
+      \begin{column}{0.85\textwidth}
+
+.. |end_nested| raw:: latex
+
+      \end{column}
+   \end{columns}
diff --git a/talk/pyconpl-2014/benchmarks/abstract.rst b/talk/pyconpl-2014/benchmarks/abstract.rst
new file mode 100644
--- /dev/null
+++ b/talk/pyconpl-2014/benchmarks/abstract.rst
@@ -0,0 +1,7 @@
+How to benchmark code
+---------------------
+
+In this talk, we would like to present basics of how the Python virtual machines
+like CPython or PyPy work and how to use that knowledge to write meaningful
+benchmarks for your programs. We'll show what's wrong with microbenchmarks
+and how to improve the situation.
diff --git a/talk/pyconpl-2014/benchmarks/f1.py b/talk/pyconpl-2014/benchmarks/f1.py
new file mode 100644
--- /dev/null
+++ b/talk/pyconpl-2014/benchmarks/f1.py
@@ -0,0 +1,8 @@
+
+def f():
+    i = 0
+    while i < 100000000:
+        i += 1
+    return i
+
+f()
diff --git a/talk/pyconpl-2014/benchmarks/f2.py b/talk/pyconpl-2014/benchmarks/f2.py
new file mode 100644
--- /dev/null
+++ b/talk/pyconpl-2014/benchmarks/f2.py
@@ -0,0 +1,10 @@
+
+def f():
+    i = 0
+    s = 0
+    while i < 100000000:
+        s += len(str(i))
+        i += 1
+    return s
+
+print f()
diff --git a/talk/pyconpl-2014/benchmarks/fib.py b/talk/pyconpl-2014/benchmarks/fib.py
new file mode 100644
--- /dev/null
+++ b/talk/pyconpl-2014/benchmarks/fib.py
@@ -0,0 +1,31 @@
+
+import time
+import numpy
+try:
+    from matplotlib import pylab
+except:
+    from embed.emb import import_mod
+    pylab = import_mod('matplotlib.pylab')
+
+def fib(n):
+    if n == 0 or n == 1:
+        return 1
+    return fib(n - 1) + fib(n - 2)
+
+def f():
+    for i in range(10000):
+        "".join(list(str(i)))
+
+times = []
+for i in xrange(1000):
+    t0 = time.time()
+    #f()
+    fib(17)
+    times.append(time.time() - t0)
+
+hist, bins = numpy.histogram(times, 20)
+#pylab.plot(bins[:-1], hist)
+pylab.ylim(0, max(times) * 1.2)
+pylab.plot(numpy.array(times))
+#pylab.hist(hist, bins, histtype='bar')
+pylab.show()
diff --git a/talk/pyconpl-2014/benchmarks/talk.rst b/talk/pyconpl-2014/benchmarks/talk.rst
new file mode 100644
--- /dev/null
+++ b/talk/pyconpl-2014/benchmarks/talk.rst
@@ -0,0 +1,163 @@
+.. include:: ../beamerdefs.txt
+
+---------------------
+How to benchmark code
+---------------------
+
+Who are we?
+------------
+
+* Maciej Fijalkowski, Armin Rigo
+
+* working on PyPy
+
+* interested in performance
+
+What is this talk is about?
+---------------------------
+
+* basics how CPython and PyPy run programs
+
+* a bit of theory about measuring performance
+
+* microbenchmarks
+
+* complicated picture of "real world"
+
+CPython
+-------
+
+* a "simple" virtual machine
+
+* compiles python code to bytecode
+
+* runs the bytecode
+
+* usually invokes tons of runtime functions written in C
+
+CPython (demo)
+--------------
+
+PyPy
+----
+
+* not so simple virtual machine
+
+* all of the above
+
+* ... and then if the loop/function gets called often enough
+  it's compiled down to an optimized assembler by the JIT
+
+PyPy (demo)
+-----------
+
+Measurments 101
+---------------
+
+* run your benchmark multiple times
+
+* the distribution should be gaussian
+
+* take the average and the variation
+
+* if the variation is too large, increase the number of iterations
+
+Let's do it (demo)
+------------------
+
+Problems
+--------
+
+* the whole previous slide is a bunch of nonsense
+
+* ...
+
+"Solution"
+----------
+
+* you try your best and do the average anyway
+
+* presumably cutting off the warmup time
+
+|pause|
+
+* not ideal at all
+
+Writing benchmarks - typical approach
+-------------------------------------
+
+* write a set of small programs that exercise one particular thing
+
+  * recursive fibonacci
+
+  * pybench
+
+PyBench
+-------
+
+* used to be a tool to compare python implementations
+
+* only uses microbenchmarks
+
+* assumes operation times are concatenative
+
+Problems
+--------
+
+* a lot of effects are not concatenative
+
+* optimizations often collapse consecutive operations
+
+* large scale effects only show up on large programs
+
+An example
+----------
+
+* python 2.6 vs python 2.7 had minimal performance changes
+
+* somewhere in the changelog, there is a gc change mentioned
+
+* it made pypy translation toolchain jump from 3h to 1h
+
+* it's "impossible" to write a microbenchmarks for this
+
+More problems
+-------------
+
+* half of the blog posts comparing VM performance uses recursive fibonacci
+
+* most of the others use computer language shootout
+
+PyPy benchmark suite
+--------------------
+
+* programs from small to medium and large
+
+* 50 LOC to 100k LOC
+
+* try to exercise various parts of language (but e.g. lack IO)
+
+Solutions
+---------
+
+* measure what you are really interested in
+
+* derive microbenchmarks from your bottlenecks
+
+* be skeptical
+
+* understand what you're measuring
+
+Q&A
+---
+
+- http://pypy.org/
+
+- http://morepypy.blogspot.com/
+
+- http://baroquesoftware.com/
+
+- ``#pypy`` at freenode.net
+
+- Any question?
+
diff --git a/talk/pyconpl-2014/speed.png b/talk/pyconpl-2014/speed.png
new file mode 100644
index 0000000000000000000000000000000000000000..33fe20ac9d81ddbd3ced48f52f9717693dc15518
GIT binary patch

[cut]

diff --git a/talk/pyconpl-2014/standards.png b/talk/pyconpl-2014/standards.png
new file mode 100644
index 0000000000000000000000000000000000000000..5d38303773dd4f1b798a91bec62d05e0423a6a0d
GIT binary patch

[cut]

diff --git a/talk/pyconpl-2014/stylesheet.latex b/talk/pyconpl-2014/stylesheet.latex
new file mode 100644
--- /dev/null
+++ b/talk/pyconpl-2014/stylesheet.latex
@@ -0,0 +1,9 @@
+\setbeamercovered{transparent}
+\setbeamertemplate{navigation symbols}{}
+
+\definecolor{darkgreen}{rgb}{0, 0.5, 0.0}
+\newcommand{\docutilsrolegreen}[1]{\color{darkgreen}#1\normalcolor}
+\newcommand{\docutilsrolered}[1]{\color{red}#1\normalcolor}
+
+\newcommand{\green}[1]{\color{darkgreen}#1\normalcolor}
+\newcommand{\red}[1]{\color{red}#1\normalcolor}
diff --git a/talk/pyconpl-2014/talk.pdf b/talk/pyconpl-2014/talk.pdf
new file mode 100644
index 0000000000000000000000000000000000000000..96cfd649da81c326dec0dbe7d92087b9864229d7
GIT binary patch

[cut]

diff --git a/talk/pyconpl-2014/talk.rst b/talk/pyconpl-2014/talk.rst
new file mode 100644
--- /dev/null
+++ b/talk/pyconpl-2014/talk.rst
@@ -0,0 +1,297 @@
+.. include:: beamerdefs.txt
+
+================================
+PyPy
+================================
+
+Who We Are
+----------
+
+* Maciej Fijałkowski
+
+* Armin Rigo
+
+* PyPy developers for a long time
+
+* baroquesoftware
+
+What is PyPy?
+--------------
+
+* Python interpreter, alternative to CPython
+
+* Supports Python 2.7 and (beta) Python 3.2/3.3
+
+* Compatible and generally much faster (JIT)
+
+Benchmarks
+--------------------------------
+
+.. image:: speed.png
+   :scale: 44%
+   :align: center
+
+Demo
+--------------------------------
+
+
+Recent developments
+--------------------------------
+
+Between PyPy 2.0 (May 2013) and PyPy 2.4 (now):
+
+.
+
+* All kinds of speed improvements for all kinds of programs
+
+  - JIT improvements, incremental GC (garbage collector),
+    specific Python corners improved, ...
+
+* Support for ARM in addition to x86
+
+  - Thanks to the Raspberry-Pi foundation
+
+* Python 3 support 
+
+  - py3k, in addition to Python 2.7
+
+* Numpy more complete (but still not done)
+
+Status
+-----------------------------
+
+- Python code "just works"
+
+  * generally much faster than with CPython
+
+- C code: improving support
+
+  * cpyext: tries to load CPython C extension modules, slowly
+
+  * CFFI: the future
+
+  * cppyy for C++
+
+  * A very small native PyPy C API for embedding, WIP
+
+- Lots of CFFI modules around:
+
+  * pyopenssl, pygame_cffi, psycopg2cffi, lxml...
+
+Fundraising Campaign
+---------------------
+
+- py3k: 55'000 $ of 105'000 $ (52%)
+
+- numpy: 48'000 $ of 60'000 $ (80%)
+
+- STM, 1st call: 38'000 $
+
+- STM, 2nd call: 17'000 $ of 80'000 $ (22%)
+
+- Thanks to all donors!
+
+Commercial support
+------------------
+
+- We offer commercial support for PyPy
+
+- Consultancy and training
+
+- Performance issues for open- or closed-source programs, porting,
+  improving support in parts of the Python or non-Python interpreters,
+  etc.
+
+- http://baroquesoftware.com
+
+Recent developments (2)
+--------------------------------
+
+* CFFI
+
+  - C Foreign Function Interface
+
+* STM
+
+  - Software Transactional Memory
+
+CFFI
+-----
+
+- Python <-> C interfacing done right
+
+  * existing shared libraries
+
+  * custom C code
+
+- Alternative to the CPython Extension API, ctypes, Cython, etc.
+
+- Fast-ish on CPython, super-fast on PyPy, Jython support in the future
+
+- Simple, does not try to be magic
+
+CFFI
+----
+
+.. image:: standards.png
+   :scale: 50%
+   :align: center
+
+CFFI demo
+---------
+
+CFFI idea
+---------
+
+* C and Python are enough, we don't need an extra language
+
+* C is well defined, let's avoid magic
+
+* all the logic (and magic!) can be done in Python
+
+* API vs ABI
+
+* Inspired by LuaJIT's FFI
+
+Work in Progress: STM
+---------------------
+
+- Software Transactional Memory
+
+- Solving the GIL problem
+
+  * GIL = Global Interpreter Lock
+
+- Without bringing the threads and locks mess
+
+- Preliminary versions of pypy-jit-stm available
+
+STM (2)
+-------
+
+- STM = Free Threading done right
+
+  * with some overhead: 30-40% so far
+
+- Done at the level of RPython
+
+- The interpreter author doesn't have to worry
+  about adding tons of locks
+  
+  - that's us
+
+- The user *can* if he likes, but doesn't have to either
+
+  - that's you ``:-)``
+
+STM (3)
+-------
+
+- Works "like a GIL" but runs optimistically in parallel
+
+- A few bytecodes from thread A run on core 1
+
+- A few bytecodes from thread B run on core 2
+
+- If there is no conflict, we're happy
+
+- If there is a conflict, one of the two aborts and retries
+
+- Same effect as transactions in databases
+
+STM (4)
+-------
+
+- Threading made simpler for the user
+
+- It is generally efficient with *very coarse locks*
+
+  * no fine-grained locking needed
+
+- Easy to convert a number of existing single-threaded programs
+
+  * start multiple threads, run blocks of code in each
+
+  * use a single lock around everything
+
+  * normally, you win absolutely nothing
+
+  * but STM can (try to) *execute the blocks in parallel* anyway
+
+STM (Demo)
+----------
+
+PyPy and RPython
+---------------------------
+
+* PyPy is an interpreter/JIT-compiled for Python
+
+* PyPy is written in RPython
+
+* RPython is a language for writing interpreters:
+  it provides GC-for-free, JIT-for-free, etc.
+
+* Ideal for writing VMs for dynamic languages
+
+More PyPy-Powered Languages
+----------------------------
+
+- Topaz: implementing Ruby
+
+  * most of the language implemented
+
+  * "definitely faster than MRI"
+
+  * https://github.com/topazproject/topaz
+
+- HippyVM: implementing PHP
+
+  * ~7x faster than standard PHP
+
+  * comparable speed as HHVM
+
+  * http://hippyvm.com/
+
+- And more
+
+Future
+------
+
+* future is hard to predict
+
+* continue working on general improvements
+
+* improved IO performance in the pipeline
+
+* warmup improvements
+
+* numpy
+
+Warmup improvements
+-------------------
+
+* biggest complain - slow to warmup, memory hog
+
+* we have ideas how to improve the situation
+
+* still looking for funding
+
+Numpy
+-----
+
+* numpy is mostly complete
+
+* performance can be improved, especially the vectorized versions
+
+* scipy, matplotlib, the entire ecosystem, we have a hackish idea
+
+Contacts, Q&A
+--------------
+
+- http://pypy.org
+
+- http://morepypy.blogspot.com/
+
+- ``#pypy`` at freenode.net
+
+- Any question?

From noreply at buildbot.pypy.org  Fri Nov 21 13:21:29 2014
From: noreply at buildbot.pypy.org (hakanardo)
Date: Fri, 21 Nov 2014 13:21:29 +0100 (CET)
Subject: [pypy-commit] extradoc extradoc: last few results
Message-ID: <20141121122129.B88D91D2487@cobra.cs.uni-duesseldorf.de>

Author: Hakan Ardo 
Branch: extradoc
Changeset: r5466:7559563b6034
Date: 2014-11-21 13:21 +0100
http://bitbucket.org/pypy/extradoc/changeset/7559563b6034/

Log:	last few results

diff --git a/talk/dls2012/benchmarks/iter/result-2.4.0.txt b/talk/dls2012/benchmarks/iter/result-2.4.0.txt
--- a/talk/dls2012/benchmarks/iter/result-2.4.0.txt
+++ b/talk/dls2012/benchmarks/iter/result-2.4.0.txt
@@ -88,3 +88,13 @@
 pypy iter/nditer.py
 sum1d: 61.2064362049 +- 0.578041254203
 sum2d: 71.1426748753 +- 1.09482960038
+whsum2d: 73.6043967247 +- 0.592259791238
+wsum1d: 62.5866453171 +- 0.677394146874
+wsum2d: 71.0087907314 +- 0.309482073549
+xsum1d: 63.4650315285 +- 0.844882977961
+xsum2d: 72.2017237663 +- 0.807526950608
+xysum2d: 71.3824438095 +- 0.28602305068
+mean1d: 83.7468230247 +- 2.77559774136
+median1d: 84.9599477768 +- 0.460831996712
+ripple1d: 83.2095424175 +- 0.427835447938
+ripple2d: 107.806012583 +- 1.06730555212

From noreply at buildbot.pypy.org  Fri Nov 21 20:37:32 2014
From: noreply at buildbot.pypy.org (dalcinl)
Date: Fri, 21 Nov 2014 20:37:32 +0100 (CET)
Subject: [pypy-commit] cffi default: CPython: Silent GCC -Wconversion
	warnings
Message-ID: <20141121193732.464701D28A4@cobra.cs.uni-duesseldorf.de>

Author: Lisandro Dalcin 
Branch: 
Changeset: r1575:1f05f7f5f966
Date: 2014-04-26 20:44 +0300
http://bitbucket.org/cffi/cffi/changeset/1f05f7f5f966/

Log:	CPython: Silent GCC -Wconversion warnings

diff --git a/cffi/vengine_cpy.py b/cffi/vengine_cpy.py
--- a/cffi/vengine_cpy.py
+++ b/cffi/vengine_cpy.py
@@ -228,7 +228,8 @@
                 converter = '_cffi_to_c_int'
                 extraarg = ', %s' % tp.name
             else:
-                converter = '_cffi_to_c_%s' % (tp.name.replace(' ', '_'),)
+                converter = '(%s)_cffi_to_c_%s' % (tp.get_c_name(''),
+                                                   tp.name.replace(' ', '_'))
             errvalue = '-1'
         #
         elif isinstance(tp, model.PointerType):
@@ -267,8 +268,8 @@
         self._prnt('  if (datasize != 0) {')
         self._prnt('    if (datasize < 0)')
         self._prnt('      %s;' % errcode)
-        self._prnt('    %s = alloca(datasize);' % (tovar,))
-        self._prnt('    memset((void *)%s, 0, datasize);' % (tovar,))
+        self._prnt('    %s = alloca((size_t)datasize);' % (tovar,))
+        self._prnt('    memset((void *)%s, 0, (size_t)datasize);' % (tovar,))
         self._prnt('    if (_cffi_convert_array_from_object('
                    '(char *)%s, _cffi_type(%d), %s) < 0)' % (
             tovar, self._gettypenum(tp), fromvar))
@@ -844,7 +845,7 @@
                                          : (type)_cffi_to_c_i32(o)) :    \
      sizeof(type) == 8 ? (((type)-1) > 0 ? (type)_cffi_to_c_u64(o)       \
                                          : (type)_cffi_to_c_i64(o)) :    \
-     (Py_FatalError("unsupported size for type " #type), 0))
+     (Py_FatalError("unsupported size for type " #type), (type)0))
 
 #define _cffi_to_c_i8                                                    \
                  ((int(*)(PyObject *))_cffi_exports[1])
diff --git a/testing/test_verify.py b/testing/test_verify.py
--- a/testing/test_verify.py
+++ b/testing/test_verify.py
@@ -92,7 +92,11 @@
 def test_rounding_1():
     ffi = FFI()
     ffi.cdef("float sin(double x);")
-    lib = ffi.verify('#include ', libraries=lib_m)
+    lib = ffi.verify('''
+    #include 
+    static float my_sin(double x) { return (float)sin(x); }
+    #define sin my_sin
+    ''', libraries=lib_m)
     res = lib.sin(1.23)
     assert res != math.sin(1.23)     # not exact, because of double->float
     assert abs(res - math.sin(1.23)) < 1E-5
@@ -113,13 +117,13 @@
 
 def test_strlen_approximate():
     ffi = FFI()
-    ffi.cdef("int strlen(char *s);")
+    ffi.cdef("size_t strlen(char *s);")
     lib = ffi.verify("#include ")
     assert lib.strlen(b"hi there!") == 9
 
 def test_strlen_array_of_char():
     ffi = FFI()
-    ffi.cdef("int strlen(char[]);")
+    ffi.cdef("size_t strlen(char[]);")
     lib = ffi.verify("#include ")
     assert lib.strlen(b"hello") == 5
 
@@ -208,8 +212,8 @@
     ffi = FFI()
     ffi.cdef('\n'.join(["%s foo_%s(%s);" % (tp, tp.replace(' ', '_'), tp)
                        for tp in typenames]))
-    lib = ffi.verify('\n'.join(["%s foo_%s(%s x) { return x+1; }" %
-                                (tp, tp.replace(' ', '_'), tp)
+    lib = ffi.verify('\n'.join(["%s foo_%s(%s x) { return (%s)(x+1); }" %
+                                (tp, tp.replace(' ', '_'), tp, tp)
                                 for tp in typenames]))
     for typename in typenames:
         foo = getattr(lib, 'foo_%s' % typename.replace(' ', '_'))
@@ -315,7 +319,7 @@
 def test_char_type():
     ffi = FFI()
     ffi.cdef("char foo(char);")
-    lib = ffi.verify("char foo(char x) { return x+1; }")
+    lib = ffi.verify("char foo(char x) { return ++x; }")
     assert lib.foo(b"A") == b"B"
     py.test.raises(TypeError, lib.foo, b"bar")
     py.test.raises(TypeError, lib.foo, "bar")
@@ -896,7 +900,7 @@
         static int foo(token_t *tk) {
             if (!tk)
                 return -42;
-            *tk += 1.601;
+            *tk += 1.601f;
             return (int)*tk;
         }
         #define TOKEN_SIZE sizeof(token_t)
@@ -991,7 +995,7 @@
             long a;
         };
         int foo(struct foo_s s) {
-            return s.a - (int)s.b;
+            return (int)s.a - (int)s.b;
         }
     """)
     s = ffi.new("struct foo_s *", [100, 1])
@@ -1008,7 +1012,7 @@
             long a;
         };
         int foo1(struct foo_s s) {
-            return s.a - (int)s.b;
+            return (int)s.a - (int)s.b;
         }
         int (*foo)(struct foo_s s) = &foo1;
     """)
@@ -1067,7 +1071,7 @@
 def test_array_as_argument():
     ffi = FFI()
     ffi.cdef("""
-        int strlen(char string[]);
+        size_t strlen(char string[]);
     """)
     ffi.verify("#include ")
 
@@ -1676,7 +1680,7 @@
         static int c_callback(int how_many, ...) {
             va_list ap;
             /* collect the "..." arguments into the values[] array */
-            int i, *values = alloca(how_many * sizeof(int));
+            int i, *values = alloca((size_t)how_many * sizeof(int));
             va_start(ap, how_many);
             for (i=0; i

Author: Lisandro Dalcin 
Branch: 
Changeset: r1576:4ebadac73fae
Date: 2014-04-30 14:18 +0300
http://bitbucket.org/cffi/cffi/changeset/4ebadac73fae/

Log:	Add test for approximate return types

diff --git a/testing/test_verify.py b/testing/test_verify.py
--- a/testing/test_verify.py
+++ b/testing/test_verify.py
@@ -109,17 +109,19 @@
     assert res != math.sin(1.23)     # not exact, because of double->float
     assert abs(res - math.sin(1.23)) < 1E-5
 
-def test_strlen_exact():
+def test_return_exact():
     ffi = FFI()
     ffi.cdef("size_t strlen(const char *s);")
     lib = ffi.verify("#include ")
     assert lib.strlen(b"hi there!") == 9
 
-def test_strlen_approximate():
-    ffi = FFI()
-    ffi.cdef("size_t strlen(char *s);")
-    lib = ffi.verify("#include ")
-    assert lib.strlen(b"hi there!") == 9
+def test_return_approximate():
+    for typename in ['short', 'int', 'long', 'long long']:
+        ffi = FFI()
+        ffi.cdef("%s foo(signed char x);" % typename)
+        lib = ffi.verify("signed char foo(signed char x) { return x;}")
+        assert lib.foo(-128) == -128
+        assert lib.foo(+127) == +127
 
 def test_strlen_array_of_char():
     ffi = FFI()

From noreply at buildbot.pypy.org  Fri Nov 21 20:37:34 2014
From: noreply at buildbot.pypy.org (dalcinl)
Date: Fri, 21 Nov 2014 20:37:34 +0100 (CET)
Subject: [pypy-commit] cffi default: CPython: Add explicit typecasts in
	_cffi_from_c_int()
Message-ID: <20141121193734.99A751D28A4@cobra.cs.uni-duesseldorf.de>

Author: Lisandro Dalcin 
Branch: 
Changeset: r1577:8106093be18d
Date: 2014-11-21 12:55 +0300
http://bitbucket.org/cffi/cffi/changeset/8106093be18d/

Log:	CPython: Add explicit typecasts in _cffi_from_c_int()

diff --git a/cffi/vengine_cpy.py b/cffi/vengine_cpy.py
--- a/cffi/vengine_cpy.py
+++ b/cffi/vengine_cpy.py
@@ -829,12 +829,15 @@
             PyLong_FromLongLong((long long)(x)))
 
 #define _cffi_from_c_int(x, type)                                        \
-    (((type)-1) > 0 ?   /* unsigned */                                   \
-        (sizeof(type) < sizeof(long) ? PyInt_FromLong(x) :               \
-         sizeof(type) == sizeof(long) ? PyLong_FromUnsignedLong(x) :     \
-                                        PyLong_FromUnsignedLongLong(x))  \
-      : (sizeof(type) <= sizeof(long) ? PyInt_FromLong(x) :              \
-                                        PyLong_FromLongLong(x)))
+    (((type)-1) > 0 ? /* unsigned */                                     \
+        (sizeof(type) < sizeof(long) ?                                   \
+            PyInt_FromLong((long)x) :                                    \
+         sizeof(type) == sizeof(long) ?                                  \
+            PyLong_FromUnsignedLong((unsigned long)x) :                  \
+            PyLong_FromUnsignedLongLong((unsigned long long)x)) :        \
+        (sizeof(type) <= sizeof(long) ?                                  \
+            PyInt_FromLong((long)x) :                                    \
+            PyLong_FromLongLong((long long)x)))
 
 #define _cffi_to_c_int(o, type)                                          \
     (sizeof(type) == 1 ? (((type)-1) > 0 ? (type)_cffi_to_c_u8(o)        \

From noreply at buildbot.pypy.org  Fri Nov 21 20:37:35 2014
From: noreply at buildbot.pypy.org (dalcinl)
Date: Fri, 21 Nov 2014 20:37:35 +0100 (CET)
Subject: [pypy-commit] cffi default: Add and fix tests for enumerations
Message-ID: <20141121193735.B05561D28A4@cobra.cs.uni-duesseldorf.de>

Author: Lisandro Dalcin 
Branch: 
Changeset: r1578:0870cf7eeeca
Date: 2014-11-21 14:23 +0300
http://bitbucket.org/cffi/cffi/changeset/0870cf7eeeca/

Log:	Add and fix tests for enumerations

diff --git a/testing/test_verify.py b/testing/test_verify.py
--- a/testing/test_verify.py
+++ b/testing/test_verify.py
@@ -1085,7 +1085,7 @@
     """)
     lib = ffi.verify("""
         enum foo_e { AA, CC, BB };
-        int foo_func(enum foo_e e) { return e; }
+        int foo_func(enum foo_e e) { return (int)e; }
     """)
     assert lib.foo_func(lib.BB) == 2
     py.test.raises(TypeError, lib.foo_func, "BB")
@@ -1098,7 +1098,7 @@
     """)
     lib = ffi.verify("""
         enum foo_e { AA, CC, BB };
-        enum foo_e foo_func(int x) { return x; }
+        enum foo_e foo_func(int x) { return (enum foo_e)x; }
     """)
     assert lib.foo_func(lib.BB) == lib.BB == 2
 
@@ -1133,6 +1133,19 @@
     assert lib.AA == 0
     assert lib.BB == 2
 
+def test_typedef_enum_as_argument():
+    ffi = FFI()
+    ffi.cdef("""
+        typedef enum { AA, BB, ... } foo_t;
+        int foo_func(foo_t);
+    """)
+    lib = ffi.verify("""
+        typedef enum { AA, CC, BB } foo_t;
+        int foo_func(foo_t e) { return (int)e; }
+    """)
+    assert lib.foo_func(lib.BB) == lib.BB == 2
+    py.test.raises(TypeError, lib.foo_func, "BB")
+
 def test_typedef_enum_as_function_result():
     ffi = FFI()
     ffi.cdef("""
@@ -1141,7 +1154,7 @@
     """)
     lib = ffi.verify("""
         typedef enum { AA, CC, BB } foo_t;
-        foo_t foo_func(int x) { return x; }
+        foo_t foo_func(int x) { return (foo_t)x; }
     """)
     assert lib.foo_func(lib.BB) == lib.BB == 2
 

From noreply at buildbot.pypy.org  Fri Nov 21 20:37:36 2014
From: noreply at buildbot.pypy.org (dalcinl)
Date: Fri, 21 Nov 2014 20:37:36 +0100 (CET)
Subject: [pypy-commit] cffi default: Fix test for approximate floating point
	return type
Message-ID: <20141121193736.C219E1D28A4@cobra.cs.uni-duesseldorf.de>

Author: Lisandro Dalcin 
Branch: 
Changeset: r1579:9642bf799905
Date: 2014-11-21 14:40 +0300
http://bitbucket.org/cffi/cffi/changeset/9642bf799905/

Log:	Fix test for approximate floating point return type

diff --git a/testing/test_verify.py b/testing/test_verify.py
--- a/testing/test_verify.py
+++ b/testing/test_verify.py
@@ -91,13 +91,9 @@
 
 def test_rounding_1():
     ffi = FFI()
-    ffi.cdef("float sin(double x);")
-    lib = ffi.verify('''
-    #include 
-    static float my_sin(double x) { return (float)sin(x); }
-    #define sin my_sin
-    ''', libraries=lib_m)
-    res = lib.sin(1.23)
+    ffi.cdef("double sinf(float x);")
+    lib = ffi.verify('#include ', libraries=lib_m)
+    res = lib.sinf(1.23)
     assert res != math.sin(1.23)     # not exact, because of double->float
     assert abs(res - math.sin(1.23)) < 1E-5
 

From noreply at buildbot.pypy.org  Fri Nov 21 20:37:37 2014
From: noreply at buildbot.pypy.org (dalcinl)
Date: Fri, 21 Nov 2014 20:37:37 +0100 (CET)
Subject: [pypy-commit] cffi default: Enable GCC/Clang -Wconversion and add
 tests for expected failures
Message-ID: <20141121193737.DD8FC1D28A4@cobra.cs.uni-duesseldorf.de>

Author: Lisandro Dalcin 
Branch: 
Changeset: r1580:d398e0a66539
Date: 2014-11-21 17:50 +0300
http://bitbucket.org/cffi/cffi/changeset/d398e0a66539/

Log:	Enable GCC/Clang -Wconversion and add tests for expected failures

diff --git a/testing/test_verify.py b/testing/test_verify.py
--- a/testing/test_verify.py
+++ b/testing/test_verify.py
@@ -14,12 +14,13 @@
 else:
     if (sys.platform == 'darwin' and
           [int(x) for x in os.uname()[2].split('.')] >= [11, 0, 0]):
+        # assume a standard clang or gcc
+        extra_compile_args = ['-Werror', '-Wconversion']
         # special things for clang
-        extra_compile_args = [
-            '-Werror', '-Qunused-arguments', '-Wno-error=shorten-64-to-32']
+        extra_compile_args.append('-Qunused-arguments')
     else:
         # assume a standard gcc
-        extra_compile_args = ['-Werror']
+        extra_compile_args = ['-Werror', '-Wconversion']
 
     class FFI(FFI):
         def verify(self, *args, **kwds):
@@ -89,6 +90,39 @@
     lib = ffi.verify('#include ', libraries=lib_m)
     assert lib.sin(1.23) == math.sin(1.23)
 
+def _Wconversion(cdef, source, **kargs):
+    if sys.platform == 'win32':
+        py.test.skip("needs GCC or Clang")
+    ffi = FFI()
+    ffi.cdef(cdef)
+    py.test.raises(VerificationError, ffi.verify, source, **kargs)
+    extra_compile_args_orig = extra_compile_args[:]
+    extra_compile_args.remove('-Wconversion')
+    try:
+        ffi.verify(source, **kargs)
+    finally:
+        extra_compile_args[:] = extra_compile_args_orig 
+
+def test_Wconversion_unsigned():
+    _Wconversion("unsigned foo(void);",
+                 "int foo(void) { return -1;}")
+
+def test_Wconversion_integer():
+    _Wconversion("short foo(void);",
+                 "long long foo(void) { return 1<", libraries=lib_m)
+
+def test_Wconversion_float2int():
+    _Wconversion("int sinf(float);",
+                 "#include ", libraries=lib_m)
+
+def test_Wconversion_double2int():
+    _Wconversion("int sin(double);",
+                 "#include ", libraries=lib_m)
+
 def test_rounding_1():
     ffi = FFI()
     ffi.cdef("double sinf(float x);")

From noreply at buildbot.pypy.org  Fri Nov 21 20:45:15 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Fri, 21 Nov 2014 20:45:15 +0100 (CET)
Subject: [pypy-commit] cffi default: Redo the original intention of a couple
	of extra tests.
Message-ID: <20141121194515.6449C1D2487@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r1581:f9bbb11363fc
Date: 2014-11-21 20:45 +0100
http://bitbucket.org/cffi/cffi/changeset/f9bbb11363fc/

Log:	Redo the original intention of a couple of extra tests.

diff --git a/testing/test_verify.py b/testing/test_verify.py
--- a/testing/test_verify.py
+++ b/testing/test_verify.py
@@ -99,9 +99,10 @@
     extra_compile_args_orig = extra_compile_args[:]
     extra_compile_args.remove('-Wconversion')
     try:
-        ffi.verify(source, **kargs)
+        lib = ffi.verify(source, **kargs)
     finally:
-        extra_compile_args[:] = extra_compile_args_orig 
+        extra_compile_args[:] = extra_compile_args_orig
+    return lib
 
 def test_Wconversion_unsigned():
     _Wconversion("unsigned foo(void);",
@@ -112,8 +113,11 @@
                  "long long foo(void) { return 1<", libraries=lib_m)
+    lib = _Wconversion("float sin(double);",
+                       "#include ", libraries=lib_m)
+    res = lib.sin(1.23)
+    assert res != math.sin(1.23)     # not exact, because of double->float
+    assert abs(res - math.sin(1.23)) < 1E-5
 
 def test_Wconversion_float2int():
     _Wconversion("int sinf(float);",
@@ -139,12 +143,17 @@
     assert res != math.sin(1.23)     # not exact, because of double->float
     assert abs(res - math.sin(1.23)) < 1E-5
 
-def test_return_exact():
+def test_strlen_exact():
     ffi = FFI()
     ffi.cdef("size_t strlen(const char *s);")
     lib = ffi.verify("#include ")
     assert lib.strlen(b"hi there!") == 9
 
+def test_strlen_approximate():
+    lib = _Wconversion("int strlen(char *s);",
+                       "#include ")
+    assert lib.strlen(b"hi there!") == 9
+
 def test_return_approximate():
     for typename in ['short', 'int', 'long', 'long long']:
         ffi = FFI()

From noreply at buildbot.pypy.org  Sat Nov 22 11:39:09 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Sat, 22 Nov 2014 11:39:09 +0100 (CET)
Subject: [pypy-commit] pypy default: Fix test
Message-ID: <20141122103909.B61EA1C022D@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r74630:79565c62bfae
Date: 2014-11-22 11:39 +0100
http://bitbucket.org/pypy/pypy/changeset/79565c62bfae/

Log:	Fix test

diff --git a/rpython/rtyper/lltypesystem/test/test_rffi.py b/rpython/rtyper/lltypesystem/test/test_rffi.py
--- a/rpython/rtyper/lltypesystem/test/test_rffi.py
+++ b/rpython/rtyper/lltypesystem/test/test_rffi.py
@@ -100,7 +100,6 @@
     def test_string_reverse(self):
         c_source = py.code.Source("""
         #include 
-        #include 
         #include 
 
         char *f(char* arg)

From noreply at buildbot.pypy.org  Sat Nov 22 18:31:19 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Sat, 22 Nov 2014 18:31:19 +0100 (CET)
Subject: [pypy-commit] pypy portable-threadlocal: Idea: group in a raw
 struct all thread-local variables, including a
Message-ID: <20141122173119.6CDF71C022D@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: portable-threadlocal
Changeset: r74631:cc7a1b2e6249
Date: 2014-11-22 11:47 +0100
http://bitbucket.org/pypy/pypy/changeset/cc7a1b2e6249/

Log:	Idea: group in a raw struct all thread-local variables, including a
	pointer to errno and the thread ident (for fast access). Then we
	basically pass into the jit assembler a pointer to this raw struct.
	So the extra cost is a "get the address of this thread-local struct"
	before entering jitted code, once.

	Benefits include: it would work on dlopened libraries, on OS/X, and
	on Windows.


From noreply at buildbot.pypy.org  Sat Nov 22 18:31:20 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Sat, 22 Nov 2014 18:31:20 +0100 (CET)
Subject: [pypy-commit] pypy portable-threadlocal: in-progress: refactor the
 threadlocal through the translation toolchain
Message-ID: <20141122173120.EFD841C022D@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: portable-threadlocal
Changeset: r74632:40454c851fe5
Date: 2014-11-22 18:31 +0100
http://bitbucket.org/pypy/pypy/changeset/40454c851fe5/

Log:	in-progress: refactor the threadlocal through the translation
	toolchain

diff --git a/rpython/annotator/bookkeeper.py b/rpython/annotator/bookkeeper.py
--- a/rpython/annotator/bookkeeper.py
+++ b/rpython/annotator/bookkeeper.py
@@ -65,6 +65,7 @@
         self.external_class_cache = {}      # cache of ExternalType classes
 
         self.needs_generic_instantiate = {}
+        self.thread_local_fields = set()
 
         delayed_imports()
 
diff --git a/rpython/rlib/rposix.py b/rpython/rlib/rposix.py
--- a/rpython/rlib/rposix.py
+++ b/rpython/rlib/rposix.py
@@ -95,12 +95,19 @@
 # the default wrapper for set_errno is not suitable for use in critical places
 # like around GIL handling logic, so we provide our own wrappers.
 
- at jit.oopspec("rposix.get_errno()")
 def get_errno():
+    if jit.we_are_jitted():
+        from rpython.rlib import rthread
+        perrno = rthread.tlfield_p_errno.getraw()
+        return intmask(perrno[0])
     return intmask(_get_errno())
 
- at jit.oopspec("rposix.set_errno(errno)")
 def set_errno(errno):
+    if jit.we_are_jitted():
+        from rpython.rlib import rthread
+        perrno = rthread.tlfield_p_errno.getraw()
+        perrno[0] = rffi.cast(INT, errno)
+        return
     _set_errno(rffi.cast(INT, errno))
 
 if os.name == 'nt':
diff --git a/rpython/rlib/rthread.py b/rpython/rlib/rthread.py
--- a/rpython/rlib/rthread.py
+++ b/rpython/rlib/rthread.py
@@ -5,8 +5,10 @@
 from rpython.rlib import jit, rgc
 from rpython.rlib.debug import ll_assert
 from rpython.rlib.objectmodel import we_are_translated, specialize
+from rpython.rlib.objectmodel import CDefinedIntSymbolic
 from rpython.rtyper.lltypesystem.lloperation import llop
 from rpython.rtyper.tool import rffi_platform
+from rpython.rtyper.extregistry import ExtRegistryEntry
 
 class RThreadError(Exception):
     pass
@@ -40,8 +42,6 @@
                             releasegil=True)  # release the GIL, but most
                                               # importantly, reacquire it
                                               # around the callback
-c_thread_get_ident = llexternal('RPyThreadGetIdent', [], rffi.LONG,
-                                _nowrapper=True)    # always call directly
 
 TLOCKP = rffi.COpaquePtr('struct RPyOpaque_ThreadLock',
                           compilation_info=eci)
@@ -83,9 +83,8 @@
 
 # wrappers...
 
- at jit.loop_invariant
 def get_ident():
-    return rffi.cast(lltype.Signed, c_thread_get_ident())
+    return tlfield_thread_ident.getraw()
 
 @specialize.arg(0)
 def start_new_thread(x, y):
@@ -265,17 +264,35 @@
 # KEEP THE REFERENCE ALIVE, THE GC DOES NOT FOLLOW THEM SO FAR!
 # We use _make_sure_does_not_move() to make sure the pointer will not move.
 
-ecitl = ExternalCompilationInfo(
-    includes = ['src/threadlocal.h'],
-    separate_module_files = [translator_c_dir / 'src' / 'threadlocal.c'])
-ensure_threadlocal = rffi.llexternal_use_eci(ecitl)
 
-class ThreadLocalReference(object):
+class ThreadLocalField(object):
+    def __init__(self, FIELDTYPE, fieldname):
+        "NOT_RPYTHON: must be prebuilt"
+        self.FIELDTYPE = FIELDTYPE
+        self.fieldname = fieldname
+        offset = CDefinedIntSymbolic('RPY_TLOFS_%s' % self.fieldname,
+                                     default='?')
+        self.offset = offset
+
+        def getraw():
+            _threadlocalref_seeme(self)
+            addr = llop.threadlocalref_addr(llmemory.Address)
+            return llop.raw_load(FIELDTYPE, addr, offset)
+
+        def setraw(value):
+            _threadlocalref_seeme(self)
+            addr = llop.threadlocalref_addr(llmemory.Address)
+            llop.raw_store(lltype.Void, addr, offset, value)
+
+        self.getraw = getraw
+        self.setraw = setraw
+
+    def _freeze_(self):
+        return True
+
+
+class ThreadLocalReference(ThreadLocalField):
     _COUNT = 1
-    OPAQUEID = lltype.OpaqueType("ThreadLocalRef",
-                                 hints={"threadlocalref": True,
-                                        "external": "C",
-                                        "c_name": "RPyThreadStaticTLS"})
 
     def __init__(self, Cls):
         "NOT_RPYTHON: must be prebuilt"
@@ -284,16 +301,14 @@
         self.local = thread._local()      # <- NOT_RPYTHON
         unique_id = ThreadLocalReference._COUNT
         ThreadLocalReference._COUNT += 1
-        opaque_id = lltype.opaqueptr(ThreadLocalReference.OPAQUEID,
-                                     'tlref%d' % unique_id)
-        self.opaque_id = opaque_id
+        ThreadLocalField.__init__(self, llmemory.GCREF, 'tlref%d' % unique_id)
+        getraw = self.getraw
+        setraw = self.setraw
 
         def get():
             if we_are_translated():
-                from rpython.rtyper import rclass
-                from rpython.rtyper.annlowlevel import cast_base_ptr_to_instance
-                ptr = llop.threadlocalref_get(rclass.OBJECTPTR, opaque_id)
-                return cast_base_ptr_to_instance(Cls, ptr)
+                from rpython.rtyper.annlowlevel import cast_gcref_to_instance
+                return cast_gcref_to_instance(Cls, getraw())
             else:
                 return getattr(self.local, 'value', None)
 
@@ -304,18 +319,30 @@
                 from rpython.rtyper.annlowlevel import cast_instance_to_base_ptr
                 from rpython.rlib.rgc import _make_sure_does_not_move
                 from rpython.rlib.objectmodel import running_on_llinterp
-                ptr = cast_instance_to_base_ptr(value)
+                gcref = cast_instance_to_gcref(value)
                 if not running_on_llinterp:
-                    gcref = lltype.cast_opaque_ptr(llmemory.GCREF, ptr)
                     if gcref:
                         _make_sure_does_not_move(gcref)
-                llop.threadlocalref_set(lltype.Void, opaque_id, ptr)
-                ensure_threadlocal()
+                setraw(gcref)
             else:
                 self.local.value = value
 
         self.get = get
         self.set = set
 
-    def _freeze_(self):
-        return True
+
+tlfield_thread_ident = ThreadLocalField(lltype.Signed, "thread_ident")
+tlfield_p_errno = ThreadLocalField(rffi.CArrayPtr(rffi.INT), "p_errno")
+
+def _threadlocalref_seeme(field):
+    "NOT_RPYTHON"
+
+class _Entry(ExtRegistryEntry):
+    _about_ = _threadlocalref_seeme
+
+    def compute_result_annotation(self, s_field):
+        field = s_field.const
+        self.bookkeeper.thread_local_fields.add(field)
+
+    def specialize_call(self, hop):
+        hop.exception_cannot_occur()
diff --git a/rpython/rtyper/lltypesystem/lloperation.py b/rpython/rtyper/lltypesystem/lloperation.py
--- a/rpython/rtyper/lltypesystem/lloperation.py
+++ b/rpython/rtyper/lltypesystem/lloperation.py
@@ -546,9 +546,7 @@
     'getslice':             LLOp(canraise=(Exception,)),
     'check_and_clear_exc':  LLOp(),
 
-    'threadlocalref_get':   LLOp(sideeffects=False),
-    'threadlocalref_getaddr': LLOp(sideeffects=False),
-    'threadlocalref_set':   LLOp(),
+    'threadlocalref_addr':  LLOp(sideeffects=False),
 
     # __________ debugging __________
     'debug_view':           LLOp(),
diff --git a/rpython/translator/c/genc.py b/rpython/translator/c/genc.py
--- a/rpython/translator/c/genc.py
+++ b/rpython/translator/c/genc.py
@@ -703,8 +703,27 @@
     for node in structdeflist:
         for line in node.definition():
             print >> f, line
+    gen_threadlocal_structdef(f, database)
     print >> f, "#endif"
 
+def gen_threadlocal_structdef(f, database):
+    bk = database.translator.annotator.bookkeeper
+    if bk.thread_local_fields:
+        from rpython.translator.c.support import cdecl
+        print >> f
+        fields = list(bk.thread_local_fields)
+        fields.sort(key=lambda field: field.fieldname)
+        print >> f, '#define RPY_HAS_THREADLOCAL_S'
+        for field in fields:
+            print >> f, ('#define RPY_TLOFS_%s  offsetof(' % field.fieldname +
+                         'struct pypy_threadlocal_s, %s)' % field.fieldname)
+        print >> f, 'struct pypy_threadlocal_s {'
+        for field in fields:
+            typename = database.gettype(field.FIELDTYPE)
+            print >> f, '\t%s;' % cdecl(typename, field.fieldname)
+        print >> f, '};'
+        print >> f
+
 def gen_forwarddecl(f, database):
     print >> f, '/***********************************************************/'
     print >> f, '/***  Forward declarations                               ***/'
@@ -748,6 +767,11 @@
                 print >> f, '\tif (error) return error;'
             for line in lines:
                 print >> f, '\t'+line
+
+    bk = database.translator.annotator.bookkeeper
+    if bk.thread_local_fields:
+        print >> f, '\tRPython_ThreadLocals_ProgramInit();'
+
     print >> f, '\treturn error;'
     print >> f, '}'
 
@@ -770,6 +794,7 @@
         srcdir / 'asm.c',
         srcdir / 'instrument.c',
         srcdir / 'int.c',
+        srcdir / 'threadlocal.c',
     ]
     if _CYGWIN:
         files.append(srcdir / 'cygwin_wait.c')
diff --git a/rpython/translator/c/node.py b/rpython/translator/c/node.py
--- a/rpython/translator/c/node.py
+++ b/rpython/translator/c/node.py
@@ -966,30 +966,12 @@
                 args.append('0')
         yield 'RPyOpaque_SETUP_%s(%s);' % (T.tag, ', '.join(args))
 
-class ThreadLocalRefOpaqueNode(ContainerNode):
-    nodekind = 'tlrefopaque'
-
-    def basename(self):
-        return self.obj._name
-
-    def enum_dependencies(self):
-        return []
-
-    def initializationexpr(self, decoration=''):
-        return ['0']
-
-    def startupcode(self):
-        p = self.getptrname()
-        yield 'RPyThreadStaticTLS_Create(%s);' % (p,)
-
 
 def opaquenode_factory(db, T, obj):
     if T == RuntimeTypeInfo:
         return db.gcpolicy.rtti_node_factory()(db, T, obj)
     if T.hints.get("render_structure", False):
         return ExtType_OpaqueNode(db, T, obj)
-    if T.hints.get("threadlocalref", False):
-        return ThreadLocalRefOpaqueNode(db, T, obj)
     raise Exception("don't know about %r" % (T,))
 
 
diff --git a/rpython/translator/c/src/g_include.h b/rpython/translator/c/src/g_include.h
--- a/rpython/translator/c/src/g_include.h
+++ b/rpython/translator/c/src/g_include.h
@@ -48,3 +48,7 @@
 #ifdef __CYGWIN__
 #include "src/cygwin_wait.h"
 #endif
+
+#ifdef RPY_HAS_THREADLOCAL_S
+#include "src/threadlocal.h"
+#endif
diff --git a/rpython/translator/c/src/g_prerequisite.h b/rpython/translator/c/src/g_prerequisite.h
--- a/rpython/translator/c/src/g_prerequisite.h
+++ b/rpython/translator/c/src/g_prerequisite.h
@@ -23,6 +23,3 @@
 # define RPY_LENGTH0     1       /* array decl [0] are bad */
 # define RPY_DUMMY_VARLENGTH     /* nothing */
 #endif
-
-
-#include "src/threadlocal.h"
diff --git a/rpython/translator/c/src/threadlocal.c b/rpython/translator/c/src/threadlocal.c
--- a/rpython/translator/c/src/threadlocal.c
+++ b/rpython/translator/c/src/threadlocal.c
@@ -1,28 +1,97 @@
+#include "common_header.h"
+#include "structdef.h"
+
+#ifdef RPY_HAS_THREADLOCAL_S     /* otherwise, this file is not needed */
+
 #include 
 #include 
+#include 
 #include "src/threadlocal.h"
+#include "src/thread.h"
 
+
+static void _RPython_ThreadLocals_Init(char *p)
+{
+    struct pypy_threadlocal_s *tl = (struct pypy_threadlocal_s *)p;
+#ifdef RPY_TLOFS_p_errno
+    tl->p_errno = &errno;
+#endif
+#ifdef RPY_TLOFS_thread_ident
+    tl->thread_ident = RPyThreadGetIdent();
+#endif
+}
+
+
+/* ------------------------------------------------------------ */
+#ifdef USE___THREAD
+/* ------------------------------------------------------------ */
+
+
+__thread struct pypy_threadlocal_s pypy_threadlocal;
+
+void RPython_ThreadLocals_ProgramInit(void)
+{
+    RPython_ThreadLocals_ThreadStart();
+}
+
+void RPython_ThreadLocals_ThreadStart(void)
+{
+    _RPython_ThreadLocals_Init(&pypy_threadlocal);
+}
+
+void RPython_ThreadLocals_ThreadDie(void)
+{
+}
+
+
+/* ------------------------------------------------------------ */
+#else
+/* ------------------------------------------------------------ */
+
+
+void RPython_ThreadLocals_ProgramInit(void)
+{
 #ifdef _WIN32
-
-void RPyThreadTLS_Create(RPyThreadTLS *result)
-{
-    *result = TlsAlloc();
-    if (*result == TLS_OUT_OF_INDEXES) {
+    pypy_threadlocal_key = TlsAlloc();
+    if (pypy_threadlocal_key == TLS_OUT_OF_INDEXES)
+#else
+    if (pthread_key_create(&pypy_threadlocal_key, NULL) != 0)
+#endif
+    {
         fprintf(stderr, "Internal RPython error: "
                         "out of thread-local storage indexes");
         abort();
     }
+    RPython_ThreadLocals_ThreadStart();
 }
 
-#else
-
-void RPyThreadTLS_Create(RPyThreadTLS *result)
+void RPython_ThreadLocals_ThreadStart(void)
 {
-    if (pthread_key_create(result, NULL) != 0) {
+    char *p = malloc(sizeof(struct pypy_threadlocal_s));
+    if (!p) {
         fprintf(stderr, "Internal RPython error: "
-                        "out of thread-local storage keys");
+                        "out of memory for the thread-local storage");
         abort();
     }
+    _RPython_ThreadLocals_Init(p);
+#ifdef _WIN32
+    TlsSetValue(pypy_threadlocal_key, p);
+#else
+    pthread_setspecific(pypy_threadlocal_key, p);
+#endif
 }
 
+void RPython_ThreadLocals_ThreadDie(void)
+{
+    char *p;
+    OP_THREADLOCALREF_ADDR(p);
+    free(p);
+}
+
+
+/* ------------------------------------------------------------ */
 #endif
+/* ------------------------------------------------------------ */
+
+
+#endif  /* RPY_HAS_THREADLOCAL_S */
diff --git a/rpython/translator/c/src/threadlocal.h b/rpython/translator/c/src/threadlocal.h
--- a/rpython/translator/c/src/threadlocal.h
+++ b/rpython/translator/c/src/threadlocal.h
@@ -5,48 +5,54 @@
 #include 
 
 
-#ifdef _WIN32
+#ifndef RPY_HAS_THREADLOCAL_S
+#  error "src/threadlocal.h should only be included if RPY_HAS_THREADLOCAL_S"
+#endif
+
+
+/* ------------------------------------------------------------ */
+#ifdef USE___THREAD
+/* ------------------------------------------------------------ */
+
+
+/* Use the '__thread' specifier, so far only on Linux */
+
+RPY_EXTERN __thread struct pypy_threadlocal_s pypy_threadlocal;
+#define OP_THREADLOCALREF_ADDR(r)    r = &pypy_threadlocal
+
+
+/* ------------------------------------------------------------ */
+#elif _WIN32
+/* ------------------------------------------------------------ */
+
 
 #include 
 #include 
-#define __thread __declspec(thread)
-typedef DWORD RPyThreadTLS;
-#define RPyThreadTLS_Get(key)		TlsGetValue(key)
-#define RPyThreadTLS_Set(key, value)	TlsSetValue(key, value)
 
+RPY_EXTERN DWORD pypy_threadlocal_key;
+#define OP_THREADLOCALREF_ADDR(r)    r = TlsGetValue(pypy_threadlocal_key)
+
+
+/* ------------------------------------------------------------ */
 #else
+/* ------------------------------------------------------------ */
+
+
+/* Other POSIX systems: use the pthread API */
 
 #include 
-typedef pthread_key_t RPyThreadTLS;
-#define RPyThreadTLS_Get(key)		pthread_getspecific(key)
-#define RPyThreadTLS_Set(key, value)	pthread_setspecific(key, value)
 
+RPY_EXTERN pthread_key_t pypy_threadlocal_key;
+#define OP_THREADLOCALREF_ADDR(r)  r = pthread_getspecific(pypy_threadlocal_key)
+
+
+/* ------------------------------------------------------------ */
 #endif
+/* ------------------------------------------------------------ */
 
 
-#ifdef USE___THREAD
-
-#define RPyThreadStaticTLS                  __thread void *
-#define RPyThreadStaticTLS_Create(tls)      (void)0
-#define RPyThreadStaticTLS_Get(tls)         tls
-#define RPyThreadStaticTLS_Set(tls, value)  tls = value
-#define OP_THREADLOCALREF_GETADDR(tlref, ptr)  ptr = tlref
-
-#endif
-
-#ifndef RPyThreadStaticTLS
-
-#define RPyThreadStaticTLS             RPyThreadTLS
-#define RPyThreadStaticTLS_Create(key) RPyThreadTLS_Create(key)
-#define RPyThreadStaticTLS_Get(key)    RPyThreadTLS_Get(key)
-#define RPyThreadStaticTLS_Set(key, value) RPyThreadTLS_Set(key, value)
-RPY_EXTERN void RPyThreadTLS_Create(RPyThreadTLS *result);
-
-#endif
-
-
-#define OP_THREADLOCALREF_SET(tlref, ptr, _) RPyThreadStaticTLS_Set(*tlref, ptr)
-#define OP_THREADLOCALREF_GET(tlref, ptr)   ptr = RPyThreadStaticTLS_Get(*tlref)
-
+RPY_EXTERN void RPython_ThreadLocals_ProgramInit(void);
+RPY_EXTERN void RPython_ThreadLocals_ThreadStart(void);
+RPY_EXTERN void RPython_ThreadLocals_ThreadDie(void);
 
 #endif /* _SRC_THREADLOCAL_H */

From noreply at buildbot.pypy.org  Sat Nov 22 18:36:47 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Sat, 22 Nov 2014 18:36:47 +0100 (CET)
Subject: [pypy-commit] pypy portable-threadlocal: small fixes
Message-ID: <20141122173647.9CECA1C022D@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: portable-threadlocal
Changeset: r74633:cba625508157
Date: 2014-11-22 18:36 +0100
http://bitbucket.org/pypy/pypy/changeset/cba625508157/

Log:	small fixes

diff --git a/rpython/rlib/rthread.py b/rpython/rlib/rthread.py
--- a/rpython/rlib/rthread.py
+++ b/rpython/rlib/rthread.py
@@ -316,7 +316,7 @@
         def set(value):
             assert isinstance(value, Cls) or value is None
             if we_are_translated():
-                from rpython.rtyper.annlowlevel import cast_instance_to_base_ptr
+                from rpython.rtyper.annlowlevel import cast_instance_to_gcref
                 from rpython.rlib.rgc import _make_sure_does_not_move
                 from rpython.rlib.objectmodel import running_on_llinterp
                 gcref = cast_instance_to_gcref(value)
diff --git a/rpython/translator/c/src/threadlocal.c b/rpython/translator/c/src/threadlocal.c
--- a/rpython/translator/c/src/threadlocal.c
+++ b/rpython/translator/c/src/threadlocal.c
@@ -6,18 +6,19 @@
 #include 
 #include 
 #include 
+#include 
 #include "src/threadlocal.h"
 #include "src/thread.h"
 
 
-static void _RPython_ThreadLocals_Init(char *p)
+static void _RPython_ThreadLocals_Init(void *p)
 {
-    struct pypy_threadlocal_s *tl = (struct pypy_threadlocal_s *)p;
+    memset(p, 0, sizeof(struct pypy_threadlocal_s));
 #ifdef RPY_TLOFS_p_errno
-    tl->p_errno = &errno;
+    ((struct pypy_threadlocal_s *)p)->p_errno = &errno;
 #endif
 #ifdef RPY_TLOFS_thread_ident
-    tl->thread_ident = RPyThreadGetIdent();
+    ((struct pypy_threadlocal_s *)p)->thread_ident = RPyThreadGetIdent();
 #endif
 }
 
@@ -67,7 +68,7 @@
 
 void RPython_ThreadLocals_ThreadStart(void)
 {
-    char *p = malloc(sizeof(struct pypy_threadlocal_s));
+    void *p = malloc(sizeof(struct pypy_threadlocal_s));
     if (!p) {
         fprintf(stderr, "Internal RPython error: "
                         "out of memory for the thread-local storage");
@@ -83,7 +84,7 @@
 
 void RPython_ThreadLocals_ThreadDie(void)
 {
-    char *p;
+    void *p;
     OP_THREADLOCALREF_ADDR(p);
     free(p);
 }

From noreply at buildbot.pypy.org  Sat Nov 22 18:51:02 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Sat, 22 Nov 2014 18:51:02 +0100 (CET)
Subject: [pypy-commit] pypy portable-threadlocal: More fixes. Now basically
 works on the framework GC too.
Message-ID: <20141122175102.0915B1C022D@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: portable-threadlocal
Changeset: r74634:e628e1043040
Date: 2014-11-22 18:50 +0100
http://bitbucket.org/pypy/pypy/changeset/e628e1043040/

Log:	More fixes. Now basically works on the framework GC too.

diff --git a/rpython/rlib/rthread.py b/rpython/rlib/rthread.py
--- a/rpython/rlib/rthread.py
+++ b/rpython/rlib/rthread.py
@@ -276,12 +276,12 @@
 
         def getraw():
             _threadlocalref_seeme(self)
-            addr = llop.threadlocalref_addr(llmemory.Address)
+            addr = llop.threadlocalref_addr(rffi.CCHARP)
             return llop.raw_load(FIELDTYPE, addr, offset)
 
         def setraw(value):
             _threadlocalref_seeme(self)
-            addr = llop.threadlocalref_addr(llmemory.Address)
+            addr = llop.threadlocalref_addr(rffi.CCHARP)
             llop.raw_store(lltype.Void, addr, offset, value)
 
         self.getraw = getraw
diff --git a/rpython/rtyper/llinterp.py b/rpython/rtyper/llinterp.py
--- a/rpython/rtyper/llinterp.py
+++ b/rpython/rtyper/llinterp.py
@@ -919,20 +919,14 @@
     def op_stack_current(self):
         return 0
 
-    def op_threadlocalref_set(self, key, value):
+    def op_threadlocalref_addr(self, key, value):
+        raise NotImplementedError("threadlocalref_addr")  # XXX implement me?
         try:
             d = self.llinterpreter.tlrefsdict
         except AttributeError:
             d = self.llinterpreter.tlrefsdict = {}
         d[key._obj] = value
 
-    def op_threadlocalref_get(self, key):
-        d = self.llinterpreter.tlrefsdict
-        return d[key._obj]
-
-    def op_threadlocalref_getaddr(self, key):
-        raise NotImplementedError("threadlocalref_getaddr")
-
     # __________________________________________________________
     # operations on addresses
 
diff --git a/rpython/translator/c/src/thread_nt.c b/rpython/translator/c/src/thread_nt.c
--- a/rpython/translator/c/src/thread_nt.c
+++ b/rpython/translator/c/src/thread_nt.c
@@ -26,15 +26,6 @@
 
 static long _pypythread_stacksize = 0;
 
-/*
- * Return the thread Id instead of an handle. The Id is said to uniquely
-   identify the thread in the system
- */
-long RPyThreadGetIdent()
-{
-  return GetCurrentThreadId();
-}
-
 static void
 bootstrap(void *call)
 {
@@ -42,7 +33,7 @@
 	/* copy callobj since other thread might free it before we're done */
 	void (*func)(void) = obj->func;
 
-	obj->id = RPyThreadGetIdent();
+	obj->id = GetCurrentThreadId();
 	ReleaseSemaphore(obj->done, 1, NULL);
 	func();
 }
diff --git a/rpython/translator/c/src/thread_nt.h b/rpython/translator/c/src/thread_nt.h
--- a/rpython/translator/c/src/thread_nt.h
+++ b/rpython/translator/c/src/thread_nt.h
@@ -13,8 +13,6 @@
 
 /* prototypes */
 RPY_EXTERN
-long RPyThreadGetIdent(void);
-RPY_EXTERN
 long RPyThreadStart(void (*func)(void));
 RPY_EXTERN
 int RPyThreadLockInit(struct RPyOpaque_ThreadLock *lock);
diff --git a/rpython/translator/c/src/thread_pthread.c b/rpython/translator/c/src/thread_pthread.c
--- a/rpython/translator/c/src/thread_pthread.c
+++ b/rpython/translator/c/src/thread_pthread.c
@@ -56,30 +56,6 @@
 # endif
 #endif
 
-/* XXX This implementation is considered (to quote Tim Peters) "inherently
-   hosed" because:
-     - It does not guarantee the promise that a non-zero integer is returned.
-     - The cast to long is inherently unsafe.
-     - It is not clear that the 'volatile' (for AIX?) and ugly casting in the
-       latter return statement (for Alpha OSF/1) are any longer necessary.
-*/
-long RPyThreadGetIdent(void)
-{
-	volatile pthread_t threadid;
-	/* Jump through some hoops for Alpha OSF/1 */
-	threadid = pthread_self();
-
-#ifdef __CYGWIN__
-	/* typedef __uint32_t pthread_t; */
-	return (long) threadid;
-#else
-	if (sizeof(pthread_t) <= sizeof(long))
-		return (long) threadid;
-	else
-		return (long) *(long *) &threadid;
-#endif
-}
-
 static long _pypythread_stacksize = 0;
 
 static void *bootstrap_pthread(void *func)
diff --git a/rpython/translator/c/src/thread_pthread.h b/rpython/translator/c/src/thread_pthread.h
--- a/rpython/translator/c/src/thread_pthread.h
+++ b/rpython/translator/c/src/thread_pthread.h
@@ -60,8 +60,6 @@
 /* prototypes */
 
 RPY_EXTERN
-long RPyThreadGetIdent(void);
-RPY_EXTERN
 long RPyThreadStart(void (*func)(void));
 RPY_EXTERN
 int RPyThreadLockInit(struct RPyOpaque_ThreadLock *lock);
diff --git a/rpython/translator/c/src/threadlocal.c b/rpython/translator/c/src/threadlocal.c
--- a/rpython/translator/c/src/threadlocal.c
+++ b/rpython/translator/c/src/threadlocal.c
@@ -11,6 +11,16 @@
 #include "src/thread.h"
 
 
+#ifdef _WIN32
+#  define RPyThreadGetIdent() GetCurrentThreadId()
+#else
+#  define RPyThreadGetIdent() ((long)pthread_self())
+/* xxx This abuses pthread_self() by assuming it just returns a long.
+   According to comments in CPython's source code, the platforms where
+   it is wrong are rather old nowadays. */
+#endif
+
+
 static void _RPython_ThreadLocals_Init(void *p)
 {
     memset(p, 0, sizeof(struct pypy_threadlocal_s));
diff --git a/rpython/translator/c/src/threadlocal.h b/rpython/translator/c/src/threadlocal.h
--- a/rpython/translator/c/src/threadlocal.h
+++ b/rpython/translator/c/src/threadlocal.h
@@ -18,7 +18,7 @@
 /* Use the '__thread' specifier, so far only on Linux */
 
 RPY_EXTERN __thread struct pypy_threadlocal_s pypy_threadlocal;
-#define OP_THREADLOCALREF_ADDR(r)    r = &pypy_threadlocal
+#define OP_THREADLOCALREF_ADDR(r)    r = (char *)&pypy_threadlocal
 
 
 /* ------------------------------------------------------------ */
@@ -30,7 +30,8 @@
 #include 
 
 RPY_EXTERN DWORD pypy_threadlocal_key;
-#define OP_THREADLOCALREF_ADDR(r)    r = TlsGetValue(pypy_threadlocal_key)
+#define OP_THREADLOCALREF_ADDR(r)    r = (char *)TlsGetValue(  \
+                                           pypy_threadlocal_key)
 
 
 /* ------------------------------------------------------------ */
@@ -43,7 +44,8 @@
 #include 
 
 RPY_EXTERN pthread_key_t pypy_threadlocal_key;
-#define OP_THREADLOCALREF_ADDR(r)  r = pthread_getspecific(pypy_threadlocal_key)
+#define OP_THREADLOCALREF_ADDR(r)    r = (char *)pthread_getspecific(  \
+                                           pypy_threadlocal_key)
 
 
 /* ------------------------------------------------------------ */

From noreply at buildbot.pypy.org  Sat Nov 22 19:12:46 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Sat, 22 Nov 2014 19:12:46 +0100 (CET)
Subject: [pypy-commit] pypy portable-threadlocal: Clean up rstack.{py, c,
 h} by removing an obscure conditional include
Message-ID: <20141122181246.76FFD1C022D@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: portable-threadlocal
Changeset: r74635:d8c6c988cd8c
Date: 2014-11-22 19:10 +0100
http://bitbucket.org/pypy/pypy/changeset/d8c6c988cd8c/

Log:	Clean up rstack.{py,c,h} by removing an obscure conditional include

diff --git a/rpython/rlib/rstack.py b/rpython/rlib/rstack.py
--- a/rpython/rlib/rstack.py
+++ b/rpython/rlib/rstack.py
@@ -1,6 +1,6 @@
 """
 This file defines utilities for manipulating the stack in an
-RPython-compliant way, intended mostly for use by the Stackless PyPy.
+RPython-compliant way.  It is mainly about the stack_check() function.
 """
 
 import py
@@ -10,18 +10,11 @@
 from rpython.rlib import rgc
 from rpython.rtyper.lltypesystem import lltype, rffi
 from rpython.rtyper.lltypesystem.lloperation import llop
-from rpython.translator import cdir
-from rpython.translator.tool.cbuild import ExternalCompilationInfo
 
 # ____________________________________________________________
 
-srcdir = py.path.local(cdir) / 'src'
-compilation_info = ExternalCompilationInfo(
-        includes=['src/stack.h'],
-        separate_module_files=[srcdir / 'stack.c', srcdir / 'threadlocal.c'])
-
 def llexternal(name, args, res, _callable=None):
-    return rffi.llexternal(name, args, res, compilation_info=compilation_info,
+    return rffi.llexternal(name, args, res,
                            sandboxsafe=True, _nowrapper=True,
                            _callable=_callable)
 
diff --git a/rpython/translator/c/genc.py b/rpython/translator/c/genc.py
--- a/rpython/translator/c/genc.py
+++ b/rpython/translator/c/genc.py
@@ -707,22 +707,21 @@
     print >> f, "#endif"
 
 def gen_threadlocal_structdef(f, database):
+    from rpython.translator.c.support import cdecl
+    print >> f
     bk = database.translator.annotator.bookkeeper
-    if bk.thread_local_fields:
-        from rpython.translator.c.support import cdecl
-        print >> f
-        fields = list(bk.thread_local_fields)
-        fields.sort(key=lambda field: field.fieldname)
-        print >> f, '#define RPY_HAS_THREADLOCAL_S'
-        for field in fields:
-            print >> f, ('#define RPY_TLOFS_%s  offsetof(' % field.fieldname +
-                         'struct pypy_threadlocal_s, %s)' % field.fieldname)
-        print >> f, 'struct pypy_threadlocal_s {'
-        for field in fields:
-            typename = database.gettype(field.FIELDTYPE)
-            print >> f, '\t%s;' % cdecl(typename, field.fieldname)
-        print >> f, '};'
-        print >> f
+    fields = list(bk.thread_local_fields)
+    fields.sort(key=lambda field: field.fieldname)
+    for field in fields:
+        print >> f, ('#define RPY_TLOFS_%s  offsetof(' % field.fieldname +
+                     'struct pypy_threadlocal_s, %s)' % field.fieldname)
+    print >> f, 'struct pypy_threadlocal_s {'
+    print >> f, '\tchar *stack_end;'
+    for field in fields:
+        typename = database.gettype(field.FIELDTYPE)
+        print >> f, '\t%s;' % cdecl(typename, field.fieldname)
+    print >> f, '};'
+    print >> f
 
 def gen_forwarddecl(f, database):
     print >> f, '/***********************************************************/'
@@ -794,6 +793,7 @@
         srcdir / 'asm.c',
         srcdir / 'instrument.c',
         srcdir / 'int.c',
+        srcdir / 'stack.c',
         srcdir / 'threadlocal.c',
     ]
     if _CYGWIN:
diff --git a/rpython/translator/c/src/g_include.h b/rpython/translator/c/src/g_include.h
--- a/rpython/translator/c/src/g_include.h
+++ b/rpython/translator/c/src/g_include.h
@@ -19,6 +19,8 @@
 #include "src/address.h"
 #include "src/unichar.h"
 #include "src/llgroup.h"
+#include "src/stack.h"
+#include "src/threadlocal.h"
 
 #include "src/instrument.h"
 #include "src/asm.h"
@@ -48,7 +50,3 @@
 #ifdef __CYGWIN__
 #include "src/cygwin_wait.h"
 #endif
-
-#ifdef RPY_HAS_THREADLOCAL_S
-#include "src/threadlocal.h"
-#endif
diff --git a/rpython/translator/c/src/stack.c b/rpython/translator/c/src/stack.c
--- a/rpython/translator/c/src/stack.c
+++ b/rpython/translator/c/src/stack.c
@@ -1,6 +1,8 @@
 /* Stack operation */
+#include "common_header.h"
+#include "structdef.h"       /* for struct pypy_threadlocal_s */
 #include 
-#include 
+#include 
 #include 
 
 
@@ -9,7 +11,6 @@
 char *_LLstacktoobig_stack_end = NULL;
 long _LLstacktoobig_stack_length = MAX_STACK_SIZE;
 char _LLstacktoobig_report_error = 1;
-static RPyThreadStaticTLS end_tls_key;
 
 void LL_stack_set_length_fraction(double fraction)
 {
@@ -20,6 +21,8 @@
 {
 	long diff, max_stack_size;
 	char *baseptr, *curptr = (char*)current;
+        char *tl;
+        struct pypy_threadlocal_s *tl1;
 
 	/* The stack_end variable is updated to match the current value
 	   if it is still 0 or if we later find a 'curptr' position
@@ -27,15 +30,9 @@
 	   thread-local storage, but we try to minimize its overhead by
 	   keeping a local copy in _LLstacktoobig_stack_end. */
 
-	if (_LLstacktoobig_stack_end == NULL) {
-		/* not initialized */
-		/* XXX We assume that initialization is performed early,
-		   when there is still only one thread running.  This
-		   allows us to ignore race conditions here */
-		RPyThreadStaticTLS_Create(&end_tls_key);
-	}
-
-	baseptr = (char *) RPyThreadStaticTLS_Get(end_tls_key);
+        OP_THREADLOCALREF_ADDR(tl);
+        tl1 = (struct pypy_threadlocal_s *)tl;
+	baseptr = tl1->stack_end;
 	max_stack_size = _LLstacktoobig_stack_length;
 	if (baseptr == NULL) {
 		/* first time we see this thread */
@@ -58,7 +55,7 @@
 
 	/* update the stack base pointer to the current value */
 	baseptr = curptr;
-	RPyThreadStaticTLS_Set(end_tls_key, baseptr);
+        tl1->stack_end = baseptr;
 	_LLstacktoobig_stack_end = baseptr;
 	return 0;
 }
diff --git a/rpython/translator/c/src/stack.h b/rpython/translator/c/src/stack.h
--- a/rpython/translator/c/src/stack.h
+++ b/rpython/translator/c/src/stack.h
@@ -2,14 +2,13 @@
 /************************************************************/
  /***  C header subsection: stack operations               ***/
 
+#include 
+
+
 #ifndef MAX_STACK_SIZE
 #    define MAX_STACK_SIZE (3 << 18)    /* 768 kb */
 #endif
 
-/* This include must be done in any case to initialise
- * the header dependencies early (winsock2, before windows.h).
- * It is needed to have RPyThreadStaticTLS, too. */
-#include "threadlocal.h"
 
 RPY_EXTERN char *_LLstacktoobig_stack_end;
 RPY_EXTERN long _LLstacktoobig_stack_length;
diff --git a/rpython/translator/c/src/threadlocal.c b/rpython/translator/c/src/threadlocal.c
--- a/rpython/translator/c/src/threadlocal.c
+++ b/rpython/translator/c/src/threadlocal.c
@@ -1,14 +1,10 @@
 #include "common_header.h"
-#include "structdef.h"
-
-#ifdef RPY_HAS_THREADLOCAL_S     /* otherwise, this file is not needed */
-
+#include "structdef.h"       /* for struct pypy_threadlocal_s */
 #include 
 #include 
 #include 
 #include 
 #include "src/threadlocal.h"
-#include "src/thread.h"
 
 
 #ifdef _WIN32
@@ -103,6 +99,3 @@
 /* ------------------------------------------------------------ */
 #endif
 /* ------------------------------------------------------------ */
-
-
-#endif  /* RPY_HAS_THREADLOCAL_S */
diff --git a/rpython/translator/c/src/threadlocal.h b/rpython/translator/c/src/threadlocal.h
--- a/rpython/translator/c/src/threadlocal.h
+++ b/rpython/translator/c/src/threadlocal.h
@@ -5,11 +5,6 @@
 #include 
 
 
-#ifndef RPY_HAS_THREADLOCAL_S
-#  error "src/threadlocal.h should only be included if RPY_HAS_THREADLOCAL_S"
-#endif
-
-
 /* ------------------------------------------------------------ */
 #ifdef USE___THREAD
 /* ------------------------------------------------------------ */

From noreply at buildbot.pypy.org  Sat Nov 22 19:12:47 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Sat, 22 Nov 2014 19:12:47 +0100 (CET)
Subject: [pypy-commit] pypy portable-threadlocal: missing include
Message-ID: <20141122181247.BCD991C022D@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: portable-threadlocal
Changeset: r74636:846aafb04a5d
Date: 2014-11-22 19:12 +0100
http://bitbucket.org/pypy/pypy/changeset/846aafb04a5d/

Log:	missing include

diff --git a/rpython/translator/c/src/threadlocal.c b/rpython/translator/c/src/threadlocal.c
--- a/rpython/translator/c/src/threadlocal.c
+++ b/rpython/translator/c/src/threadlocal.c
@@ -10,6 +10,7 @@
 #ifdef _WIN32
 #  define RPyThreadGetIdent() GetCurrentThreadId()
 #else
+#  include 
 #  define RPyThreadGetIdent() ((long)pthread_self())
 /* xxx This abuses pthread_self() by assuming it just returns a long.
    According to comments in CPython's source code, the platforms where

From noreply at buildbot.pypy.org  Sat Nov 22 19:18:24 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Sat, 22 Nov 2014 19:18:24 +0100 (CET)
Subject: [pypy-commit] pypy portable-threadlocal: Use tabs here (very old
	function)
Message-ID: <20141122181824.0FA441C022D@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: portable-threadlocal
Changeset: r74637:d747e002d8cb
Date: 2014-11-22 19:16 +0100
http://bitbucket.org/pypy/pypy/changeset/d747e002d8cb/

Log:	Use tabs here (very old function)

diff --git a/rpython/translator/c/src/stack.c b/rpython/translator/c/src/stack.c
--- a/rpython/translator/c/src/stack.c
+++ b/rpython/translator/c/src/stack.c
@@ -21,8 +21,8 @@
 {
 	long diff, max_stack_size;
 	char *baseptr, *curptr = (char*)current;
-        char *tl;
-        struct pypy_threadlocal_s *tl1;
+	char *tl;
+	struct pypy_threadlocal_s *tl1;
 
 	/* The stack_end variable is updated to match the current value
 	   if it is still 0 or if we later find a 'curptr' position
@@ -30,8 +30,8 @@
 	   thread-local storage, but we try to minimize its overhead by
 	   keeping a local copy in _LLstacktoobig_stack_end. */
 
-        OP_THREADLOCALREF_ADDR(tl);
-        tl1 = (struct pypy_threadlocal_s *)tl;
+	OP_THREADLOCALREF_ADDR(tl);
+	tl1 = (struct pypy_threadlocal_s *)tl;
 	baseptr = tl1->stack_end;
 	max_stack_size = _LLstacktoobig_stack_length;
 	if (baseptr == NULL) {
@@ -55,7 +55,7 @@
 
 	/* update the stack base pointer to the current value */
 	baseptr = curptr;
-        tl1->stack_end = baseptr;
+	tl1->stack_end = baseptr;
 	_LLstacktoobig_stack_end = baseptr;
 	return 0;
 }

From noreply at buildbot.pypy.org  Sat Nov 22 19:27:48 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Sat, 22 Nov 2014 19:27:48 +0100 (CET)
Subject: [pypy-commit] pypy portable-threadlocal: rthread.get_ident()
	non-translated
Message-ID: <20141122182748.8E5EF1C31CF@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: portable-threadlocal
Changeset: r74638:b604e908b50c
Date: 2014-11-22 19:23 +0100
http://bitbucket.org/pypy/pypy/changeset/b604e908b50c/

Log:	rthread.get_ident() non-translated

diff --git a/rpython/rlib/rthread.py b/rpython/rlib/rthread.py
--- a/rpython/rlib/rthread.py
+++ b/rpython/rlib/rthread.py
@@ -84,7 +84,11 @@
 # wrappers...
 
 def get_ident():
-    return tlfield_thread_ident.getraw()
+    if we_are_translated():
+        return tlfield_thread_ident.getraw()
+    else:
+        import thread
+        return thread.get_ident()
 
 @specialize.arg(0)
 def start_new_thread(x, y):
diff --git a/rpython/rlib/test/test_rthread.py b/rpython/rlib/test/test_rthread.py
--- a/rpython/rlib/test/test_rthread.py
+++ b/rpython/rlib/test/test_rthread.py
@@ -47,6 +47,10 @@
     time.sleep(0.5)
     assert results == [True] * 15
 
+def test_get_ident():
+    import thread
+    assert get_ident() == thread.get_ident()
+
 
 class AbstractThreadTests(AbstractGCTestClass):
     use_threads = True

From noreply at buildbot.pypy.org  Sat Nov 22 20:22:30 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Sat, 22 Nov 2014 20:22:30 +0100 (CET)
Subject: [pypy-commit] pypy portable-threadlocal: in-progress
Message-ID: <20141122192230.EFD631D288E@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: portable-threadlocal
Changeset: r74639:2d35c55a318d
Date: 2014-11-22 20:22 +0100
http://bitbucket.org/pypy/pypy/changeset/2d35c55a318d/

Log:	in-progress

diff --git a/rpython/memory/gctransform/framework.py b/rpython/memory/gctransform/framework.py
--- a/rpython/memory/gctransform/framework.py
+++ b/rpython/memory/gctransform/framework.py
@@ -1080,6 +1080,9 @@
             assert not livevars, "live GC var around %s!" % (hop.spaceop,)
             hop.genop("direct_call", [self.root_walker.thread_run_ptr])
             self.pop_roots(hop, livevars)
+        else:
+            hop.rename("gc_thread_run")     # keep it around for c/gc.py,
+                                            # unless handled specially above
 
     def gct_gc_thread_start(self, hop):
         assert self.translator.config.translation.thread
@@ -1095,6 +1098,7 @@
             assert not livevars, "live GC var around %s!" % (hop.spaceop,)
             hop.genop("direct_call", [self.root_walker.thread_die_ptr])
             self.pop_roots(hop, livevars)
+        hop.rename("gc_thread_die")     # keep it around for c/gc.py
 
     def gct_gc_thread_before_fork(self, hop):
         if (self.translator.config.translation.thread
diff --git a/rpython/memory/gctransform/shadowstack.py b/rpython/memory/gctransform/shadowstack.py
--- a/rpython/memory/gctransform/shadowstack.py
+++ b/rpython/memory/gctransform/shadowstack.py
@@ -132,8 +132,12 @@
             gcdata.root_stack_top/root_stack_base is the one corresponding
             to the current thread.
             No GC operation here, e.g. no mallocs or storing in a dict!
+
+            Note that here specifically we don't call rthread.get_ident(),
+            but rthread.get_or_make_ident().  We are possibly in a fresh
+            new thread, so we need to be careful.
             """
-            tid = get_tid()
+            tid = rthread.get_or_make_ident()
             if gcdata.active_tid != tid:
                 switch_shadow_stacks(tid)
 
diff --git a/rpython/rlib/rthread.py b/rpython/rlib/rthread.py
--- a/rpython/rlib/rthread.py
+++ b/rpython/rlib/rthread.py
@@ -90,6 +90,10 @@
         import thread
         return thread.get_ident()
 
+def get_or_make_ident():
+    assert we_are_translated()
+    return tlfield_thread_ident.get_or_make_raw()
+
 @specialize.arg(0)
 def start_new_thread(x, y):
     """In RPython, no argument can be passed.  You have to use global
@@ -283,12 +287,18 @@
             addr = llop.threadlocalref_addr(rffi.CCHARP)
             return llop.raw_load(FIELDTYPE, addr, offset)
 
+        def get_or_make_raw():
+            _threadlocalref_seeme(self)
+            addr = llop.threadlocalref_make(rffi.CCHARP)
+            return llop.raw_load(FIELDTYPE, addr, offset)
+
         def setraw(value):
             _threadlocalref_seeme(self)
             addr = llop.threadlocalref_addr(rffi.CCHARP)
             llop.raw_store(lltype.Void, addr, offset, value)
 
         self.getraw = getraw
+        self.get_or_make_raw = get_or_make_raw
         self.setraw = setraw
 
     def _freeze_(self):
diff --git a/rpython/rtyper/llinterp.py b/rpython/rtyper/llinterp.py
--- a/rpython/rtyper/llinterp.py
+++ b/rpython/rtyper/llinterp.py
@@ -920,12 +920,7 @@
         return 0
 
     def op_threadlocalref_addr(self, key, value):
-        raise NotImplementedError("threadlocalref_addr")  # XXX implement me?
-        try:
-            d = self.llinterpreter.tlrefsdict
-        except AttributeError:
-            d = self.llinterpreter.tlrefsdict = {}
-        d[key._obj] = value
+        raise NotImplementedError("threadlocalref_addr")
 
     # __________________________________________________________
     # operations on addresses
diff --git a/rpython/rtyper/lltypesystem/lloperation.py b/rpython/rtyper/lltypesystem/lloperation.py
--- a/rpython/rtyper/lltypesystem/lloperation.py
+++ b/rpython/rtyper/lltypesystem/lloperation.py
@@ -547,6 +547,7 @@
     'check_and_clear_exc':  LLOp(),
 
     'threadlocalref_addr':  LLOp(sideeffects=False),
+    'threadlocalref_make':  LLOp(),
 
     # __________ debugging __________
     'debug_view':           LLOp(),
diff --git a/rpython/translator/c/gc.py b/rpython/translator/c/gc.py
--- a/rpython/translator/c/gc.py
+++ b/rpython/translator/c/gc.py
@@ -71,13 +71,13 @@
         return ''
 
     def OP_GC_THREAD_RUN(self, funcgen, op):
-        return ''
+        return '{ char *r; OP_THREADLOCALREF_MAKE(r); (void)r; } '
 
     def OP_GC_THREAD_START(self, funcgen, op):
         return ''
 
     def OP_GC_THREAD_DIE(self, funcgen, op):
-        return ''
+        return 'RPython_ThreadLocals_ThreadDie();'
 
     def OP_GC_THREAD_BEFORE_FORK(self, funcgen, op):
         return '%s = NULL;' % funcgen.expr(op.result)
diff --git a/rpython/translator/c/genc.py b/rpython/translator/c/genc.py
--- a/rpython/translator/c/genc.py
+++ b/rpython/translator/c/genc.py
@@ -716,6 +716,7 @@
         print >> f, ('#define RPY_TLOFS_%s  offsetof(' % field.fieldname +
                      'struct pypy_threadlocal_s, %s)' % field.fieldname)
     print >> f, 'struct pypy_threadlocal_s {'
+    print >> f, '\tint ready;'
     print >> f, '\tchar *stack_end;'
     for field in fields:
         typename = database.gettype(field.FIELDTYPE)
diff --git a/rpython/translator/c/src/threadlocal.c b/rpython/translator/c/src/threadlocal.c
--- a/rpython/translator/c/src/threadlocal.c
+++ b/rpython/translator/c/src/threadlocal.c
@@ -18,7 +18,7 @@
 #endif
 
 
-static void _RPython_ThreadLocals_Init(void *p)
+static void _RPy_ThreadLocals_Init(void *p)
 {
     memset(p, 0, sizeof(struct pypy_threadlocal_s));
 #ifdef RPY_TLOFS_p_errno
@@ -27,6 +27,7 @@
 #ifdef RPY_TLOFS_thread_ident
     ((struct pypy_threadlocal_s *)p)->thread_ident = RPyThreadGetIdent();
 #endif
+    ((struct pypy_threadlocal_s *)p)->ready = 1;
 }
 
 
@@ -35,20 +36,24 @@
 /* ------------------------------------------------------------ */
 
 
+/* in this situation, we always have one full 'struct pypy_threadlocal_s'
+   available, managed by gcc. */
 __thread struct pypy_threadlocal_s pypy_threadlocal;
 
 void RPython_ThreadLocals_ProgramInit(void)
 {
-    RPython_ThreadLocals_ThreadStart();
+    _RPy_ThreadLocals_Init(&pypy_threadlocal);
 }
 
-void RPython_ThreadLocals_ThreadStart(void)
+char *_RPython_ThreadLocals_Build(void)
 {
-    _RPython_ThreadLocals_Init(&pypy_threadlocal);
+    _RPy_ThreadLocals_Init(&pypy_threadlocal);
+    return (char *)&pypy_threadlocal;
 }
 
 void RPython_ThreadLocals_ThreadDie(void)
 {
+    pypy_threadlocal.ready = 0;
 }
 
 
@@ -57,6 +62,16 @@
 /* ------------------------------------------------------------ */
 
 
+/* this is the case where the 'struct pypy_threadlocal_s' is allocated
+   explicitly, with malloc()/free(), and attached to (a single) thread-
+   local key using the API of Windows or pthread. */
+
+#ifdef _WIN32
+#  define _RPy_ThreadLocals_Set(p)  TlsSetValue(pypy_threadlocal_key, p)
+#else
+#  define _RPy_ThreadLocals_Set(p)  pthread_setspecific(pypy_threadlocal_key, p)
+#endif
+
 void RPython_ThreadLocals_ProgramInit(void)
 {
 #ifdef _WIN32
@@ -70,10 +85,10 @@
                         "out of thread-local storage indexes");
         abort();
     }
-    RPython_ThreadLocals_ThreadStart();
+    _RPython_ThreadLocals_Build();
 }
 
-void RPython_ThreadLocals_ThreadStart(void)
+char *_RPython_ThreadLocals_Build(void)
 {
     void *p = malloc(sizeof(struct pypy_threadlocal_s));
     if (!p) {
@@ -81,18 +96,16 @@
                         "out of memory for the thread-local storage");
         abort();
     }
-    _RPython_ThreadLocals_Init(p);
-#ifdef _WIN32
-    TlsSetValue(pypy_threadlocal_key, p);
-#else
-    pthread_setspecific(pypy_threadlocal_key, p);
-#endif
+    _RPy_ThreadLocals_Init(p);
+    _RPy_ThreadLocals_Set(p);
+    return (char *)p;
 }
 
 void RPython_ThreadLocals_ThreadDie(void)
 {
     void *p;
     OP_THREADLOCALREF_ADDR(p);
+    _RPy_ThreadLocals_Set(NULL);
     free(p);
 }
 
diff --git a/rpython/translator/c/src/threadlocal.h b/rpython/translator/c/src/threadlocal.h
--- a/rpython/translator/c/src/threadlocal.h
+++ b/rpython/translator/c/src/threadlocal.h
@@ -5,6 +5,22 @@
 #include 
 
 
+/* RPython_ThreadLocals_ProgramInit() is called once at program start-up. */
+RPY_EXTERN void RPython_ThreadLocals_ProgramInit(void);
+
+/* RPython_ThreadLocals_ThreadDie() is called in a thread that is about
+   to die. */
+RPY_EXTERN void RPython_ThreadLocals_ThreadDie(void);
+
+/* There are two llops: 'threadlocalref_addr' and 'threadlocalref_make'.
+   They both return the address of the thread-local structure (of the
+   C type 'struct pypy_threadlocal_s').  The difference is that
+   OP_THREADLOCALREF_MAKE() checks if we have initialized this thread-
+   local structure in the current thread, and if not, calls the following
+   helper. */
+RPY_EXTERN char *_RPython_ThreadLocals_Build(void);
+
+
 /* ------------------------------------------------------------ */
 #ifdef USE___THREAD
 /* ------------------------------------------------------------ */
@@ -14,6 +30,9 @@
 
 RPY_EXTERN __thread struct pypy_threadlocal_s pypy_threadlocal;
 #define OP_THREADLOCALREF_ADDR(r)    r = (char *)&pypy_threadlocal
+#define OP_THREADLOCALREF_MAKE(r)                               \
+    (OP_THREADLOCALREF_ADDR(r),                                 \
+     (pypy_threadlocal.ready || (r = _RPython_ThreadLocals_Build())))
 
 
 /* ------------------------------------------------------------ */
@@ -27,6 +46,9 @@
 RPY_EXTERN DWORD pypy_threadlocal_key;
 #define OP_THREADLOCALREF_ADDR(r)    r = (char *)TlsGetValue(  \
                                            pypy_threadlocal_key)
+#define OP_THREADLOCALREF_MAKE(r)                       \
+    (OP_THREADLOCALREF_ADDR(r),                         \
+     ((r) || (r = _RPython_ThreadLocals_Build())))
 
 
 /* ------------------------------------------------------------ */
@@ -41,6 +63,9 @@
 RPY_EXTERN pthread_key_t pypy_threadlocal_key;
 #define OP_THREADLOCALREF_ADDR(r)    r = (char *)pthread_getspecific(  \
                                            pypy_threadlocal_key)
+#define OP_THREADLOCALREF_MAKE(r)                       \
+    (OP_THREADLOCALREF_ADDR(r),                         \
+     ((r) || (r = _RPython_ThreadLocals_Build())))
 
 
 /* ------------------------------------------------------------ */
@@ -48,8 +73,4 @@
 /* ------------------------------------------------------------ */
 
 
-RPY_EXTERN void RPython_ThreadLocals_ProgramInit(void);
-RPY_EXTERN void RPython_ThreadLocals_ThreadStart(void);
-RPY_EXTERN void RPython_ThreadLocals_ThreadDie(void);
-
 #endif /* _SRC_THREADLOCAL_H */

From noreply at buildbot.pypy.org  Sat Nov 22 20:46:47 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Sat, 22 Nov 2014 20:46:47 +0100 (CET)
Subject: [pypy-commit] pypy portable-threadlocal: For debugging,
 in ThreadDie(), set the thread-local data to 0xDD
Message-ID: <20141122194647.456F51D2859@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: portable-threadlocal
Changeset: r74640:361743064297
Date: 2014-11-22 20:46 +0100
http://bitbucket.org/pypy/pypy/changeset/361743064297/

Log:	For debugging, in ThreadDie(), set the thread-local data to 0xDD

diff --git a/rpython/translator/c/src/threadlocal.c b/rpython/translator/c/src/threadlocal.c
--- a/rpython/translator/c/src/threadlocal.c
+++ b/rpython/translator/c/src/threadlocal.c
@@ -53,6 +53,8 @@
 
 void RPython_ThreadLocals_ThreadDie(void)
 {
+    memset(&pypy_threadlocal, 0xDD,
+           sizeof(struct pypy_threadlocal_s));  /* debug */
     pypy_threadlocal.ready = 0;
 }
 
@@ -106,6 +108,7 @@
     void *p;
     OP_THREADLOCALREF_ADDR(p);
     _RPy_ThreadLocals_Set(NULL);
+    memset(p, 0xDD, sizeof(struct pypy_threadlocal_s));  /* debug */
     free(p);
 }
 

From noreply at buildbot.pypy.org  Sat Nov 22 21:54:59 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Sat, 22 Nov 2014 21:54:59 +0100 (CET)
Subject: [pypy-commit] pypy portable-threadlocal: Kill RPyThreadGetIdent(),
 and just copy the (easy) logic in module/_ssl.
Message-ID: <20141122205459.BD41D1C3140@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: portable-threadlocal
Changeset: r74641:d622788cadbe
Date: 2014-11-22 21:54 +0100
http://bitbucket.org/pypy/pypy/changeset/d622788cadbe/

Log:	Kill RPyThreadGetIdent(), and just copy the (easy) logic in
	module/_ssl.

diff --git a/pypy/module/_ssl/thread_lock.py b/pypy/module/_ssl/thread_lock.py
--- a/pypy/module/_ssl/thread_lock.py
+++ b/pypy/module/_ssl/thread_lock.py
@@ -24,12 +24,19 @@
 
 separate_module_source = """
 #include 
+#ifndef _WIN32
+# include 
+#endif
 
 static unsigned int _ssl_locks_count = 0;
 static struct RPyOpaque_ThreadLock *_ssl_locks;
 
 static unsigned long _ssl_thread_id_function(void) {
-    return RPyThreadGetIdent();
+#ifdef _WIN32
+    return (unsigned long)GetCurrentThreadId();
+#else
+    return (unsigned long)pthread_self();
+#endif
 }
 
 static void _ssl_thread_locking_function(int mode, int n, const char *file,
diff --git a/rpython/translator/c/src/threadlocal.c b/rpython/translator/c/src/threadlocal.c
--- a/rpython/translator/c/src/threadlocal.c
+++ b/rpython/translator/c/src/threadlocal.c
@@ -4,20 +4,12 @@
 #include 
 #include 
 #include 
+#ifndef _WIN32
+# include 
+#endif
 #include "src/threadlocal.h"
 
 
-#ifdef _WIN32
-#  define RPyThreadGetIdent() GetCurrentThreadId()
-#else
-#  include 
-#  define RPyThreadGetIdent() ((long)pthread_self())
-/* xxx This abuses pthread_self() by assuming it just returns a long.
-   According to comments in CPython's source code, the platforms where
-   it is wrong are rather old nowadays. */
-#endif
-
-
 static void _RPy_ThreadLocals_Init(void *p)
 {
     memset(p, 0, sizeof(struct pypy_threadlocal_s));
@@ -25,7 +17,15 @@
     ((struct pypy_threadlocal_s *)p)->p_errno = &errno;
 #endif
 #ifdef RPY_TLOFS_thread_ident
-    ((struct pypy_threadlocal_s *)p)->thread_ident = RPyThreadGetIdent();
+    ((struct pypy_threadlocal_s *)p)->thread_ident =
+#    ifdef _WIN32
+        GetCurrentThreadId();
+#    else
+        (long)pthread_self();    /* xxx This abuses pthread_self() by
+                  assuming it just returns a integer.  According to
+                  comments in CPython's source code, the platforms
+                  where it is not the case are rather old nowadays. */
+#    endif
 #endif
     ((struct pypy_threadlocal_s *)p)->ready = 1;
 }

From noreply at buildbot.pypy.org  Sat Nov 22 22:44:54 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Sat, 22 Nov 2014 22:44:54 +0100 (CET)
Subject: [pypy-commit] pypy portable-threadlocal: add comments
Message-ID: <20141122214454.A9CE01C3140@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: portable-threadlocal
Changeset: r74642:b3923721d666
Date: 2014-11-22 22:02 +0100
http://bitbucket.org/pypy/pypy/changeset/b3923721d666/

Log:	add comments

diff --git a/rpython/translator/c/gc.py b/rpython/translator/c/gc.py
--- a/rpython/translator/c/gc.py
+++ b/rpython/translator/c/gc.py
@@ -71,12 +71,19 @@
         return ''
 
     def OP_GC_THREAD_RUN(self, funcgen, op):
+        # The gc transformer leaves this operation in the graphs
+        # in all cases except with framework+shadowstack.  In that
+        # case the operation is removed because redundant with
+        # rthread.get_or_make_ident().
         return '{ char *r; OP_THREADLOCALREF_MAKE(r); (void)r; } '
 
     def OP_GC_THREAD_START(self, funcgen, op):
         return ''
 
     def OP_GC_THREAD_DIE(self, funcgen, op):
+        # The gc transformer leaves this operation in the graphs
+        # (but may insert a call to a gcrootfinder-specific
+        # function just before).
         return 'RPython_ThreadLocals_ThreadDie();'
 
     def OP_GC_THREAD_BEFORE_FORK(self, funcgen, op):

From noreply at buildbot.pypy.org  Sat Nov 22 22:44:55 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Sat, 22 Nov 2014 22:44:55 +0100 (CET)
Subject: [pypy-commit] pypy portable-threadlocal: one more
Message-ID: <20141122214455.EEE1D1C3140@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: portable-threadlocal
Changeset: r74643:ab0e68bc2ce0
Date: 2014-11-22 22:44 +0100
http://bitbucket.org/pypy/pypy/changeset/ab0e68bc2ce0/

Log:	one more

diff --git a/pypy/module/cpyext/src/pythread.c b/pypy/module/cpyext/src/pythread.c
--- a/pypy/module/cpyext/src/pythread.c
+++ b/pypy/module/cpyext/src/pythread.c
@@ -1,11 +1,18 @@
 #include 
+#ifndef _WIN32
+# include 
+#endif
 #include "pythread.h"
 #include "src/thread.h"
 
 long
 PyThread_get_thread_ident(void)
 {
-    return RPyThreadGetIdent();
+#ifdef _WIN32
+    return (long)GetCurrentThreadId();
+#else
+    return (long)pthread_self();
+#endif
 }
 
 PyThread_type_lock

From noreply at buildbot.pypy.org  Sun Nov 23 08:52:40 2014
From: noreply at buildbot.pypy.org (fijal)
Date: Sun, 23 Nov 2014 08:52:40 +0100 (CET)
Subject: [pypy-commit] pypy optresult: A bunch of progress
Message-ID: <20141123075240.95DC21C02EE@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: optresult
Changeset: r74644:2806b8d08734
Date: 2014-11-23 09:52 +0200
http://bitbucket.org/pypy/pypy/changeset/2806b8d08734/

Log:	A bunch of progress

diff --git a/rpython/jit/backend/llgraph/runner.py b/rpython/jit/backend/llgraph/runner.py
--- a/rpython/jit/backend/llgraph/runner.py
+++ b/rpython/jit/backend/llgraph/runner.py
@@ -497,7 +497,9 @@
         array = a._obj
         return support.cast_result(descr.A.OF, array.getitem(index))
 
-    bh_getarrayitem_gc_pure = bh_getarrayitem_gc
+    bh_getarrayitem_gc_pure_i = bh_getarrayitem_gc
+    bh_getarrayitem_gc_pure_r = bh_getarrayitem_gc
+    bh_getarrayitem_gc_pure_f = bh_getarrayitem_gc
     bh_getarrayitem_gc_i = bh_getarrayitem_gc
     bh_getarrayitem_gc_r = bh_getarrayitem_gc
     bh_getarrayitem_gc_f = bh_getarrayitem_gc
diff --git a/rpython/jit/metainterp/executor.py b/rpython/jit/metainterp/executor.py
--- a/rpython/jit/metainterp/executor.py
+++ b/rpython/jit/metainterp/executor.py
@@ -466,6 +466,15 @@
     else:
         assert lltype.typeOf(value) == llmemory.GCREF
         return ConstPtr(value)
+
+def constant_from_op(op):
+    if op.type == 'i':
+        return ConstInt(op.getint())
+    elif op.type == 'r':
+        return ConstPtr(op.getref_base())
+    else:
+        assert op.type == 'f'
+        return ConstFloat(op.getfloatstorage())
     
 def execute_nonspec_const(cpu, metainterp, opnum, argboxes, descr=None,
                           type='i'):
diff --git a/rpython/jit/metainterp/history.py b/rpython/jit/metainterp/history.py
--- a/rpython/jit/metainterp/history.py
+++ b/rpython/jit/metainterp/history.py
@@ -780,6 +780,11 @@
         self.operations.append(op)
         return op
 
+    def record_default_val(self, opnum, argboxes, descr=None):
+        op = ResOperation(opnum, argboxes, descr)
+        self.operations.append(op)
+        return op        
+
     def substitute_operation(self, position, opnum, argboxes, descr=None):
         resbox = self.operations[position].result
         op = ResOperation(opnum, argboxes, resbox, descr)
diff --git a/rpython/jit/metainterp/optimizeopt/optimizer.py b/rpython/jit/metainterp/optimizeopt/optimizer.py
--- a/rpython/jit/metainterp/optimizeopt/optimizer.py
+++ b/rpython/jit/metainterp/optimizeopt/optimizer.py
@@ -340,20 +340,6 @@
             return self.optimizer.optpure.get_pure_result(key)
         return None
 
-    def getfield_pure_for_descr(self, descr):
-        if descr.is_pointer_field():
-            return rop.GETFIELD_GC_PURE_R
-        elif descr.is_float_field():
-            return rop.GETFIELD_GC_PURE_F
-        return rop.GETFIELD_GC_PURE_I
-
-    def getarrayitem_pure_for_descr(self, descr):
-        if descr.is_array_of_pointers():
-            return rop.GETARRAYITEM_GC_PURE_R
-        elif descr.is_array_of_floats():
-            return rop.GETARRAYITEM_GC_PURE_F
-        return rop.GETARRAYITEM_GC_PURE_I
-
     def setup(self):
         pass
 
@@ -716,7 +702,7 @@
     # dissabled and unrolling emits some SAME_AS ops to setup the
     # optimizier state. These needs to always be optimized out.
     def optimize_SAME_AS_I(self, op):
-        self.make_equal_to(op.result, self.getvalue(op.getarg(0)))
+        self.make_equal_to(op, self.getvalue(op.getarg(0)))
     optimize_SAME_AS_R = optimize_SAME_AS_I
     optimize_SAME_AS_F = optimize_SAME_AS_I
 
diff --git a/rpython/jit/metainterp/optimizeopt/simplify.py b/rpython/jit/metainterp/optimizeopt/simplify.py
--- a/rpython/jit/metainterp/optimizeopt/simplify.py
+++ b/rpython/jit/metainterp/optimizeopt/simplify.py
@@ -1,6 +1,6 @@
 from rpython.jit.metainterp.optimizeopt.optimizer import Optimization
 from rpython.jit.metainterp.optimizeopt.util import make_dispatcher_method
-from rpython.jit.metainterp.resoperation import ResOperation, rop
+from rpython.jit.metainterp.resoperation import ResOperation, rop, OpHelpers
 from rpython.jit.metainterp.history import TargetToken, JitCellToken
 
 class OptSimplify(Optimization):
@@ -15,9 +15,9 @@
         Optimization.emit_operation(self, op)
 
     def optimize_CALL_PURE_I(self, op):
-        args = op.getarglist()
-        self.emit_operation(ResOperation(rop.CALL, args, op.result,
-                                         op.getdescr()))
+        opnum = OpHelpers.call_for_descr(op.getdescr())
+        newop = self.optimizer.replace_op_with(op, opnum)
+        self.emit_operation(newop)
     optimize_CALL_PURE_R = optimize_CALL_PURE_I
     optimize_CALL_PURE_F = optimize_CALL_PURE_I
     optimize_CALL_PURE_N = optimize_CALL_PURE_I
diff --git a/rpython/jit/metainterp/pyjitpl.py b/rpython/jit/metainterp/pyjitpl.py
--- a/rpython/jit/metainterp/pyjitpl.py
+++ b/rpython/jit/metainterp/pyjitpl.py
@@ -157,13 +157,13 @@
         return env
 
     def replace_active_box_in_frame(self, oldbox, newbox):
-        if isinstance(oldbox, history.BoxInt):
+        if oldbox.type == 'i':
             count = self.jitcode.num_regs_i()
             registers = self.registers_i
-        elif isinstance(oldbox, history.BoxPtr):
+        elif oldbox.type == 'r':
             count = self.jitcode.num_regs_r()
             registers = self.registers_r
-        elif isinstance(oldbox, history.BoxFloat):
+        elif oldbox.type == 'f':
             count = self.jitcode.num_regs_f()
             registers = self.registers_f
         else:
@@ -377,7 +377,7 @@
             if not isinstance(box, Const):
                 self.metainterp.generate_guard(rop.GUARD_ISNULL, box,
                                                resumepc=orgpc)
-                promoted_box = box.constbox()
+                promoted_box = executor.constant_from_op(box)
                 self.metainterp.replace_box(box, promoted_box)
         return value
 
@@ -440,23 +440,30 @@
         if tobox:
             # sanity check: see whether the current array value
             # corresponds to what the cache thinks the value is
+            xxx
             resbox = executor.execute(self.metainterp.cpu, self.metainterp, op,
                                       arraydescr, arraybox, indexbox)
             assert resbox.constbox().same_constant(tobox.constbox())
             return tobox
-        resbox = self.execute_with_descr(op, arraydescr, arraybox, indexbox)
+        resop = self.execute_with_descr(op, arraydescr, arraybox, indexbox)
         self.metainterp.heapcache.getarrayitem_now_known(
-                arraybox, indexbox, resbox, arraydescr)
-        return resbox
+                arraybox, indexbox, resop, arraydescr)
+        return resop
 
     @arguments("box", "box", "descr")
-    def _opimpl_getarrayitem_gc_any(self, arraybox, indexbox, arraydescr):
-        return self._do_getarrayitem_gc_any(rop.GETARRAYITEM_GC, arraybox,
+    def opimpl_getarrayitem_gc_i(self, arraybox, indexbox, arraydescr):
+        return self._do_getarrayitem_gc_any(rop.GETARRAYITEM_GC_I, arraybox,
                                             indexbox, arraydescr)
 
-    opimpl_getarrayitem_gc_i = _opimpl_getarrayitem_gc_any
-    opimpl_getarrayitem_gc_r = _opimpl_getarrayitem_gc_any
-    opimpl_getarrayitem_gc_f = _opimpl_getarrayitem_gc_any
+    @arguments("box", "box", "descr")
+    def opimpl_getarrayitem_gc_r(self, arraybox, indexbox, arraydescr):
+        return self._do_getarrayitem_gc_any(rop.GETARRAYITEM_GC_R, arraybox,
+                                            indexbox, arraydescr)
+
+    @arguments("box", "box", "descr")
+    def opimpl_getarrayitem_gc_f(self, arraybox, indexbox, arraydescr):
+        return self._do_getarrayitem_gc_any(rop.GETARRAYITEM_GC_F, arraybox,
+                                            indexbox, arraydescr)
 
     @arguments("box", "box", "descr")
     def _opimpl_getarrayitem_raw_any(self, arraybox, indexbox, arraydescr):
@@ -476,20 +483,40 @@
     opimpl_getarrayitem_raw_f_pure = _opimpl_getarrayitem_raw_pure_any
 
     @arguments("box", "box", "descr")
-    def _opimpl_getarrayitem_gc_pure_any(self, arraybox, indexbox, arraydescr):
+    def opimpl_getarrayitem_gc_i_pure(self, arraybox, indexbox, arraydescr):
         if isinstance(arraybox, ConstPtr) and isinstance(indexbox, ConstInt):
             # if the arguments are directly constants, bypass the heapcache
             # completely
-            resbox = executor.execute(self.metainterp.cpu, self.metainterp,
-                                      rop.GETARRAYITEM_GC_PURE, arraydescr,
+            val = executor.execute(self.metainterp.cpu, self.metainterp,
+                                      rop.GETARRAYITEM_GC_PURE_I, arraydescr,
                                       arraybox, indexbox)
-            return resbox.constbox()
-        return self._do_getarrayitem_gc_any(rop.GETARRAYITEM_GC_PURE, arraybox,
-                                            indexbox, arraydescr)
+            return executor.wrap_constant(val)
+        return self._do_getarrayitem_gc_any(rop.GETARRAYITEM_GC_PURE_I,
+                                            arraybox, indexbox, arraydescr)
 
-    opimpl_getarrayitem_gc_i_pure = _opimpl_getarrayitem_gc_pure_any
-    opimpl_getarrayitem_gc_r_pure = _opimpl_getarrayitem_gc_pure_any
-    opimpl_getarrayitem_gc_f_pure = _opimpl_getarrayitem_gc_pure_any
+    @arguments("box", "box", "descr")
+    def opimpl_getarrayitem_gc_f_pure(self, arraybox, indexbox, arraydescr):
+        if isinstance(arraybox, ConstPtr) and isinstance(indexbox, ConstInt):
+            # if the arguments are directly constants, bypass the heapcache
+            # completely
+            resval = executor.execute(self.metainterp.cpu, self.metainterp,
+                                      rop.GETARRAYITEM_GC_PURE_F, arraydescr,
+                                      arraybox, indexbox)
+            return executor.wrap_constant(resval)
+        return self._do_getarrayitem_gc_any(rop.GETARRAYITEM_GC_PURE_F,
+                                            arraybox, indexbox, arraydescr)
+
+    @arguments("box", "box", "descr")
+    def opimpl_getarrayitem_gc_r_pure(self, arraybox, indexbox, arraydescr):
+        if isinstance(arraybox, ConstPtr) and isinstance(indexbox, ConstInt):
+            # if the arguments are directly constants, bypass the heapcache
+            # completely
+            val = executor.execute(self.metainterp.cpu, self.metainterp,
+                                      rop.GETARRAYITEM_GC_PURE_R, arraydescr,
+                                      arraybox, indexbox)
+            return executor.wrap_constant(val)
+        return self._do_getarrayitem_gc_any(rop.GETARRAYITEM_GC_PURE_R,
+                                            arraybox, indexbox, arraydescr)
 
     @arguments("box", "box", "box", "descr")
     def _opimpl_setarrayitem_gc_any(self, arraybox, indexbox, itembox,
@@ -1355,7 +1382,7 @@
         if isinstance(box, Const):
             return box     # no promotion needed, already a Const
         else:
-            promoted_box = box.constbox()
+            promoted_box = executor.constant_from_op(box)
             self.metainterp.generate_guard(rop.GUARD_VALUE, box, [promoted_box],
                                            resumepc=orgpc)
             self.metainterp.replace_box(box, promoted_box)
@@ -1375,20 +1402,20 @@
     @specialize.arg(1)
     def execute_varargs(self, opnum, argboxes, descr, exc, pure):
         self.metainterp.clear_exception()
-        resbox = self.metainterp.execute_and_record_varargs(opnum, argboxes,
+        op = self.metainterp.execute_and_record_varargs(opnum, argboxes,
                                                             descr=descr)
-        if pure and self.metainterp.last_exc_value_box is None and resbox:
-            resbox = self.metainterp.record_result_of_call_pure(resbox)
-            exc = exc and not isinstance(resbox, Const)
+        if pure and self.metainterp.last_exc_value_box is None and op:
+            op = self.metainterp.record_result_of_call_pure(op)
+            exc = exc and not isinstance(op, Const)
         if exc:
-            if resbox is not None:
-                self.make_result_of_lastop(resbox)
+            if op is not None:
+                self.make_result_of_lastop(op)
                 # ^^^ this is done before handle_possible_exception() because we
                 # need the box to show up in get_list_of_active_boxes()
             self.metainterp.handle_possible_exception()
         else:
             self.metainterp.assert_no_exception()
-        return resbox
+        return op
 
     def _build_allboxes(self, funcbox, argboxes, descr):
         allboxes = [None] * (len(argboxes)+1)
@@ -2158,10 +2185,9 @@
         for i in range(endindex):
             box = boxes[i]
             if isinstance(box, Const) or box in duplicates:
-                oldbox = box
-                box = oldbox.clonebox()
-                boxes[i] = box
-                self.history.record(rop.SAME_AS, [oldbox], box)
+                opnum = OpHelpers.same_as_for_type(box.type)
+                op = self.history.record_default_val(opnum, [box])
+                boxes[i] = op
             else:
                 duplicates[box] = None
 
@@ -2708,7 +2734,6 @@
                                 None, descr=vinfo.vable_token_descr)
 
     def replace_box(self, oldbox, newbox):
-        assert isinstance(oldbox, Box)
         for frame in self.framestack:
             frame.replace_active_box_in_frame(oldbox, newbox)
         boxes = self.virtualref_boxes
@@ -2745,12 +2770,12 @@
                 max_key = key
         return max_key
 
-    def record_result_of_call_pure(self, resbox):
+    def record_result_of_call_pure(self, op):
         """ Patch a CALL into a CALL_PURE.
         """
-        op = self.history.operations[-1]
-        assert op.getopnum() == rop.CALL
-        resbox_as_const = resbox.constbox()
+        opnum = op.getopnum()
+        assert opnum in [rop.CALL_R, rop.CALL_N, rop.CALL_I, rop.CALL_F]
+        resbox_as_const = executor.constant_from_op(op)
         for i in range(op.numargs()):
             if not isinstance(op.getarg(i), Const):
                 break
@@ -2761,11 +2786,12 @@
             return resbox_as_const
         # not all constants (so far): turn CALL into CALL_PURE, which might
         # be either removed later by optimizeopt or turned back into CALL.
-        arg_consts = [a.constbox() for a in op.getarglist()]
+        arg_consts = [executor.constant_from_op(a) for a in op.getarglist()]
         self.call_pure_results[arg_consts] = resbox_as_const
-        newop = op.copy_and_change(rop.CALL_PURE, args=op.getarglist())
+        opnum = OpHelpers.call_pure_for_descr(op.getdescr())
+        newop = op._copy_and_change(opnum, args=op.getarglist())
         self.history.operations[-1] = newop
-        return resbox
+        return newop
 
     def direct_assembler_call(self, targetjitdriver_sd):
         """ Generate a direct call to assembler for portal entry point,
diff --git a/rpython/jit/metainterp/resoperation.py b/rpython/jit/metainterp/resoperation.py
--- a/rpython/jit/metainterp/resoperation.py
+++ b/rpython/jit/metainterp/resoperation.py
@@ -1,6 +1,7 @@
 from rpython.rlib.objectmodel import we_are_translated, specialize
 from rpython.rlib.objectmodel import compute_identity_hash
 from rpython.rtyper.lltypesystem import lltype, llmemory
+from rpython.jit.codewriter import longlong
 
 class AbstractValue(object):
     def _get_hash_(self):
@@ -303,6 +304,9 @@
     def copy_value_from(self, other):
         self.setint(other.getint())
 
+    def nonnull(self):
+        return self._resint != 0
+
 class FloatOp(object):
     _mixin_ = True
 
@@ -319,6 +323,9 @@
     def copy_value_from(self, other):
         self.setfloatstorage(other.getfloatstorage())
 
+    def nonnull(self):
+        return bool(longlong.extract_bits(self._resfloat))
+
 class RefOp(object):
     _mixin_ = True
 
@@ -335,6 +342,9 @@
     def copy_value_from(self, other):
         self.setref_base(other.getref_base())
 
+    def nonnull(self):
+        return bool(self._resref)
+
 class AbstractInputArg(AbstractValue):    
     def repr(self, memo):
         try:
@@ -880,3 +890,41 @@
             return rop.CALL_F
         assert tp == 'v'
         return rop.CALL_N
+
+    @staticmethod
+    def call_pure_for_descr(descr):
+        tp = descr.get_result_type()
+        if tp == 'i':
+            return rop.CALL_PURE_I
+        elif tp == 'r':
+            return rop.CALL_PURE_R
+        elif tp == 'f':
+            return rop.CALL_PURE_F
+        assert tp == 'v'
+        return rop.CALL_PURE_N
+
+    @staticmethod
+    def getfield_pure_for_descr(descr):
+        if descr.is_pointer_field():
+            return rop.GETFIELD_GC_PURE_R
+        elif descr.is_float_field():
+            return rop.GETFIELD_GC_PURE_F
+        return rop.GETFIELD_GC_PURE_I
+
+    @staticmethod
+    def getarrayitem_pure_for_descr(descr):
+        if descr.is_array_of_pointers():
+            return rop.GETARRAYITEM_GC_PURE_R
+        elif descr.is_array_of_floats():
+            return rop.GETARRAYITEM_GC_PURE_F
+        return rop.GETARRAYITEM_GC_PURE_I
+
+    @staticmethod
+    def same_as_for_type(tp):
+        if tp == 'i':
+            return rop.SAME_AS_I
+        elif tp == 'r':
+            return rop.SAME_AS_R
+        else:
+            assert tp == 'f'
+            return rop.SAME_AS_F
diff --git a/rpython/jit/metainterp/test/test_loop.py b/rpython/jit/metainterp/test/test_loop.py
--- a/rpython/jit/metainterp/test/test_loop.py
+++ b/rpython/jit/metainterp/test/test_loop.py
@@ -191,8 +191,8 @@
                 if op.getopname() == 'guard_true':
                     liveboxes = op.getfailargs()
                     assert len(liveboxes) == 2     # x, y (in some order)
-                    assert isinstance(liveboxes[0], history.BoxInt)
-                    assert isinstance(liveboxes[1], history.BoxInt)
+                    assert liveboxes[0].type == 'i'
+                    assert liveboxes[1].type == 'i'
                     found += 1
             if 'unroll' in self.enable_opts:
                 assert found == 2
diff --git a/rpython/jit/metainterp/typesystem.py b/rpython/jit/metainterp/typesystem.py
--- a/rpython/jit/metainterp/typesystem.py
+++ b/rpython/jit/metainterp/typesystem.py
@@ -66,7 +66,8 @@
         return llmemory.cast_ptr_to_adr(fnptr)
 
     def cls_of_box(self, box):
-        obj = box.getref(lltype.Ptr(rclass.OBJECT))
+        PTR = lltype.Ptr(rclass.OBJECT)
+        obj = lltype.cast_opaque_ptr(PTR, box.getref_base())
         cls = llmemory.cast_ptr_to_adr(obj.typeptr)
         return history.ConstInt(heaptracker.adr2int(cls))
 

From noreply at buildbot.pypy.org  Sun Nov 23 09:09:00 2014
From: noreply at buildbot.pypy.org (fijal)
Date: Sun, 23 Nov 2014 09:09:00 +0100 (CET)
Subject: [pypy-commit] pypy optresult: fix fix fix
Message-ID: <20141123080900.B68971C31B8@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: optresult
Changeset: r74645:fc8a57d8fa92
Date: 2014-11-23 10:08 +0200
http://bitbucket.org/pypy/pypy/changeset/fc8a57d8fa92/

Log:	fix fix fix

diff --git a/rpython/jit/metainterp/optimizeopt/heap.py b/rpython/jit/metainterp/optimizeopt/heap.py
--- a/rpython/jit/metainterp/optimizeopt/heap.py
+++ b/rpython/jit/metainterp/optimizeopt/heap.py
@@ -7,7 +7,7 @@
 from rpython.jit.metainterp.optimizeopt.optimizer import Optimization, MODE_ARRAY, LEVEL_KNOWNCLASS, REMOVED
 from rpython.jit.metainterp.optimizeopt.util import make_dispatcher_method
 from rpython.jit.metainterp.optimize import InvalidLoop
-from rpython.jit.metainterp.resoperation import rop, ResOperation
+from rpython.jit.metainterp.resoperation import rop, ResOperation, OpHelpers
 from rpython.rlib.objectmodel import we_are_translated
 
 
@@ -462,7 +462,7 @@
     optimize_GETFIELD_GC_PURE_F = optimize_GETFIELD_GC_PURE_I
 
     def optimize_SETFIELD_GC(self, op):
-        opnum = self.optimizer.getfield_pure_for_descr(op.getdescr())
+        opnum = OpHelpers.getfield_pure_for_descr(op.getdescr())
         if self.has_pure_result(opnum, [op.getarg(0)],
                                 op.getdescr()):
             os.write(2, '[bogus _immutable_field_ declaration: %s]\n' %
@@ -520,7 +520,7 @@
     optimize_GETARRAYITEM_GC_PURE_F = optimize_GETARRAYITEM_GC_PURE_I
 
     def optimize_SETARRAYITEM_GC(self, op):
-        opnum = self.optimizer.getarrayitem_pure_for_descr(op.getdescr())
+        opnum = OpHelpers.getarrayitem_pure_for_descr(op.getdescr())
         if self.has_pure_result(opnum, [op.getarg(0), op.getarg(1)],
                                 op.getdescr()):
             os.write(2, '[bogus immutable array declaration: %s]\n' %
diff --git a/rpython/jit/metainterp/optimizeopt/pure.py b/rpython/jit/metainterp/optimizeopt/pure.py
--- a/rpython/jit/metainterp/optimizeopt/pure.py
+++ b/rpython/jit/metainterp/optimizeopt/pure.py
@@ -1,5 +1,5 @@
 from rpython.jit.metainterp.optimizeopt.optimizer import Optimization, REMOVED
-from rpython.jit.metainterp.resoperation import rop, ResOperation
+from rpython.jit.metainterp.resoperation import rop, ResOperation, OpHelpers
 from rpython.jit.metainterp.optimizeopt.util import (make_dispatcher_method,
     args_dict)
 
@@ -82,7 +82,7 @@
 
         # replace CALL_PURE with just CALL
         args = op.getarglist()
-        opnum = self.optimizer.call_for_descr(op.getdescr())
+        opnum = OpHelpers.call_for_descr(op.getdescr())
         newop = ResOperation(opnum, args, op.getdescr())
         self.getvalue(op).box = newop
         self.emit_operation(newop)
diff --git a/rpython/jit/metainterp/optimizeopt/rewrite.py b/rpython/jit/metainterp/optimizeopt/rewrite.py
--- a/rpython/jit/metainterp/optimizeopt/rewrite.py
+++ b/rpython/jit/metainterp/optimizeopt/rewrite.py
@@ -8,7 +8,8 @@
 from rpython.jit.metainterp.optimizeopt.optimizer import (Optimization, REMOVED,
     CONST_0, CONST_1)
 from rpython.jit.metainterp.optimizeopt.util import _findall, make_dispatcher_method
-from rpython.jit.metainterp.resoperation import rop, ResOperation, opclasses
+from rpython.jit.metainterp.resoperation import rop, ResOperation, opclasses,\
+     OpHelpers
 from rpython.rlib.rarithmetic import highest_bit
 import math
 
@@ -404,7 +405,7 @@
         # change the op to be a normal call, from the backend's point of view
         # there is no reason to have a separate operation for this
         self.loop_invariant_producer[key] = op
-        opnum = self.optimizer.call_for_descr(op.getdescr())
+        opnum = OpHelpers.call_for_descr(op.getdescr())
         newop = self.optimizer.replace_op_with(op, opnum)
         self.emit_operation(newop)
         resvalue = self.getvalue(op)
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py
@@ -6,16 +6,16 @@
 import rpython.jit.metainterp.optimizeopt.optimizer as optimizeopt
 import rpython.jit.metainterp.optimizeopt.virtualize as virtualize
 from rpython.jit.metainterp.optimize import InvalidLoop
-from rpython.jit.metainterp.history import ConstInt, BoxInt, get_const_ptr_for_string
+from rpython.jit.metainterp.history import ConstInt, get_const_ptr_for_string
 from rpython.jit.metainterp import executor, compile, resume
-from rpython.jit.metainterp.resoperation import rop, ResOperation
+from rpython.jit.metainterp.resoperation import rop, ResOperation, InputArgInt
 from rpython.rlib.rarithmetic import LONG_BIT
 
 def test_store_final_boxes_in_guard():
     from rpython.jit.metainterp.compile import ResumeGuardDescr
     from rpython.jit.metainterp.resume import tag, TAGBOX
-    b0 = BoxInt()
-    b1 = BoxInt()
+    b0 = InputArgInt()
+    b1 = InputArgInt()
     opt = optimizeopt.Optimizer(FakeMetaInterpStaticData(LLtypeMixin.cpu),
                                 None)
     fdescr = ResumeGuardDescr()
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_util.py b/rpython/jit/metainterp/optimizeopt/test/test_util.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_util.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_util.py
@@ -6,20 +6,18 @@
     OBJECT, OBJECT_VTABLE, FieldListAccessor, IR_QUASIIMMUTABLE)
 
 from rpython.jit.backend.llgraph import runner
-from rpython.jit.metainterp.history import (BoxInt, BoxPtr, ConstInt, ConstPtr,
-                                         Const, TreeLoop, AbstractDescr,
-                                         JitCellToken, TargetToken)
+from rpython.jit.metainterp.history import (TreeLoop, AbstractDescr,
+                                            JitCellToken, TargetToken)
 from rpython.jit.metainterp.optimizeopt.util import sort_descrs, equaloplists
-from rpython.jit.metainterp.optimize import InvalidLoop
 from rpython.jit.codewriter.effectinfo import EffectInfo
-from rpython.jit.codewriter.heaptracker import register_known_gctype, adr2int
+from rpython.jit.codewriter.heaptracker import register_known_gctype
 from rpython.jit.tool.oparser import parse, pure_parse
 from rpython.jit.metainterp.quasiimmut import QuasiImmutDescr
 from rpython.jit.metainterp import compile, resume, history
 from rpython.jit.metainterp.jitprof import EmptyProfiler
 from rpython.jit.metainterp.counter import DeterministicJitCounter
 from rpython.config.translationoption import get_combined_translation_config
-from rpython.jit.metainterp.resoperation import rop, opname, ResOperation
+from rpython.jit.metainterp.resoperation import rop, ResOperation
 from rpython.jit.metainterp.optimizeopt.unroll import Inliner
 
 def test_sort_descrs():
@@ -101,11 +99,10 @@
     node.parent.typeptr = node_vtable
     node2 = lltype.malloc(NODE2)
     node2.parent.parent.typeptr = node_vtable2
-    nodebox = BoxPtr(lltype.cast_opaque_ptr(llmemory.GCREF, node))
-    myptr = nodebox.value
+    myptr = lltype.cast_opaque_ptr(llmemory.GCREF, node)
     myptr2 = lltype.cast_opaque_ptr(llmemory.GCREF, lltype.malloc(NODE))
     nullptr = lltype.nullptr(llmemory.GCREF.TO)
-    nodebox2 = BoxPtr(lltype.cast_opaque_ptr(llmemory.GCREF, node2))
+    #nodebox2 = BoxPtr(lltype.cast_opaque_ptr(llmemory.GCREF, node2))
     nodesize = cpu.sizeof(NODE)
     nodesize2 = cpu.sizeof(NODE2)
     valuedescr = cpu.fielddescrof(NODE, 'value')
@@ -123,10 +120,8 @@
     quasi = lltype.malloc(QUASI, immortal=True)
     quasi.inst_field = -4247
     quasifielddescr = cpu.fielddescrof(QUASI, 'inst_field')
-    quasibox = BoxPtr(lltype.cast_opaque_ptr(llmemory.GCREF, quasi))
-    quasiptr = quasibox.value
-    quasiimmutdescr = QuasiImmutDescr(cpu, quasibox,
-                                      quasifielddescr,
+    quasiptr = lltype.cast_opaque_ptr(llmemory.GCREF, quasi)
+    quasiimmutdescr = QuasiImmutDescr(cpu, quasiptr, quasifielddescr,
                                       cpu.fielddescrof(QUASI, 'mutate_field'))
 
     NODEOBJ = lltype.GcStruct('NODEOBJ', ('parent', OBJECT),
@@ -159,7 +154,7 @@
     ssize = cpu.sizeof(S)
     adescr = cpu.fielddescrof(S, 'a')
     bdescr = cpu.fielddescrof(S, 'b')
-    sbox = BoxPtr(lltype.cast_opaque_ptr(llmemory.GCREF, lltype.malloc(S)))
+    #sbox = BoxPtr(lltype.cast_opaque_ptr(llmemory.GCREF, lltype.malloc(S)))
     arraydescr2 = cpu.arraydescrof(lltype.GcArray(lltype.Ptr(S)))
 
     T = lltype.GcStruct('TUPLE',
diff --git a/rpython/jit/metainterp/pyjitpl.py b/rpython/jit/metainterp/pyjitpl.py
--- a/rpython/jit/metainterp/pyjitpl.py
+++ b/rpython/jit/metainterp/pyjitpl.py
@@ -466,21 +466,24 @@
                                             indexbox, arraydescr)
 
     @arguments("box", "box", "descr")
-    def _opimpl_getarrayitem_raw_any(self, arraybox, indexbox, arraydescr):
-        return self.execute_with_descr(rop.GETARRAYITEM_RAW,
+    def opimpl_getarrayitem_raw_i(self, arraybox, indexbox, arraydescr):
+        return self.execute_with_descr(rop.GETARRAYITEM_RAW_I,
                                        arraydescr, arraybox, indexbox)
 
-    opimpl_getarrayitem_raw_i = _opimpl_getarrayitem_raw_any
-    opimpl_getarrayitem_raw_f = _opimpl_getarrayitem_raw_any
+    @arguments("box", "box", "descr")
+    def opimpl_getarrayitem_raw_f(self, arraybox, indexbox, arraydescr):
+        return self.execute_with_descr(rop.GETARRAYITEM_RAW_F,
+                                       arraydescr, arraybox, indexbox)
 
     @arguments("box", "box", "descr")
-    def _opimpl_getarrayitem_raw_pure_any(self, arraybox, indexbox,
-                                          arraydescr):
-        return self.execute_with_descr(rop.GETARRAYITEM_RAW_PURE,
+    def opimpl_getarrayitem_raw_pure_i(self, arraybox, indexbox, arraydescr):
+        return self.execute_with_descr(rop.GETARRAYITEM_RAW_PURE_I,
                                        arraydescr, arraybox, indexbox)
 
-    opimpl_getarrayitem_raw_i_pure = _opimpl_getarrayitem_raw_pure_any
-    opimpl_getarrayitem_raw_f_pure = _opimpl_getarrayitem_raw_pure_any
+    @arguments("box", "box", "descr")
+    def opimpl_getarrayitem_raw_pure_f(self, arraybox, indexbox, arraydescr):
+        return self.execute_with_descr(rop.GETARRAYITEM_RAW_PURE_F,
+                                       arraydescr, arraybox, indexbox)
 
     @arguments("box", "box", "descr")
     def opimpl_getarrayitem_gc_i_pure(self, arraybox, indexbox, arraydescr):
diff --git a/rpython/jit/metainterp/quasiimmut.py b/rpython/jit/metainterp/quasiimmut.py
--- a/rpython/jit/metainterp/quasiimmut.py
+++ b/rpython/jit/metainterp/quasiimmut.py
@@ -101,32 +101,23 @@
 
 
 class QuasiImmutDescr(AbstractDescr):
-    structbox = None
-
-    def __init__(self, cpu, structbox, fielddescr, mutatefielddescr):
+    def __init__(self, cpu, struct, fielddescr, mutatefielddescr):
         self.cpu = cpu
-        self.structbox = structbox
+        self.struct = struct
         self.fielddescr = fielddescr
         self.mutatefielddescr = mutatefielddescr
-        gcref = structbox.getref_base()
-        self.qmut = get_current_qmut_instance(cpu, gcref, mutatefielddescr)
+        self.qmut = get_current_qmut_instance(cpu, struct, mutatefielddescr)
         self.constantfieldbox = self.get_current_constant_fieldvalue()
 
     def get_current_constant_fieldvalue(self):
-        from rpython.jit.metainterp import executor
-        from rpython.jit.metainterp.resoperation import rop
+        struct = self.struct
+        fielddescr = self.fielddescr
         if self.fielddescr.is_pointer_field():
-            return ConstPtr(executor.do_getfield_gc_r(self.cpu, None, rop.GETFIELD_GC_R,
-                                             self.fielddescr, self.structbox))
+            return ConstPtr(self.cpu.bh_getfield_gc_r(struct, fielddescr))
         elif self.fielddescr.is_float_field():
-            return ConstFloat(executor.execute(self.cpu, None,
-                                               rop.GETFIELD_GC_F,
-                                               self.fielddescr, self.structbox))
-            
+            return ConstFloat(self.cpu.bh_getfield_gc_f(struct, fielddescr))
         else:
-            return ConstInt(executor.do_getfield_gc_i(self.cpu, None,
-                                                      self.structbox,
-                                                      self.fielddescr))
+            return ConstInt(self.cpu.bh_getfield_gc_i(struct, fielddescr))
 
     def is_still_valid_for(self, structconst):
         assert self.structbox is not None
diff --git a/rpython/jit/metainterp/resume.py b/rpython/jit/metainterp/resume.py
--- a/rpython/jit/metainterp/resume.py
+++ b/rpython/jit/metainterp/resume.py
@@ -1142,9 +1142,9 @@
                 box = self.consts[num]
         elif tag == TAGVIRTUAL:
             if kind == INT:
-                box = self.getvirtual_int(num)
+                box = InputArgInt(self.getvirtual_int(num))
             else:
-                box = self.getvirtual_ptr(num)
+                box = InputArgRef(self.getvirtual_ptr(num))
         elif tag == TAGINT:
             box = ConstInt(num)
         else:

From noreply at buildbot.pypy.org  Sun Nov 23 09:35:35 2014
From: noreply at buildbot.pypy.org (fijal)
Date: Sun, 23 Nov 2014 09:35:35 +0100 (CET)
Subject: [pypy-commit] pypy optresult: hack at tests for the first
	test_expand_xxx to pass
Message-ID: <20141123083535.429971C022D@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: optresult
Changeset: r74646:e40642f20836
Date: 2014-11-23 10:35 +0200
http://bitbucket.org/pypy/pypy/changeset/e40642f20836/

Log:	hack at tests for the first test_expand_xxx to pass

diff --git a/rpython/jit/metainterp/history.py b/rpython/jit/metainterp/history.py
--- a/rpython/jit/metainterp/history.py
+++ b/rpython/jit/metainterp/history.py
@@ -240,6 +240,8 @@
     def getint(self):
         return self.value
 
+    getvalue = getint
+
     def getaddr(self):
         return heaptracker.int2adr(self.value)
 
@@ -280,6 +282,8 @@
     def getfloatstorage(self):
         return self.value
 
+    getvalue = getfloatstorage
+
     def _get_hash_(self):
         return longlong.gethash(self.value)
 
@@ -322,6 +326,8 @@
     def getref_base(self):
         return self.value
 
+    getvalue = getref_base
+
     def getref(self, PTR):
         return lltype.cast_opaque_ptr(PTR, self.getref_base())
     getref._annspecialcase_ = 'specialize:arg(1)'
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py
@@ -8,7 +8,8 @@
 from rpython.jit.metainterp.optimize import InvalidLoop
 from rpython.jit.metainterp.history import ConstInt, get_const_ptr_for_string
 from rpython.jit.metainterp import executor, compile, resume
-from rpython.jit.metainterp.resoperation import rop, ResOperation, InputArgInt
+from rpython.jit.metainterp.resoperation import rop, ResOperation, InputArgInt,\
+     OpHelpers
 from rpython.rlib.rarithmetic import LONG_BIT
 
 def test_store_final_boxes_in_guard():
@@ -2420,12 +2421,12 @@
             pfieldvar = match.group(3)
             pendingfields.append((pvar, pfieldname, pfieldvar))
         #
-        def _variables_equal(box, varname, strict):
+        def _variables_equal(value, varname, strict):
             if varname not in virtuals:
                 if strict:
                     assert box.same_box(oparse.getvar(varname))
                 else:
-                    assert box.value == oparse.getvar(varname).value
+                    assert value == oparse.getvar(varname).getvalue()
             else:
                 tag, resolved, fieldstext = virtuals[varname]
                 if tag[0] == 'virtual':
@@ -2469,8 +2470,8 @@
                 if tag[0] in ('virtual', 'vstruct'):
                     fieldname, fieldvalue = fieldtext.split('=')
                     fielddescr = self.namespace[fieldname.strip()]
-                    fieldbox = executor.execute(self.cpu, None,
-                                                rop.GETFIELD_GC,
+                    opnum = OpHelpers.getfield_for_descr(fielddescr)
+                    fieldval = executor.execute(self.cpu, None, opnum,
                                                 fielddescr,
                                                 resolved)
                 elif tag[0] == 'varray':
@@ -2481,7 +2482,7 @@
                                                 resolved, ConstInt(index))
                 else:
                     assert 0
-                _variables_equal(fieldbox, fieldvalue.strip(), strict=False)
+                _variables_equal(fieldval, fieldvalue.strip(), strict=False)
                 index += 1
 
     def check_expanded_fail_descr(self, expectedtext, guard_opnum):
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_util.py b/rpython/jit/metainterp/optimizeopt/test/test_util.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_util.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_util.py
@@ -77,7 +77,8 @@
     type_system = 'lltype'
 
     def get_class_of_box(self, box):
-        return box.getref(rclass.OBJECTPTR).typeptr
+        base = box.getref_base()
+        return lltype.cast_opaque_ptr(rclass.OBJECTPTR, base).typeptr
 
     node_vtable = lltype.malloc(OBJECT_VTABLE, immortal=True)
     node_vtable.name = rclass.alloc_array_name('node')
diff --git a/rpython/jit/metainterp/resoperation.py b/rpython/jit/metainterp/resoperation.py
--- a/rpython/jit/metainterp/resoperation.py
+++ b/rpython/jit/metainterp/resoperation.py
@@ -298,6 +298,8 @@
     def getint(self):
         return self._resint
 
+    getvalue = getint
+
     def setint(self, intval):
         self._resint = intval
 
@@ -317,6 +319,8 @@
     def getfloatstorage(self):
         return self._resfloat
 
+    getvalue = getfloatstorage
+
     def setfloatstorage(self, floatval):
         self._resfloat = floatval
 
@@ -336,6 +340,8 @@
     def getref_base(self):
         return self._resref
 
+    getvalue = getref_base
+
     def setref_base(self, refval):
         self._resref = refval
 
@@ -912,6 +918,14 @@
         return rop.GETFIELD_GC_PURE_I
 
     @staticmethod
+    def getfield_for_descr(descr):
+        if descr.is_pointer_field():
+            return rop.GETFIELD_GC_R
+        elif descr.is_float_field():
+            return rop.GETFIELD_GC_F
+        return rop.GETFIELD_GC_I
+
+    @staticmethod
     def getarrayitem_pure_for_descr(descr):
         if descr.is_array_of_pointers():
             return rop.GETARRAYITEM_GC_PURE_R
diff --git a/rpython/jit/metainterp/resume.py b/rpython/jit/metainterp/resume.py
--- a/rpython/jit/metainterp/resume.py
+++ b/rpython/jit/metainterp/resume.py
@@ -1142,9 +1142,9 @@
                 box = self.consts[num]
         elif tag == TAGVIRTUAL:
             if kind == INT:
-                box = InputArgInt(self.getvirtual_int(num))
+                box = self.getvirtual_int(num)
             else:
-                box = InputArgRef(self.getvirtual_ptr(num))
+                box = self.getvirtual_ptr(num)
         elif tag == TAGINT:
             box = ConstInt(num)
         else:
diff --git a/rpython/jit/metainterp/test/test_resume.py b/rpython/jit/metainterp/test/test_resume.py
--- a/rpython/jit/metainterp/test/test_resume.py
+++ b/rpython/jit/metainterp/test/test_resume.py
@@ -11,6 +11,7 @@
 from rpython.jit.metainterp.optimizeopt.test.test_util import LLtypeMixin
 from rpython.jit.metainterp import executor
 from rpython.jit.codewriter import heaptracker, longlong
+from rpython.jit.metainterp.resoperation import ResOperation
 
 class Storage:
     rd_frame_info_list = None
@@ -144,7 +145,6 @@
         self.cpu = cpu
         self.trace = []
         self.framestack = []
-        self.resboxes = []
 
     def newframe(self, jitcode):
         frame = FakeFrame(jitcode, -1)
@@ -152,14 +152,11 @@
         return frame    
 
     def execute_and_record(self, opnum, descr, *argboxes):
-        resbox = executor.execute(self.cpu, None, opnum, descr, *argboxes)
-        self.trace.append((opnum,
-                           list(argboxes),
-                           resbox,
-                           descr))
-        if resbox is not None:
-            self.resboxes.append(resbox)
-        return resbox
+        resvalue = executor.execute(self.cpu, None, opnum, descr, *argboxes)
+        op = ResOperation(opnum, list(argboxes), descr)
+        op.setvalue(resvalue)
+        self.trace.append(op)
+        return op
 
     def execute_new_with_vtable(self, known_class):
         return self.execute_and_record(rop.NEW_WITH_VTABLE, None,

From noreply at buildbot.pypy.org  Sun Nov 23 10:27:04 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Sun, 23 Nov 2014 10:27:04 +0100 (CET)
Subject: [pypy-commit] pypy default: remove outdated comment
Message-ID: <20141123092704.A3A341C02EE@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r74647:6825260d6f69
Date: 2014-11-23 10:26 +0100
http://bitbucket.org/pypy/pypy/changeset/6825260d6f69/

Log:	remove outdated comment

diff --git a/pypy/interpreter/pyframe.py b/pypy/interpreter/pyframe.py
--- a/pypy/interpreter/pyframe.py
+++ b/pypy/interpreter/pyframe.py
@@ -131,7 +131,6 @@
         # class bodies only have CO_NEWLOCALS.
         # CO_NEWLOCALS: make a locals dict unless optimized is also set
         # CO_OPTIMIZED: no locals dict needed at all
-        # NB: this method is overridden in nestedscope.py
         flags = code.co_flags
         if not (flags & pycode.CO_OPTIMIZED):
             if flags & pycode.CO_NEWLOCALS:

From noreply at buildbot.pypy.org  Sun Nov 23 10:55:33 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Sun, 23 Nov 2014 10:55:33 +0100 (CET)
Subject: [pypy-commit] pypy default: fix comments
Message-ID: <20141123095533.48DC61C31B8@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r74648:472f5815b5e2
Date: 2014-11-23 10:55 +0100
http://bitbucket.org/pypy/pypy/changeset/472f5815b5e2/

Log:	fix comments

diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py
--- a/rpython/jit/backend/x86/assembler.py
+++ b/rpython/jit/backend/x86/assembler.py
@@ -1832,15 +1832,17 @@
             offset = self.cpu.get_ofs_of_frame_field('jf_guard_exc')
             mc.MOV_br(offset, ebx.value)
 
-        # now we return from the complete frame, which starts from
-        # _call_header_with_stack_check().  The LEA in _call_footer below
-        # throws away most of the frame, including all the PUSHes that we
-        # did just above.
+        # fill in the jf_descr and jf_gcmap fields of the frame according
+        # to which failure we are resuming from.  These are constants
+        # pushed on the stack just before we jump to the current helper,
+        # in generate_quick_failure().
         ofs = self.cpu.get_ofs_of_frame_field('jf_descr')
         ofs2 = self.cpu.get_ofs_of_frame_field('jf_gcmap')
         mc.POP_b(ofs2)
         mc.POP_b(ofs)
 
+        # now we return from the complete frame, which starts from
+        # _call_header_with_stack_check().  The _call_footer below does it.
         self._call_footer()
         rawstart = mc.materialize(self.cpu.asmmemmgr, [])
         self.failure_recovery_code[exc + 2 * withfloats] = rawstart

From noreply at buildbot.pypy.org  Sun Nov 23 12:07:59 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Sun, 23 Nov 2014 12:07:59 +0100 (CET)
Subject: [pypy-commit] pypy portable-threadlocal: Test and fix
Message-ID: <20141123110759.6E7581C3288@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: portable-threadlocal
Changeset: r74649:9c067d3de94c
Date: 2014-11-23 11:59 +0100
http://bitbucket.org/pypy/pypy/changeset/9c067d3de94c/

Log:	Test and fix

diff --git a/rpython/translator/c/genc.py b/rpython/translator/c/genc.py
--- a/rpython/translator/c/genc.py
+++ b/rpython/translator/c/genc.py
@@ -749,6 +749,11 @@
     # generate the start-up code and put it into a function
     print >> f, 'char *RPython_StartupCode(void) {'
     print >> f, '\tchar *error = NULL;'
+
+    bk = database.translator.annotator.bookkeeper
+    if bk.thread_local_fields:
+        print >> f, '\tRPython_ThreadLocals_ProgramInit();'
+
     for line in database.gcpolicy.gc_startup_code():
         print >> f,"\t" + line
 
@@ -768,10 +773,6 @@
             for line in lines:
                 print >> f, '\t'+line
 
-    bk = database.translator.annotator.bookkeeper
-    if bk.thread_local_fields:
-        print >> f, '\tRPython_ThreadLocals_ProgramInit();'
-
     print >> f, '\treturn error;'
     print >> f, '}'
 
diff --git a/rpython/translator/c/src/support.h b/rpython/translator/c/src/support.h
--- a/rpython/translator/c/src/support.h
+++ b/rpython/translator/c/src/support.h
@@ -2,6 +2,9 @@
 /************************************************************/
  /***  C header subsection: support functions              ***/
 
+#ifndef _SRC_SUPPORT_H
+#define _SRC_SUPPORT_H
+
 #define RUNNING_ON_LLINTERP	0
 #define OP_JIT_RECORD_KNOWN_CLASS(i, c, r)  /* nothing */
 
@@ -65,3 +68,5 @@
 #  define RPyNLenItem(array, index)          ((array)->items[index])
 #  define RPyBareItem(array, index)          ((array)[index])
 #endif
+
+#endif  /* _SRC_SUPPORT_H */
diff --git a/rpython/translator/c/src/threadlocal.c b/rpython/translator/c/src/threadlocal.c
--- a/rpython/translator/c/src/threadlocal.c
+++ b/rpython/translator/c/src/threadlocal.c
@@ -27,7 +27,7 @@
                   where it is not the case are rather old nowadays. */
 #    endif
 #endif
-    ((struct pypy_threadlocal_s *)p)->ready = 1;
+    ((struct pypy_threadlocal_s *)p)->ready = 42;
 }
 
 
@@ -47,6 +47,7 @@
 
 char *_RPython_ThreadLocals_Build(void)
 {
+    RPyAssert(pypy_threadlocal.ready == 0, "corrupted thread-local");
     _RPy_ThreadLocals_Init(&pypy_threadlocal);
     return (char *)&pypy_threadlocal;
 }
diff --git a/rpython/translator/c/src/threadlocal.h b/rpython/translator/c/src/threadlocal.h
--- a/rpython/translator/c/src/threadlocal.h
+++ b/rpython/translator/c/src/threadlocal.h
@@ -2,7 +2,8 @@
 #ifndef _SRC_THREADLOCAL_H
 #define _SRC_THREADLOCAL_H
 
-#include 
+#include "src/precommondefs.h"
+#include "src/support.h"
 
 
 /* RPython_ThreadLocals_ProgramInit() is called once at program start-up. */
@@ -29,10 +30,20 @@
 /* Use the '__thread' specifier, so far only on Linux */
 
 RPY_EXTERN __thread struct pypy_threadlocal_s pypy_threadlocal;
-#define OP_THREADLOCALREF_ADDR(r)    r = (char *)&pypy_threadlocal
-#define OP_THREADLOCALREF_MAKE(r)                               \
-    (OP_THREADLOCALREF_ADDR(r),                                 \
-     (pypy_threadlocal.ready || (r = _RPython_ThreadLocals_Build())))
+
+#define OP_THREADLOCALREF_ADDR(r)                       \
+    do {                                                \
+        RPyAssert(pypy_threadlocal.ready == 42,         \
+                  "uninitialized thread-local!");       \
+        r = (char *)&pypy_threadlocal;                  \
+    } while (0)
+
+#define OP_THREADLOCALREF_MAKE(r)               \
+    do {                                        \
+        r = (char *)&pypy_threadlocal;          \
+        if (pypy_threadlocal.ready != 42)       \
+            r = _RPython_ThreadLocals_Build();  \
+    } while (0)
 
 
 /* ------------------------------------------------------------ */

From noreply at buildbot.pypy.org  Sun Nov 23 14:18:24 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Sun, 23 Nov 2014 14:18:24 +0100 (CET)
Subject: [pypy-commit] pypy portable-threadlocal: Clean up and adpat the JIT
 to the new threadlocals. Still missing:
Message-ID: <20141123131824.3D6B11D35DE@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: portable-threadlocal
Changeset: r74650:5291d2692c23
Date: 2014-11-23 14:18 +0100
http://bitbucket.org/pypy/pypy/changeset/5291d2692c23/

Log:	Clean up and adpat the JIT to the new threadlocals. Still missing:
	optimization for OS_THREADLOCALREF_ADDR in the x86 and arm backends

diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py
--- a/rpython/jit/backend/x86/assembler.py
+++ b/rpython/jit/backend/x86/assembler.py
@@ -2320,24 +2320,9 @@
         assert isinstance(reg, RegLoc)
         self.mc.MOV_rr(reg.value, ebp.value)
 
-    def threadlocalref_get(self, op, resloc):
-        # this function is only called on Linux
-        from rpython.jit.codewriter.jitcode import ThreadLocalRefDescr
-        from rpython.jit.backend.x86 import stmtlocal
+    def threadlocalref_addr(self, resloc):
         assert isinstance(resloc, RegLoc)
-        effectinfo = op.getdescr().get_extra_info()
-        assert effectinfo.extradescrs is not None
-        ed = effectinfo.extradescrs[0]
-        assert isinstance(ed, ThreadLocalRefDescr)
-        addr1 = rffi.cast(lltype.Signed, ed.get_tlref_addr())
-        # 'addr1' is the address is the current thread, but we assume that
-        # it is a thread-local at a constant offset from %fs/%gs.
-        addr0 = stmtlocal.threadlocal_base()
-        addr = addr1 - addr0
-        assert rx86.fits_in_32bits(addr)
-        mc = self.mc
-        mc.writechar(stmtlocal.SEGMENT_TL)     # prefix: %fs or %gs
-        mc.MOV_rj(resloc.value, addr)          # memory read
+        XXX
 
     def get_set_errno(self, op, loc, issue_a_write):
         # this function is only called on Linux
diff --git a/rpython/jit/backend/x86/regalloc.py b/rpython/jit/backend/x86/regalloc.py
--- a/rpython/jit/backend/x86/regalloc.py
+++ b/rpython/jit/backend/x86/regalloc.py
@@ -693,31 +693,9 @@
         loc0 = self.xrm.force_result_in_reg(op.result, op.getarg(1))
         self.perform_math(op, [loc0], loc0)
 
-    TLREF_SUPPORT = sys.platform.startswith('linux')
-    ERRNO_SUPPORT = sys.platform.startswith('linux')
-
-    def _consider_threadlocalref_get(self, op):
-        if self.TLREF_SUPPORT:
-            resloc = self.force_allocate_reg(op.result)
-            self.assembler.threadlocalref_get(op, resloc)
-        else:
-            self._consider_call(op)
-
-    def _consider_get_errno(self, op):
-        if self.ERRNO_SUPPORT:
-            resloc = self.force_allocate_reg(op.result)
-            self.assembler.get_set_errno(op, resloc, issue_a_write=False)
-        else:
-            self._consider_call(op)
-
-    def _consider_set_errno(self, op):
-        if self.ERRNO_SUPPORT:
-            # op.getarg(0) is the function set_errno; op.getarg(1) is
-            # the new errno value
-            loc0 = self.rm.make_sure_var_in_reg(op.getarg(1))
-            self.assembler.get_set_errno(op, loc0, issue_a_write=True)
-        else:
-            self._consider_call(op)
+    def _consider_threadlocalref_addr(self, op):
+        resloc = self.force_allocate_reg(op.result)
+        self.assembler.threadlocalref_addr(resloc)
 
     def _call(self, op, arglocs, force_store=[], guard_not_forced_op=None):
         # we need to save registers on the stack:
@@ -796,12 +774,8 @@
                         return
             if oopspecindex == EffectInfo.OS_MATH_SQRT:
                 return self._consider_math_sqrt(op)
-            if oopspecindex == EffectInfo.OS_THREADLOCALREF_GET:
-                return self._consider_threadlocalref_get(op)
-            if oopspecindex == EffectInfo.OS_GET_ERRNO:
-                return self._consider_get_errno(op)
-            if oopspecindex == EffectInfo.OS_SET_ERRNO:
-                return self._consider_set_errno(op)
+            #if oopspecindex == EffectInfo.OS_THREADLOCALREF_ADDR:
+            #    return self._consider_threadlocalref_addr(op)
             if oopspecindex == EffectInfo.OS_MATH_READ_TIMESTAMP:
                 return self._consider_math_read_timestamp(op)
         self._consider_call(op)
diff --git a/rpython/jit/backend/x86/stmtlocal.py b/rpython/jit/backend/x86/stmtlocal.py
deleted file mode 100644
--- a/rpython/jit/backend/x86/stmtlocal.py
+++ /dev/null
@@ -1,43 +0,0 @@
-from rpython.rtyper.lltypesystem import lltype, rffi
-from rpython.translator.tool.cbuild import ExternalCompilationInfo
-from rpython.jit.backend.x86.arch import WORD
-
-SEGMENT_FS = '\x64'
-SEGMENT_GS = '\x65'
-
-if WORD == 4:
-    SEGMENT_TL = SEGMENT_GS
-    _instruction = "movl %%gs:0, %0"
-else:
-    SEGMENT_TL = SEGMENT_FS
-    _instruction = "movq %%fs:0, %0"
-
-eci = ExternalCompilationInfo(post_include_bits=['''
-#define RPY_STM_JIT  1
-static long pypy__threadlocal_base(void)
-{
-    /* XXX ONLY LINUX WITH GCC/CLANG FOR NOW XXX */
-    long result;
-    asm("%s" : "=r"(result));
-    return result;
-}
-static long pypy__get_errno_tl(void)
-{
-    return ((long)&errno) - pypy__threadlocal_base();
-}
-''' % _instruction])
-
-
-threadlocal_base = rffi.llexternal(
-    'pypy__threadlocal_base',
-    [], lltype.Signed,
-    compilation_info=eci,
-    _nowrapper=True,
-    ) #transactionsafe=True)
-
-get_errno_tl = rffi.llexternal(
-    'pypy__get_errno_tl',
-    [], lltype.Signed,
-    compilation_info=eci,
-    _nowrapper=True,
-    ) #transactionsafe=True)
diff --git a/rpython/jit/codewriter/effectinfo.py b/rpython/jit/codewriter/effectinfo.py
--- a/rpython/jit/codewriter/effectinfo.py
+++ b/rpython/jit/codewriter/effectinfo.py
@@ -22,9 +22,7 @@
     OS_STR2UNICODE              = 2    # "str.str2unicode"
     OS_SHRINK_ARRAY             = 3    # rgc.ll_shrink_array
     OS_DICT_LOOKUP              = 4    # ll_dict_lookup
-    OS_THREADLOCALREF_GET       = 5    # llop.threadlocalref_get
-    OS_GET_ERRNO                = 6    # rposix.get_errno
-    OS_SET_ERRNO                = 7    # rposix.set_errno
+    OS_THREADLOCALREF_ADDR      = 5    # llop.threadlocalref_addr
     OS_NOT_IN_TRACE             = 8    # for calls not recorded in the jit trace
     #
     OS_STR_CONCAT               = 22   # "stroruni.concat"
diff --git a/rpython/jit/codewriter/jitcode.py b/rpython/jit/codewriter/jitcode.py
--- a/rpython/jit/codewriter/jitcode.py
+++ b/rpython/jit/codewriter/jitcode.py
@@ -117,26 +117,6 @@
         raise NotImplementedError
 
 
-class ThreadLocalRefDescr(AbstractDescr):
-    # A special descr used as the extradescr in a call to a
-    # threadlocalref_get function.  If the backend supports it,
-    # it can use this 'get_tlref_addr()' to get the address *in the
-    # current thread* of the thread-local variable.  If, on the current
-    # platform, the "__thread" variables are implemented as an offset
-    # from some base register (e.g. %fs on x86-64), then the backend will
-    # immediately substract the current value of the base register.
-    # This gives an offset from the base register, and this can be
-    # written down in an assembler instruction to load the "__thread"
-    # variable from anywhere.
-
-    def __init__(self, opaque_id):
-        from rpython.rtyper.lltypesystem.lloperation import llop
-        from rpython.rtyper.lltypesystem import llmemory
-        def get_tlref_addr():
-            return llop.threadlocalref_getaddr(llmemory.Address, opaque_id)
-        self.get_tlref_addr = get_tlref_addr
-
-
 class LiveVarsInfo(object):
     def __init__(self, live_i, live_r, live_f):
         self.live_i = live_i
diff --git a/rpython/jit/codewriter/jtransform.py b/rpython/jit/codewriter/jtransform.py
--- a/rpython/jit/codewriter/jtransform.py
+++ b/rpython/jit/codewriter/jtransform.py
@@ -439,8 +439,6 @@
         elif oopspec_name.endswith('dict.lookup'):
             # also ordereddict.lookup
             prepare = self._handle_dict_lookup_call
-        elif oopspec_name.startswith('rposix.'):
-            prepare = self._handle_rposix_call
         else:
             prepare = self.prepare_builtin_call
         try:
@@ -1986,16 +1984,6 @@
         else:
             raise NotImplementedError(oopspec_name)
 
-    def _handle_rposix_call(self, op, oopspec_name, args):
-        if oopspec_name == 'rposix.get_errno':
-            return self._handle_oopspec_call(op, args, EffectInfo.OS_GET_ERRNO,
-                                             EffectInfo.EF_CANNOT_RAISE)
-        elif oopspec_name == 'rposix.set_errno':
-            return self._handle_oopspec_call(op, args, EffectInfo.OS_SET_ERRNO,
-                                             EffectInfo.EF_CANNOT_RAISE)
-        else:
-            raise NotImplementedError(oopspec_name)
-
     def rewrite_op_ll_read_timestamp(self, op):
         op1 = self.prepare_builtin_call(op, "ll_read_timestamp", [])
         return self.handle_residual_call(op1,
@@ -2011,17 +1999,11 @@
                              None)
         return [op0, op1]
 
-    def rewrite_op_threadlocalref_get(self, op):
-        from rpython.jit.codewriter.jitcode import ThreadLocalRefDescr
-        opaqueid = op.args[0].value
-        op1 = self.prepare_builtin_call(op, 'threadlocalref_getter', [],
-                                        extra=(opaqueid,),
-                                        extrakey=opaqueid._obj)
-        extradescr = ThreadLocalRefDescr(opaqueid)
+    def rewrite_op_threadlocalref_addr(self, op):
+        op1 = self.prepare_builtin_call(op, 'threadlocalref_addr', [])
         return self.handle_residual_call(op1,
-            oopspecindex=EffectInfo.OS_THREADLOCALREF_GET,
-            extraeffect=EffectInfo.EF_LOOPINVARIANT,
-            extradescr=[extradescr])
+            oopspecindex=EffectInfo.OS_THREADLOCALREF_ADDR,
+            extraeffect=EffectInfo.EF_CANNOT_RAISE)
 
 # ____________________________________________________________
 
diff --git a/rpython/jit/codewriter/support.py b/rpython/jit/codewriter/support.py
--- a/rpython/jit/codewriter/support.py
+++ b/rpython/jit/codewriter/support.py
@@ -702,10 +702,8 @@
     build_ll_1_raw_free_no_track_allocation = (
         build_raw_free_builder(track_allocation=False))
 
-    def build_ll_0_threadlocalref_getter(opaqueid):
-        def _ll_0_threadlocalref_getter():
-            return llop.threadlocalref_get(rclass.OBJECTPTR, opaqueid)
-        return _ll_0_threadlocalref_getter
+    def _ll_0_threadlocalref_addr():
+        return llop.threadlocalref_addr(llmemory.Address)
 
     def _ll_1_weakref_create(obj):
         return llop.weakref_create(llmemory.WeakRefPtr, obj)
diff --git a/rpython/jit/codewriter/test/test_jtransform.py b/rpython/jit/codewriter/test/test_jtransform.py
--- a/rpython/jit/codewriter/test/test_jtransform.py
+++ b/rpython/jit/codewriter/test/test_jtransform.py
@@ -148,9 +148,7 @@
              EI.OS_UNIEQ_LENGTHOK:       ([PUNICODE, PUNICODE], INT),
              EI.OS_RAW_MALLOC_VARSIZE_CHAR: ([INT], ARRAYPTR),
              EI.OS_RAW_FREE:             ([ARRAYPTR], lltype.Void),
-             EI.OS_THREADLOCALREF_GET:   ([], rclass.OBJECTPTR),
-             EI.OS_GET_ERRNO:            ([], INT),
-             EI.OS_SET_ERRNO:            ([INT], lltype.Void),
+             EI.OS_THREADLOCALREF_ADDR:  ([], llmemory.Address),
             }
             argtypes = argtypes[oopspecindex]
             assert argtypes[0] == [v.concretetype for v in op.args[1:]]
@@ -160,11 +158,8 @@
             elif oopspecindex == EI.OS_RAW_MALLOC_VARSIZE_CHAR:
                 assert extraeffect == EI.EF_CAN_RAISE
             elif oopspecindex in (EI.OS_RAW_FREE,
-                                  EI.OS_GET_ERRNO,
-                                  EI.OS_SET_ERRNO):
+                                  EI.OS_THREADLOCALREF_ADDR):
                 assert extraeffect == EI.EF_CANNOT_RAISE
-            elif oopspecindex == EI.OS_THREADLOCALREF_GET:
-                assert extraeffect == EI.EF_LOOPINVARIANT
             else:
                 assert extraeffect == EI.EF_ELIDABLE_CANNOT_RAISE
         return 'calldescr-%d' % oopspecindex
@@ -1346,54 +1341,17 @@
     assert op1.result is None
     assert op2 is None
 
-def test_threadlocalref_get():
-    from rpython.rtyper import rclass
-    from rpython.rlib.rthread import ThreadLocalReference
-    OS_THREADLOCALREF_GET = effectinfo.EffectInfo.OS_THREADLOCALREF_GET
-    class Foo: pass
-    t = ThreadLocalReference(Foo)
-    v2 = varoftype(rclass.OBJECTPTR)
-    c_opaqueid = const(t.opaque_id)
-    op = SpaceOperation('threadlocalref_get', [c_opaqueid], v2)
+def test_threadlocalref_addr():
+    OS_THREADLOCALREF_ADDR = effectinfo.EffectInfo.OS_THREADLOCALREF_ADDR
+    v = varoftype(llmemory.Address)
+    op = SpaceOperation('threadlocalref_addr', [], v)
     tr = Transformer(FakeCPU(), FakeBuiltinCallControl())
     op0 = tr.rewrite_operation(op)
-    assert op0.opname == 'residual_call_r_r'
-    assert op0.args[0].value == 'threadlocalref_getter' # pseudo-function as str
+    assert op0.opname == 'residual_call_r_i'
+    assert op0.args[0].value == 'threadlocalref_addr' # pseudo-function as str
     assert op0.args[1] == ListOfKind("ref", [])
-    assert op0.args[2] == 'calldescr-%d' % OS_THREADLOCALREF_GET
-    assert op0.result == v2
-
-def test_get_errno():
-    # test that the oopspec is present and correctly transformed
-    from rpython.rlib import rposix
-    FUNC = lltype.FuncType([], lltype.Signed)
-    func = lltype.functionptr(FUNC, 'get_errno', _callable=rposix.get_errno)
-    v3 = varoftype(lltype.Signed)
-    op = SpaceOperation('direct_call', [const(func)], v3)
-    tr = Transformer(FakeCPU(), FakeBuiltinCallControl())
-    op1 = tr.rewrite_operation(op)
-    assert op1.opname == 'residual_call_r_i'
-    assert op1.args[0].value == func
-    assert op1.args[1] == ListOfKind('ref', [])
-    assert op1.args[2] == 'calldescr-%d' % effectinfo.EffectInfo.OS_GET_ERRNO
-    assert op1.result == v3
-
-def test_set_errno():
-    # test that the oopspec is present and correctly transformed
-    from rpython.rlib import rposix
-    FUNC = lltype.FuncType([lltype.Signed], lltype.Void)
-    func = lltype.functionptr(FUNC, 'set_errno', _callable=rposix.set_errno)
-    v1 = varoftype(lltype.Signed)
-    v3 = varoftype(lltype.Void)
-    op = SpaceOperation('direct_call', [const(func), v1], v3)
-    tr = Transformer(FakeCPU(), FakeBuiltinCallControl())
-    op1 = tr.rewrite_operation(op)
-    assert op1.opname == 'residual_call_ir_v'
-    assert op1.args[0].value == func
-    assert op1.args[1] == ListOfKind('int', [v1])
-    assert op1.args[2] == ListOfKind('ref', [])
-    assert op1.args[3] == 'calldescr-%d' % effectinfo.EffectInfo.OS_SET_ERRNO
-    assert op1.result == v3
+    assert op0.args[2] == 'calldescr-%d' % OS_THREADLOCALREF_ADDR
+    assert op0.result == v
 
 def test_unknown_operation():
     op = SpaceOperation('foobar', [], varoftype(lltype.Void))
diff --git a/rpython/jit/metainterp/test/test_threadlocal.py b/rpython/jit/metainterp/test/test_threadlocal.py
--- a/rpython/jit/metainterp/test/test_threadlocal.py
+++ b/rpython/jit/metainterp/test/test_threadlocal.py
@@ -1,29 +1,19 @@
 import py
 from rpython.jit.metainterp.test.support import LLJitMixin
-from rpython.rlib.rthread import ThreadLocalReference
-from rpython.rlib.jit import dont_look_inside
+from rpython.rtyper.lltypesystem import llmemory
+from rpython.rtyper.lltypesystem.lloperation import llop
 
 
 class ThreadLocalTest(object):
 
     def test_threadlocalref_get(self):
-        class Foo:
-            pass
-        t = ThreadLocalReference(Foo)
-        x = Foo()
-
-        @dont_look_inside
-        def setup():
-            t.set(x)
-
         def f():
-            setup()
-            if t.get() is x:
-                return 42
-            return -666
+            addr1 = llop.threadlocalref_addr(llmemory.Address)
+            # a "does not crash" test only
+            return 1
 
         res = self.interp_operations(f, [])
-        assert res == 42
+        assert res == 1
 
 
 class TestLLtype(ThreadLocalTest, LLJitMixin):
diff --git a/rpython/rlib/rthread.py b/rpython/rlib/rthread.py
--- a/rpython/rlib/rthread.py
+++ b/rpython/rlib/rthread.py
@@ -284,17 +284,17 @@
 
         def getraw():
             _threadlocalref_seeme(self)
-            addr = llop.threadlocalref_addr(rffi.CCHARP)
+            addr = llop.threadlocalref_addr(llmemory.Address)
             return llop.raw_load(FIELDTYPE, addr, offset)
 
         def get_or_make_raw():
             _threadlocalref_seeme(self)
-            addr = llop.threadlocalref_make(rffi.CCHARP)
+            addr = llop.threadlocalref_make(llmemory.Address)
             return llop.raw_load(FIELDTYPE, addr, offset)
 
         def setraw(value):
             _threadlocalref_seeme(self)
-            addr = llop.threadlocalref_addr(rffi.CCHARP)
+            addr = llop.threadlocalref_addr(llmemory.Address)
             llop.raw_store(lltype.Void, addr, offset, value)
 
         self.getraw = getraw
@@ -315,14 +315,16 @@
         self.local = thread._local()      # <- NOT_RPYTHON
         unique_id = ThreadLocalReference._COUNT
         ThreadLocalReference._COUNT += 1
-        ThreadLocalField.__init__(self, llmemory.GCREF, 'tlref%d' % unique_id)
+        ThreadLocalField.__init__(self, lltype.Signed, 'tlref%d' % unique_id)
         getraw = self.getraw
         setraw = self.setraw
 
         def get():
             if we_are_translated():
                 from rpython.rtyper.annlowlevel import cast_gcref_to_instance
-                return cast_gcref_to_instance(Cls, getraw())
+                value = getraw()
+                value = lltype.cast_int_to_ptr(llmemory.GCREF, value)
+                return cast_gcref_to_instance(Cls, value)
             else:
                 return getattr(self.local, 'value', None)
 
@@ -337,7 +339,8 @@
                 if not running_on_llinterp:
                     if gcref:
                         _make_sure_does_not_move(gcref)
-                setraw(gcref)
+                value = lltype.cast_ptr_to_int(gcref)
+                setraw(value)
             else:
                 self.local.value = value
 
diff --git a/rpython/rtyper/llinterp.py b/rpython/rtyper/llinterp.py
--- a/rpython/rtyper/llinterp.py
+++ b/rpython/rtyper/llinterp.py
@@ -919,8 +919,16 @@
     def op_stack_current(self):
         return 0
 
-    def op_threadlocalref_addr(self, key, value):
+    def op_threadlocalref_addr(self):
         raise NotImplementedError("threadlocalref_addr")
+        ## class FakeThreadLocalAddr(object):
+        ##     is_fake_thread_local_addr = True
+        ##     _TYPE = llmemory.Address
+        ##     def _cast_to_int(self, symbolic=None):
+        ##         return FakeThreadLocalAddrAsInt()
+        ## class FakeThreadLocalAddrAsInt(object):
+        ##     _TYPE = lltype.Signed
+        ## return FakeThreadLocalAddr()
 
     # __________________________________________________________
     # operations on addresses
@@ -967,6 +975,9 @@
             ll_p = rffi.cast(rffi.CArrayPtr(RESTYPE),
                              rffi.ptradd(ll_p, offset))
             value = ll_p[0]
+        ## elif getattr(addr, 'is_fake_thread_local_addr', False):
+        ##     assert type(offset) is CDefinedIntSymbolic
+        ##     value = self.llinterpreter.tlobj[offset.expr]
         else:
             assert offset.TYPE == RESTYPE
             value = getattr(addr, str(RESTYPE).lower())[offset.repeat]
@@ -987,6 +998,9 @@
             ll_p = rffi.cast(rffi.CArrayPtr(ARGTYPE),
                              rffi.ptradd(ll_p, offset))
             ll_p[0] = value
+        ## elif getattr(addr, 'is_fake_thread_local_addr', False):
+        ##     assert type(offset) is CDefinedIntSymbolic
+        ##     self.llinterpreter.tlobj[offset.expr] = value
         else:
             assert offset.TYPE == ARGTYPE
             getattr(addr, str(ARGTYPE).lower())[offset.repeat] = value
diff --git a/rpython/rtyper/lltypesystem/test/test_llmemory.py b/rpython/rtyper/lltypesystem/test/test_llmemory.py
--- a/rpython/rtyper/lltypesystem/test/test_llmemory.py
+++ b/rpython/rtyper/lltypesystem/test/test_llmemory.py
@@ -649,3 +649,13 @@
     #assert cast_int_to_adr(i) == adr -- depends on ll2ctypes details
     i = cast_adr_to_int(NULL, mode="forced")
     assert is_valid_int(i) and i == 0
+
+def test_cast_gcref_to_int():
+    A = lltype.GcArray(Address)
+    def f():
+        ptr = lltype.malloc(A, 10)
+        gcref = lltype.cast_opaque_ptr(GCREF, ptr)
+        adr = lltype.cast_ptr_to_int(gcref)
+        assert adr == lltype.cast_ptr_to_int(ptr)
+    f()
+    interpret(f, [])

From noreply at buildbot.pypy.org  Sun Nov 23 15:25:26 2014
From: noreply at buildbot.pypy.org (fijal)
Date: Sun, 23 Nov 2014 15:25:26 +0100 (CET)
Subject: [pypy-commit] pypy optresult: fix test_resume
Message-ID: <20141123142526.279BC1C3288@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: optresult
Changeset: r74651:41ade226e7be
Date: 2014-11-23 16:25 +0200
http://bitbucket.org/pypy/pypy/changeset/41ade226e7be/

Log:	fix test_resume

diff --git a/rpython/jit/metainterp/test/test_resume.py b/rpython/jit/metainterp/test/test_resume.py
--- a/rpython/jit/metainterp/test/test_resume.py
+++ b/rpython/jit/metainterp/test/test_resume.py
@@ -11,7 +11,10 @@
 from rpython.jit.metainterp.optimizeopt.test.test_util import LLtypeMixin
 from rpython.jit.metainterp import executor
 from rpython.jit.codewriter import heaptracker, longlong
-from rpython.jit.metainterp.resoperation import ResOperation
+from rpython.jit.metainterp.resoperation import ResOperation, InputArgInt,\
+     InputArgRef, InputArgFloat
+from rpython.rlib.debug import debug_start, debug_stop, debug_print,\
+     have_debug_prints
 
 class Storage:
     rd_frame_info_list = None
@@ -39,6 +42,7 @@
 def dump_storage(storage, liveboxes):
     "For profiling only."
     debug_start("jit-resume")
+    return # XXX refactor if needed
     if have_debug_prints():
         debug_print('Log storage', compute_unique_id(storage))
         frameinfo = storage.rd_frame_info_list
@@ -125,7 +129,7 @@
         def visitor_dispatch_virtual_type(self, *args):
             return FakeVInfo()
     modifier = ResumeDataVirtualAdder(None, None)
-    v1 = FakeVirtualValue(None, None)
+    v1 = FakeVirtualValue(None)
     vinfo1 = modifier.make_virtual_info(v1, [1, 2, 4])
     vinfo2 = modifier.make_virtual_info(v1, [1, 2, 4])
     assert vinfo1 is vinfo2
@@ -155,7 +159,7 @@
         resvalue = executor.execute(self.cpu, None, opnum, descr, *argboxes)
         op = ResOperation(opnum, list(argboxes), descr)
         op.setvalue(resvalue)
-        self.trace.append(op)
+        self.trace.append((opnum, list(argboxes), resvalue, descr))
         return op
 
     def execute_new_with_vtable(self, known_class):
@@ -260,7 +264,7 @@
     return numb
 
 def test_simple_read():
-    #b1, b2, b3 = [BoxInt(), BoxPtr(), BoxInt()]
+    #b1, b2, b3 = [BoxInt(), InputArgRef(), BoxInt()]
     c1, c2, c3 = [ConstInt(111), ConstInt(222), ConstInt(333)]
     storage = Storage()
     storage.rd_consts = [c1, c2, c3]
@@ -534,7 +538,7 @@
     assert list(numb1.nums) == l1
 
 def test_capture_resumedata():
-    b1, b2, b3 = [BoxInt(), BoxPtr(), BoxInt()]
+    b1, b2, b3 = [InputArgInt(), InputArgRef(), InputArgInt()]
     c1, c2, c3 = [ConstInt(1), ConstInt(2), ConstInt(3)]
     fs = [FakeFrame("code0", 0, b1, c1, b2)]
 
@@ -607,7 +611,7 @@
 
 def test_rebuild_from_resumedata():
     py.test.skip("XXX rewrite")
-    b1, b2, b3 = [BoxInt(), BoxPtr(), BoxInt()]
+    b1, b2, b3 = [BoxInt(), InputArgRef(), BoxInt()]
     c1, c2, c3 = [ConstInt(1), ConstInt(2), ConstInt(3)]    
     storage = Storage()
     fs = [FakeFrame("code0", 0, b1, c1, b2),
@@ -619,7 +623,7 @@
     liveboxes = modifier.finish(FakeOptimizer({}))
     metainterp = MyMetaInterp()
 
-    b1t, b2t, b3t = [BoxInt(), BoxPtr(), BoxInt()]
+    b1t, b2t, b3t = [BoxInt(), InputArgRef(), BoxInt()]
     newboxes = _resume_remap(liveboxes, [b1, b2, b3], b1t, b2t, b3t)
 
     result = rebuild_from_resumedata(metainterp, storage, False)
@@ -631,7 +635,7 @@
 
 def test_rebuild_from_resumedata_with_virtualizable():
     py.test.skip("XXX rewrite")
-    b1, b2, b3, b4 = [BoxInt(), BoxPtr(), BoxInt(), BoxPtr()]
+    b1, b2, b3, b4 = [BoxInt(), InputArgRef(), BoxInt(), InputArgRef()]
     c1, c2, c3 = [ConstInt(1), ConstInt(2), ConstInt(3)]    
     storage = Storage()
     fs = [FakeFrame("code0", 0, b1, c1, b2),
@@ -643,7 +647,7 @@
     liveboxes = modifier.finish(FakeOptimizer({}))
     metainterp = MyMetaInterp()
 
-    b1t, b2t, b3t, b4t = [BoxInt(), BoxPtr(), BoxInt(), BoxPtr()]
+    b1t, b2t, b3t, b4t = [BoxInt(), InputArgRef(), BoxInt(), InputArgRef()]
     newboxes = _resume_remap(liveboxes, [b1, b2, b3, b4], b1t, b2t, b3t, b4t)
 
     result = rebuild_from_resumedata(metainterp, newboxes, storage,
@@ -656,7 +660,7 @@
 
 def test_rebuild_from_resumedata_two_guards():
     py.test.skip("XXX rewrite")
-    b1, b2, b3, b4 = [BoxInt(), BoxPtr(), BoxInt(), BoxInt()]
+    b1, b2, b3, b4 = [BoxInt(), InputArgRef(), BoxInt(), BoxInt()]
     c1, c2, c3 = [ConstInt(1), ConstInt(2), ConstInt(3)]    
     storage = Storage()
     fs = [FakeFrame("code0", 0, b1, c1, b2),
@@ -676,7 +680,7 @@
 
     metainterp = MyMetaInterp()
 
-    b1t, b2t, b3t, b4t = [BoxInt(), BoxPtr(), BoxInt(), BoxInt()]
+    b1t, b2t, b3t, b4t = [BoxInt(), InputArgRef(), BoxInt(), BoxInt()]
     newboxes = _resume_remap(liveboxes, [b1, b2, b3], b1t, b2t, b3t)
 
     result = rebuild_from_resumedata(metainterp, newboxes, storage,
@@ -718,7 +722,7 @@
 def test_rebuild_from_resumedata_two_guards_w_virtuals():
     py.test.skip("XXX rewrite")
     
-    b1, b2, b3, b4, b5 = [BoxInt(), BoxPtr(), BoxInt(), BoxInt(), BoxInt()]
+    b1, b2, b3, b4, b5 = [BoxInt(), InputArgRef(), BoxInt(), BoxInt(), BoxInt()]
     c1, c2, c3, c4 = [ConstInt(1), ConstInt(2), ConstInt(3),
                       LLtypeMixin.nodebox.constbox()]
     storage = Storage()
@@ -738,7 +742,7 @@
     assert storage.rd_virtuals[0].fieldnums == [tag(-1, TAGBOX),
                                                 tag(0, TAGCONST)]
 
-    b6 = BoxPtr()
+    b6 = InputArgRef()
     v6 = virtual_value(b6, c2, None)
     v6.setfield(LLtypeMixin.nextdescr, v6)    
     values = {b2: virtual_value(b2, b4, v6), b6: v6}
@@ -780,7 +784,7 @@
 
 def test_rebuild_from_resumedata_two_guards_w_shared_virtuals():
     py.test.skip("XXX rewrite")
-    b1, b2, b3, b4, b5, b6 = [BoxPtr(), BoxPtr(), BoxInt(), BoxPtr(), BoxInt(), BoxInt()]
+    b1, b2, b3, b4, b5, b6 = [InputArgRef(), InputArgRef(), BoxInt(), InputArgRef(), BoxInt(), BoxInt()]
     c1, c2, c3, c4 = [ConstInt(1), ConstInt(2), ConstInt(3),
                       LLtypeMixin.nodebox.constbox()]
     storage = Storage()
@@ -808,7 +812,7 @@
 
 def test_resumedata_top_recursive_virtuals():
     py.test.skip("XXX rewrite")
-    b1, b2, b3 = [BoxPtr(), BoxPtr(), BoxInt()]
+    b1, b2, b3 = [InputArgRef(), InputArgRef(), BoxInt()]
     storage = Storage()
     fs = [FakeFrame("code0", 0, b1, b2)]
     capture_resumedata(fs, None, [], storage)
@@ -880,7 +884,8 @@
     assert memo.consts[index] is const
 
 def test_ResumeDataLoopMemo_number():
-    b1, b2, b3, b4, b5 = [BoxInt(), BoxInt(), BoxInt(), BoxPtr(), BoxPtr()]
+    b1, b2, b3, b4, b5 = [InputArgInt(), InputArgInt(), InputArgInt(),
+                          InputArgRef(), InputArgRef()]
     c1, c2, c3, c4 = [ConstInt(1), ConstInt(2), ConstInt(3), ConstInt(4)]    
 
     env = [b1, c1, b2, b1, c2]
@@ -968,7 +973,7 @@
 
 def test_ResumeDataLoopMemo_number_boxes():
     memo = ResumeDataLoopMemo(FakeMetaInterpStaticData())
-    b1, b2 = [BoxInt(), BoxInt()]
+    b1, b2 = [InputArgInt(), InputArgInt()]
     assert memo.num_cached_boxes() == 0
     boxes = []
     num = memo.assign_number_to_box(b1, boxes)
@@ -997,7 +1002,7 @@
 
 def test_ResumeDataLoopMemo_number_virtuals():
     memo = ResumeDataLoopMemo(FakeMetaInterpStaticData())
-    b1, b2 = [BoxInt(), BoxInt()]
+    b1, b2 = [InputArgInt(), InputArgInt()]
     assert memo.num_cached_virtuals() == 0
     num = memo.assign_number_to_virtual(b1)
     assert num == -1
@@ -1017,8 +1022,8 @@
     assert memo.num_cached_virtuals() == 0
 
 def test_register_virtual_fields():
-    b1, b2 = BoxInt(), BoxInt()
-    vbox = BoxPtr()
+    b1, b2 = InputArgInt(), InputArgInt()
+    vbox = InputArgRef()
     modifier = ResumeDataVirtualAdder(None, None)
     modifier.liveboxes_from_env = {}
     modifier.liveboxes = {}
@@ -1131,11 +1136,11 @@
 
 
 def test_virtual_adder_no_op_renaming():
-    b1s, b2s, b3s = [BoxInt(1), BoxInt(2), BoxInt(3)]
+    b1s, b2s, b3s = [InputArgInt(1), InputArgInt(2), InputArgInt(3)]
     storage = make_storage(b1s, b2s, b3s)
     memo = ResumeDataLoopMemo(FakeMetaInterpStaticData())
     modifier = ResumeDataVirtualAdder(storage, memo)
-    b1_2 = BoxInt()
+    b1_2 = InputArgInt()
     class FakeValue(object):
 
         def is_virtual(self):
@@ -1148,7 +1153,7 @@
     values = {b1s: val, b2s: val}  
     liveboxes = modifier.finish(FakeOptimizer(values))
     assert storage.rd_snapshot is None
-    b1t, b3t = [BoxInt(11), BoxInt(33)]
+    b1t, b3t = [InputArgInt(11), InputArgInt(33)]
     newboxes = _resume_remap(liveboxes, [b1_2, b3s], b1t, b3t)
     metainterp = MyMetaInterp()
     reader = ResumeDataFakeReader(storage, newboxes, metainterp)
@@ -1162,13 +1167,13 @@
 
 
 def test_virtual_adder_make_constant():
-    b1s, b2s, b3s = [BoxInt(1), BoxPtr(), BoxInt(3)]
+    b1s, b2s, b3s = [InputArgInt(1), InputArgRef(), InputArgInt(3)]
     b1s = ConstInt(111)
     storage = make_storage(b1s, b2s, b3s)
     memo = ResumeDataLoopMemo(FakeMetaInterpStaticData())        
     modifier = ResumeDataVirtualAdder(storage, memo)
     liveboxes = modifier.finish(FakeOptimizer({}))
-    b2t, b3t = [BoxPtr(demo55o), BoxInt(33)]
+    b2t, b3t = [InputArgRef(demo55o), InputArgInt(33)]
     newboxes = _resume_remap(liveboxes, [b2s, b3s], b2t, b3t)
     metainterp = MyMetaInterp()
     reader = ResumeDataFakeReader(storage, newboxes, metainterp)
@@ -1183,7 +1188,7 @@
 
 
 def test_virtual_adder_make_virtual():
-    b2s, b3s, b4s, b5s = [BoxPtr(), BoxInt(3), BoxPtr(), BoxPtr()]  
+    b2s, b3s, b4s, b5s = [InputArgRef(), InputArgInt(3), InputArgRef(), InputArgRef()]  
     c1s = ConstInt(111)
     storage = Storage()
     memo = ResumeDataLoopMemo(FakeMetaInterpStaticData())
@@ -1214,7 +1219,7 @@
     storage.rd_consts = memo.consts[:]
     storage.rd_numb = None
     # resume
-    b3t, b5t = [BoxInt(33), BoxPtr(demo55o)]
+    b3t, b5t = [InputArgInt(33), InputArgRef(demo55o)]
     newboxes = _resume_remap(liveboxes, [#b2s -- virtual
                                          b3s,
                                          #b4s -- virtual
@@ -1230,10 +1235,10 @@
     trace = metainterp.trace
     b2new = (rop.NEW_WITH_VTABLE, [ConstAddr(LLtypeMixin.node_vtable_adr,
                                          LLtypeMixin.cpu)],
-                              b2t, None)
+                              b2t.getref_base(), None)
     b4new = (rop.NEW_WITH_VTABLE, [ConstAddr(LLtypeMixin.node_vtable_adr2,
                                          LLtypeMixin.cpu)],
-                              b4t, None)
+                              b4t.getref_base(), None)
     b2set = [(rop.SETFIELD_GC, [b2t, b4t],      None, LLtypeMixin.nextdescr),
              (rop.SETFIELD_GC, [b2t, c1s],      None, LLtypeMixin.valuedescr)]
     b4set = [(rop.SETFIELD_GC, [b4t, b2t],     None, LLtypeMixin.nextdescr),
@@ -1249,7 +1254,7 @@
             assert x in expected
             expected.remove(x)
 
-    ptr = b2t.value._obj.container._as_ptr()
+    ptr = b2t.getref_base()._obj.container._as_ptr()
     assert lltype.typeOf(ptr) == lltype.Ptr(LLtypeMixin.NODE)
     assert ptr.value == 111
     ptr2 = ptr.next
@@ -1266,7 +1271,7 @@
         del Const.__eq__
 
 def test_virtual_adder_make_varray():
-    b2s, b4s = [BoxPtr(), BoxInt(4)]
+    b2s, b4s = [InputArgRef(), InputArgInt(4)]
     c1s = ConstInt(111)
     storage = Storage()
     memo = ResumeDataLoopMemo(FakeMetaInterpStaticData())
@@ -1285,7 +1290,7 @@
     storage.rd_consts = memo.consts[:]
     storage.rd_numb = None
     # resume
-    b1t, b3t, b4t = [BoxInt(11), BoxInt(33), BoxInt(44)]
+    b1t, b3t, b4t = [InputArgInt(11), InputArgInt(33), InputArgInt(44)]
     newboxes = _resume_remap(liveboxes, [#b2s -- virtual
                                          b4s],
                                          b4t)
@@ -1296,7 +1301,8 @@
     b2t = reader.decode_ref(tag(0, TAGVIRTUAL))
     trace = metainterp.trace
     expected = [
-        (rop.NEW_ARRAY, [ConstInt(2)], b2t, LLtypeMixin.arraydescr),
+        (rop.NEW_ARRAY, [ConstInt(2)], b2t.getref_base(),
+         LLtypeMixin.arraydescr),
         (rop.SETARRAYITEM_GC, [b2t,ConstInt(0), b4t],None,
                               LLtypeMixin.arraydescr),
         (rop.SETARRAYITEM_GC, [b2t,ConstInt(1), c1s], None,
@@ -1306,7 +1312,7 @@
         for x, y in zip(expected, trace):
             assert x == y
     #
-    ptr = b2t.value._obj.container._as_ptr()
+    ptr = b2t.getref_base()._obj.container._as_ptr()
     assert lltype.typeOf(ptr) == lltype.Ptr(lltype.GcArray(lltype.Signed))
     assert len(ptr) == 2
     assert ptr[0] == 44
@@ -1314,7 +1320,7 @@
 
 
 def test_virtual_adder_make_vstruct():
-    b2s, b4s = [BoxPtr(), BoxPtr()]
+    b2s, b4s = [InputArgRef(), InputArgRef()]
     c1s = ConstInt(111)
     storage = Storage()
     memo = ResumeDataLoopMemo(FakeMetaInterpStaticData())
@@ -1331,7 +1337,7 @@
     dump_storage(storage, liveboxes)
     storage.rd_consts = memo.consts[:]
     storage.rd_numb = None
-    b4t = BoxPtr()
+    b4t = InputArgRef()
     newboxes = _resume_remap(liveboxes, [#b2s -- virtual
                                          b4s], b4t)
     #
@@ -1343,7 +1349,7 @@
 
     trace = metainterp.trace
     expected = [
-        (rop.NEW, [], b2t, LLtypeMixin.ssize),
+        (rop.NEW, [], b2t.getref_base(), LLtypeMixin.ssize),
         (rop.SETFIELD_GC, [b2t, c1s],  None, LLtypeMixin.adescr),
         (rop.SETFIELD_GC, [b2t, b4t], None, LLtypeMixin.bdescr),
         ]
@@ -1351,14 +1357,14 @@
         for x, y in zip(expected, trace):
             assert x == y
     #
-    ptr = b2t.value._obj.container._as_ptr()
+    ptr = b2t.getref_base()._obj.container._as_ptr()
     assert lltype.typeOf(ptr) == lltype.Ptr(LLtypeMixin.S)
     assert ptr.a == 111
     assert ptr.b == lltype.nullptr(LLtypeMixin.NODE)
 
 
 def test_virtual_adder_pending_fields():
-    b2s, b4s = [BoxPtr(), BoxPtr()]
+    b2s, b4s = [InputArgRef(), InputArgRef()]
     storage = Storage()
     memo = ResumeDataLoopMemo(FakeMetaInterpStaticData())
     modifier = ResumeDataVirtualAdder(storage, memo)
@@ -1380,8 +1386,8 @@
     storage.rd_numb = None
     # resume
     demo55.next = lltype.nullptr(LLtypeMixin.NODE)
-    b2t = BoxPtr(demo55o)
-    b4t = BoxPtr(demo66o)
+    b2t = InputArgRef(demo55o)
+    b4t = InputArgRef(demo66o)
     newboxes = _resume_remap(liveboxes, [b2s, b4s], b2t, b4t)
 
     metainterp = MyMetaInterp()

From noreply at buildbot.pypy.org  Sun Nov 23 15:40:02 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Sun, 23 Nov 2014 15:40:02 +0100 (CET)
Subject: [pypy-commit] pypy portable-threadlocal: Implement
 OS_THREADLOCALREF_ADDR in the x86 backend.
Message-ID: <20141123144002.2425E1C022D@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: portable-threadlocal
Changeset: r74652:c187d3b63e14
Date: 2014-11-23 15:39 +0100
http://bitbucket.org/pypy/pypy/changeset/c187d3b63e14/

Log:	Implement OS_THREADLOCALREF_ADDR in the x86 backend.

diff --git a/rpython/jit/backend/llsupport/llmodel.py b/rpython/jit/backend/llsupport/llmodel.py
--- a/rpython/jit/backend/llsupport/llmodel.py
+++ b/rpython/jit/backend/llsupport/llmodel.py
@@ -217,7 +217,13 @@
         return lltype.cast_opaque_ptr(llmemory.GCREF, frame)
 
     def make_execute_token(self, *ARGS):
-        FUNCPTR = lltype.Ptr(lltype.FuncType([llmemory.GCREF],
+        # The JIT backend must generate functions with the following
+        # signature: it takes the jitframe and the threadlocal_addr
+        # as arguments, and it returns the (possibly reallocated) jitframe.
+        # The backend can optimize OS_THREADLOCALREF_ADDR calls to return
+        # this threadlocal_addr, but only if 'translate_support_code':
+        # in untranslated tests, threadlocal_addr is a dummy NULL.
+        FUNCPTR = lltype.Ptr(lltype.FuncType([llmemory.GCREF, llmemory.Address],
                                              llmemory.GCREF))
 
         lst = [(i, history.getkind(ARG)[0]) for i, ARG in enumerate(ARGS)]
@@ -249,8 +255,13 @@
                     else:
                         assert kind == history.REF
                         self.set_ref_value(ll_frame, num, arg)
+                if self.translate_support_code:
+                    ll_threadlocal_addr = llop.threadlocalref_addr(
+                        llmemory.Address)
+                else:
+                    ll_threadlocal_addr = llmemory.NULL
                 llop.gc_writebarrier(lltype.Void, ll_frame)
-                ll_frame = func(ll_frame)
+                ll_frame = func(ll_frame, ll_threadlocal_addr)
             finally:
                 if not self.translate_support_code:
                     LLInterpreter.current_interpreter = prev_interpreter
diff --git a/rpython/jit/backend/llsupport/test/ztranslation_test.py b/rpython/jit/backend/llsupport/test/ztranslation_test.py
--- a/rpython/jit/backend/llsupport/test/ztranslation_test.py
+++ b/rpython/jit/backend/llsupport/test/ztranslation_test.py
@@ -26,8 +26,6 @@
         # - profiler
         # - full optimizer
         # - floats neg and abs
-        # - threadlocalref_get
-        # - get_errno, set_errno
         # - llexternal with macro=True
 
         class Frame(object):
@@ -36,10 +34,6 @@
             def __init__(self, i):
                 self.i = i
 
-        class Foo(object):
-            pass
-        t = ThreadLocalReference(Foo)
-
         eci = ExternalCompilationInfo(post_include_bits=['''
 #define pypy_my_fabs(x)  fabs(x)
 '''])
@@ -74,9 +68,6 @@
                 k = myabs1(myabs2(j))
                 if k - abs(j):  raise ValueError
                 if k - abs(-j): raise ValueError
-                if t.get().nine != 9: raise ValueError
-                rposix.set_errno(total)
-                if rposix.get_errno() != total: raise ValueError
             return chr(total % 253)
         #
         class Virt2(object):
@@ -104,12 +95,8 @@
             return res
         #
         def main(i, j):
-            foo = Foo()
-            foo.nine = -(i + j)
-            t.set(foo)
             a_char = f(i, j)
             a_float = libffi_stuff(i, j)
-            keepalive_until_here(foo)
             return ord(a_char) * 10 + int(a_float)
         expected = main(40, -49)
         res = self.meta_interp(main, [40, -49])
@@ -121,6 +108,7 @@
 
     def test_direct_assembler_call_translates(self):
         """Test CALL_ASSEMBLER and the recursion limit"""
+        # - also tests threadlocalref_get
         from rpython.rlib.rstackovf import StackOverflow
 
         class Thing(object):
@@ -138,6 +126,10 @@
 
         somewhere_else = SomewhereElse()
 
+        class Foo(object):
+            pass
+        t = ThreadLocalReference(Foo)
+
         def change(newthing):
             somewhere_else.frame.thing = newthing
 
@@ -163,6 +155,7 @@
                     nextval = 13
                 frame.thing = Thing(nextval + 1)
                 i += 1
+                if t.get().nine != 9: raise ValueError
             return frame.thing.val
 
         driver2 = JitDriver(greens = [], reds = ['n'])
@@ -184,13 +177,24 @@
                 n = portal2(n)
         assert portal2(10) == -9
 
+        def setup(value):
+            foo = Foo()
+            foo.nine = value
+            t.set(foo)
+            return foo
+
         def mainall(codeno, bound):
-            return main(codeno) + main2(bound)
+            foo = setup(bound + 8)
+            result = main(codeno) + main2(bound)
+            keepalive_until_here(foo)
+            return result
 
+        tmp_obj = setup(9)
+        expected_1 = main(0)
         res = self.meta_interp(mainall, [0, 1], inline=True,
                                policy=StopAtXPolicy(change))
         print hex(res)
-        assert res & 255 == main(0)
+        assert res & 255 == expected_1
         bound = res & ~255
         assert 1024 <= bound <= 131072
         assert bound & (bound-1) == 0       # a power of two
diff --git a/rpython/jit/backend/x86/arch.py b/rpython/jit/backend/x86/arch.py
--- a/rpython/jit/backend/x86/arch.py
+++ b/rpython/jit/backend/x86/arch.py
@@ -34,10 +34,16 @@
     FRAME_FIXED_SIZE = 19
     PASS_ON_MY_FRAME = 15
     JITFRAME_FIXED_SIZE = 6 + 8 * 2 # 6 GPR + 8 XMM * 2 WORDS/float
+    # 'threadlocal_addr' is passed as 2nd argument on the stack,
+    # and it can be left here for when it is needed
+    THREADLOCAL_OFS = (FRAME_FIXED_SIZE + 2) * WORD
 else:
-    # rbp + rbx + r12 + r13 + r14 + r15 + 13 extra words = 19
+    # rbp + rbx + r12 + r13 + r14 + r15 + threadlocal + 12 extra words = 19
     FRAME_FIXED_SIZE = 19
-    PASS_ON_MY_FRAME = 13
+    PASS_ON_MY_FRAME = 12
     JITFRAME_FIXED_SIZE = 28 # 13 GPR + 15 XMM
+    # 'threadlocal_addr' is passed as 2nd argument in %esi,
+    # and is moved into this frame location
+    THREADLOCAL_OFS = (FRAME_FIXED_SIZE - 1) * WORD
 
 assert PASS_ON_MY_FRAME >= 12       # asmgcc needs at least JIT_USE_WORDS + 3
diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py
--- a/rpython/jit/backend/x86/assembler.py
+++ b/rpython/jit/backend/x86/assembler.py
@@ -18,7 +18,7 @@
 from rpython.jit.backend.llsupport.regalloc import (get_scale, valid_addressing_size)
 from rpython.jit.backend.x86.arch import (FRAME_FIXED_SIZE, WORD, IS_X86_64,
                                        JITFRAME_FIXED_SIZE, IS_X86_32,
-                                       PASS_ON_MY_FRAME)
+                                       PASS_ON_MY_FRAME, THREADLOCAL_OFS)
 from rpython.jit.backend.x86.regloc import (eax, ecx, edx, ebx, esp, ebp, esi,
     xmm0, xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7, r8, r9, r10, r11, edi,
     r12, r13, r14, r15, X86_64_SCRATCH_REG, X86_64_XMM_SCRATCH_REG,
@@ -730,6 +730,7 @@
         self.mc.SUB_ri(esp.value, FRAME_FIXED_SIZE * WORD)
         self.mc.MOV_sr(PASS_ON_MY_FRAME * WORD, ebp.value)
         if IS_X86_64:
+            self.mc.MOV_sr(THREADLOCAL_OFS, esi.value)
             self.mc.MOV_rr(ebp.value, edi.value)
         else:
             self.mc.MOV_rs(ebp.value, (FRAME_FIXED_SIZE + 1) * WORD)
@@ -1955,7 +1956,8 @@
         self._emit_guard_not_forced(guard_token)
 
     def _call_assembler_emit_call(self, addr, argloc, _):
-        self.simple_call(addr, [argloc])
+        threadlocal_loc = RawEspLoc(THREADLOCAL_OFS, INT)
+        self.simple_call(addr, [argloc, threadlocal_loc])
 
     def _call_assembler_emit_helper_call(self, addr, arglocs, result_loc):
         self.simple_call(addr, arglocs, result_loc)
@@ -2321,8 +2323,13 @@
         self.mc.MOV_rr(reg.value, ebp.value)
 
     def threadlocalref_addr(self, resloc):
+        # This simply loads the stack location THREADLOCAL_OFS into a
+        # register.  It is only supported if 'translate_support_code' is
+        # true; otherwise, the original call to the piece of assembler
+        # was done with a dummy NULL value.
+        assert self.cpu.translate_support_code
         assert isinstance(resloc, RegLoc)
-        XXX
+        self.mc.MOV_rs(resloc.value, THREADLOCAL_OFS)
 
     def get_set_errno(self, op, loc, issue_a_write):
         # this function is only called on Linux
diff --git a/rpython/jit/backend/x86/regalloc.py b/rpython/jit/backend/x86/regalloc.py
--- a/rpython/jit/backend/x86/regalloc.py
+++ b/rpython/jit/backend/x86/regalloc.py
@@ -694,8 +694,11 @@
         self.perform_math(op, [loc0], loc0)
 
     def _consider_threadlocalref_addr(self, op):
-        resloc = self.force_allocate_reg(op.result)
-        self.assembler.threadlocalref_addr(resloc)
+        if self.translate_support_code:
+            resloc = self.force_allocate_reg(op.result)
+            self.assembler.threadlocalref_addr(resloc)
+        else:
+            self._consider_call(op)
 
     def _call(self, op, arglocs, force_store=[], guard_not_forced_op=None):
         # we need to save registers on the stack:
@@ -774,8 +777,8 @@
                         return
             if oopspecindex == EffectInfo.OS_MATH_SQRT:
                 return self._consider_math_sqrt(op)
-            #if oopspecindex == EffectInfo.OS_THREADLOCALREF_ADDR:
-            #    return self._consider_threadlocalref_addr(op)
+            if oopspecindex == EffectInfo.OS_THREADLOCALREF_ADDR:
+                return self._consider_threadlocalref_addr(op)
             if oopspecindex == EffectInfo.OS_MATH_READ_TIMESTAMP:
                 return self._consider_math_read_timestamp(op)
         self._consider_call(op)

From noreply at buildbot.pypy.org  Sun Nov 23 16:26:58 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Sun, 23 Nov 2014 16:26:58 +0100 (CET)
Subject: [pypy-commit] pypy portable-threadlocal: in-progress: ARM support
Message-ID: <20141123152658.598751D3592@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: portable-threadlocal
Changeset: r74653:c97dcdd74131
Date: 2014-11-23 16:26 +0100
http://bitbucket.org/pypy/pypy/changeset/c97dcdd74131/

Log:	in-progress: ARM support

diff --git a/rpython/jit/backend/arm/assembler.py b/rpython/jit/backend/arm/assembler.py
--- a/rpython/jit/backend/arm/assembler.py
+++ b/rpython/jit/backend/arm/assembler.py
@@ -497,9 +497,11 @@
         if self.cpu.supports_floats:
             mc.VPOP([reg.value for reg in r.callee_saved_vfp_registers],
                                                                     cond=cond)
-        # pop all callee saved registers and IP to keep the alignment
+        # pop all callee saved registers.  This pops 'pc' last.
+        # It also pops the threadlocal_addr back into 'r1', but it
+        # is not needed any more and will be discarded.
         mc.POP([reg.value for reg in r.callee_restored_registers] +
-                                                       [r.ip.value], cond=cond)
+                                                       [r.r1.value], cond=cond)
         mc.BKPT()
 
     def gen_func_prolog(self):
@@ -508,11 +510,16 @@
         if self.cpu.supports_floats:
             stack_size += len(r.callee_saved_vfp_registers) * 2 * WORD
 
-        # push all callee saved registers and IP to keep the alignment
+        # push all callee saved registers including lr; and push r1 as
+        # well, which contains the threadlocal_addr argument.  Note that
+        # we're pushing a total of 10 words, which keeps the stack aligned.
         self.mc.PUSH([reg.value for reg in r.callee_saved_registers] +
-                                                        [r.ip.value])
+                                                        [r.r1.value])
+        self.saved_threadlocal_addr = 0   # at offset 0 from location 'sp'
         if self.cpu.supports_floats:
             self.mc.VPUSH([reg.value for reg in r.callee_saved_vfp_registers])
+            self.saved_threadlocal_addr += (
+                len(r.callee_saved_vfp_registers) * 2 * WORD)
         assert stack_size % 8 == 0 # ensure we keep alignment
 
         # set fp to point to the JITFRAME
@@ -952,16 +959,11 @@
             regalloc._check_invariants()
         self.mc.mark_op(None)  # end of the loop
 
-    def regalloc_emit_llong(self, op, arglocs, fcond, regalloc):
+    def regalloc_emit_extra(self, op, arglocs, fcond, regalloc):
+        # for calls to a function with a specifically-supported OS_xxx
         effectinfo = op.getdescr().get_extra_info()
         oopspecindex = effectinfo.oopspecindex
-        asm_llong_operations[oopspecindex](self, op, arglocs, regalloc, fcond)
-        return fcond
-
-    def regalloc_emit_math(self, op, arglocs, fcond, regalloc):
-        effectinfo = op.getdescr().get_extra_info()
-        oopspecindex = effectinfo.oopspecindex
-        asm_math_operations[oopspecindex](self, op, arglocs, regalloc, fcond)
+        asm_extra_operations[oopspecindex](self, op, arglocs, regalloc, fcond)
         return fcond
 
     def patch_trace(self, faildescr, looptoken, bridge_addr, regalloc):
@@ -1509,22 +1511,17 @@
 
 asm_operations = [notimplemented_op] * (rop._LAST + 1)
 asm_operations_with_guard = [notimplemented_op_with_guard] * (rop._LAST + 1)
-asm_llong_operations = {}
-asm_math_operations = {}
+asm_extra_operations = {}
 
 for name, value in ResOpAssembler.__dict__.iteritems():
     if name.startswith('emit_guard_'):
         opname = name[len('emit_guard_'):]
         num = getattr(rop, opname.upper())
         asm_operations_with_guard[num] = value
-    elif name.startswith('emit_op_llong_'):
-        opname = name[len('emit_op_llong_'):]
-        num = getattr(EffectInfo, 'OS_LLONG_' + opname.upper())
-        asm_llong_operations[num] = value
-    elif name.startswith('emit_op_math_'):
-        opname = name[len('emit_op_math_'):]
-        num = getattr(EffectInfo, 'OS_MATH_' + opname.upper())
-        asm_math_operations[num] = value
+    elif name.startswith('emit_opx_'):
+        opname = name[len('emit_opx_'):]
+        num = getattr(EffectInfo, 'OS_' + opname.upper())
+        asm_extra_operations[num] = value
     elif name.startswith('emit_op_'):
         opname = name[len('emit_op_'):]
         num = getattr(rop, opname.upper())
diff --git a/rpython/jit/backend/arm/opassembler.py b/rpython/jit/backend/arm/opassembler.py
--- a/rpython/jit/backend/arm/opassembler.py
+++ b/rpython/jit/backend/arm/opassembler.py
@@ -19,7 +19,7 @@
 from rpython.jit.backend.arm.codebuilder import InstrBuilder, OverwritingBuilder
 from rpython.jit.backend.arm.jump import remap_frame_layout
 from rpython.jit.backend.arm.regalloc import TempBox
-from rpython.jit.backend.arm.locations import imm
+from rpython.jit.backend.arm.locations import imm, RawSPStackLocation
 from rpython.jit.backend.llsupport import symbolic
 from rpython.jit.backend.llsupport.gcmap import allocate_gcmap
 from rpython.jit.backend.llsupport.descr import InteriorFieldDescr
@@ -971,7 +971,9 @@
         return fcond
 
     def _call_assembler_emit_call(self, addr, argloc, resloc):
-        self.simple_call(addr, [argloc], result_loc=resloc)
+        ofs = self.saved_threadlocal_addr
+        threadlocal_loc = RawSPStackLocation(ofs, INT)
+        self.simple_call(addr, [argloc, threadlocal_loc], result_loc=resloc)
 
     def _call_assembler_emit_helper_call(self, addr, arglocs, resloc):
         self.simple_call(addr, arglocs, result_loc=resloc)
@@ -1097,7 +1099,7 @@
 
     emit_op_float_neg = gen_emit_unary_float_op('float_neg', 'VNEG')
     emit_op_float_abs = gen_emit_unary_float_op('float_abs', 'VABS')
-    emit_op_math_sqrt = gen_emit_unary_float_op('math_sqrt', 'VSQRT')
+    emit_opx_math_sqrt = gen_emit_unary_float_op('math_sqrt', 'VSQRT')
 
     emit_op_float_lt = gen_emit_float_cmp_op('float_lt', c.VFP_LT)
     emit_op_float_le = gen_emit_float_cmp_op('float_le', c.VFP_LE)
@@ -1131,13 +1133,13 @@
 
     # the following five instructions are only ARMv7;
     # regalloc.py won't call them at all on ARMv6
-    emit_op_llong_add = gen_emit_float_op('llong_add', 'VADD_i64')
-    emit_op_llong_sub = gen_emit_float_op('llong_sub', 'VSUB_i64')
-    emit_op_llong_and = gen_emit_float_op('llong_and', 'VAND_i64')
-    emit_op_llong_or = gen_emit_float_op('llong_or', 'VORR_i64')
-    emit_op_llong_xor = gen_emit_float_op('llong_xor', 'VEOR_i64')
+    emit_opx_llong_add = gen_emit_float_op('llong_add', 'VADD_i64')
+    emit_opx_llong_sub = gen_emit_float_op('llong_sub', 'VSUB_i64')
+    emit_opx_llong_and = gen_emit_float_op('llong_and', 'VAND_i64')
+    emit_opx_llong_or = gen_emit_float_op('llong_or', 'VORR_i64')
+    emit_opx_llong_xor = gen_emit_float_op('llong_xor', 'VEOR_i64')
 
-    def emit_op_llong_to_int(self, op, arglocs, regalloc, fcond):
+    def emit_opx_llong_to_int(self, op, arglocs, regalloc, fcond):
         loc = arglocs[0]
         res = arglocs[1]
         assert loc.is_vfp_reg()
@@ -1271,3 +1273,9 @@
             regalloc.rm.possibly_free_var(length_box)
         regalloc.rm.possibly_free_var(dstaddr_box)
         return fcond
+
+    def emit_opx_threadlocalref_addr(self, op, arglocs, regalloc, fcond):
+        res, = arglocs
+        ofs = self.saved_threadlocal_addr
+        self.load_reg(self.mc, res, r.sp, ofs)
+        return fcond
diff --git a/rpython/jit/backend/arm/regalloc.py b/rpython/jit/backend/arm/regalloc.py
--- a/rpython/jit/backend/arm/regalloc.py
+++ b/rpython/jit/backend/arm/regalloc.py
@@ -373,11 +373,8 @@
         return gcmap
 
     # ------------------------------------------------------------
-    def perform_llong(self, op, args, fcond):
-        return self.assembler.regalloc_emit_llong(op, args, fcond, self)
-
-    def perform_math(self, op, args, fcond):
-        return self.assembler.regalloc_emit_math(op, args, self, fcond)
+    def perform_extra(self, op, args, fcond):
+        return self.assembler.regalloc_emit_extra(op, args, fcond, self)
 
     def force_spill_var(self, var):
         if var.type == FLOAT:
@@ -552,15 +549,19 @@
                             EffectInfo.OS_LLONG_XOR):
                 if self.cpu.cpuinfo.arch_version >= 7:
                     args = self._prepare_llong_binop_xx(op, fcond)
-                    self.perform_llong(op, args, fcond)
+                    self.perform_extra(op, args, fcond)
                     return
             elif oopspecindex == EffectInfo.OS_LLONG_TO_INT:
                 args = self._prepare_llong_to_int(op, fcond)
-                self.perform_llong(op, args, fcond)
+                self.perform_extra(op, args, fcond)
                 return
             elif oopspecindex == EffectInfo.OS_MATH_SQRT:
-                args = self.prepare_op_math_sqrt(op, fcond)
-                self.perform_math(op, args, fcond)
+                args = self._prepare_op_math_sqrt(op, fcond)
+                self.perform_extra(op, args, fcond)
+                return
+            elif oopspecindex == EffectInfo.OS_THREADLOCALREF_ADDR:
+                args = self._prepare_threadlocalref_addr(op, fcond)
+                self.perform_extra(op, args, fcond)
                 return
             #elif oopspecindex == EffectInfo.OS_MATH_READ_TIMESTAMP:
             #    ...
@@ -618,6 +619,10 @@
         res = self.force_allocate_reg(op.result)
         return [loc0, res]
 
+    def _prepare_threadlocalref_addr(self, op, fcond):
+        res = self.force_allocate_reg(op.result)
+        return [res]
+
     def _prepare_guard(self, op, args=None):
         if args is None:
             args = []
@@ -1278,7 +1283,7 @@
     prepare_guard_float_ge = prepare_float_op(guard=True,
                             float_result=False, name='prepare_guard_float_ge')
 
-    def prepare_op_math_sqrt(self, op, fcond):
+    def _prepare_op_math_sqrt(self, op, fcond):
         loc = self.make_sure_var_in_reg(op.getarg(1))
         self.possibly_free_vars_for_op(op)
         self.free_temp_vars()

From noreply at buildbot.pypy.org  Sun Nov 23 23:54:23 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Sun, 23 Nov 2014 23:54:23 +0100 (CET)
Subject: [pypy-commit] pypy default: Tweak the logic for ll_isinstance(): in
 the common case of no
Message-ID: <20141123225423.8C8C41C31B8@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r74654:c286dc658327
Date: 2014-11-23 23:54 +0100
http://bitbucket.org/pypy/pypy/changeset/c286dc658327/

Log:	Tweak the logic for ll_isinstance(): in the common case of no
	subclasses, improve the performance (a little bit). Fixes an age-
	old "XXX re-implement the following optimization".

diff --git a/rpython/rtyper/normalizecalls.py b/rpython/rtyper/normalizecalls.py
--- a/rpython/rtyper/normalizecalls.py
+++ b/rpython/rtyper/normalizecalls.py
@@ -298,12 +298,16 @@
 
 # ____________________________________________________________
 
+class TooLateForNewSubclass(Exception):
+    pass
+
 class TotalOrderSymbolic(ComputedIntSymbolic):
 
     def __init__(self, orderwitness, peers):
         self.orderwitness = orderwitness
         self.peers = peers
         self.value = None
+        self._with_subclasses = None    # unknown
         peers.append(self)
 
     def __cmp__(self, other):
@@ -320,12 +324,34 @@
     def __rsub__(self, other):
         return other - self.compute_fn()
 
+    def check_any_subclass_in_peer_list(self, i):
+        # check if the next peer, in order, is or not the end
+        # marker for this start marker
+        assert self.peers[i] is self
+        return self.peers[i + 1].orderwitness != self.orderwitness + [MAX]
+
+    def number_with_subclasses(self):
+        # Return True or False depending on whether this is the
+        # subclassrange_min corresponding to a class which has subclasses
+        # or not.  If this is called and returns False, then adding later
+        # new subclasses will crash in compute_fn().
+        if self._with_subclasses is None:     # unknown so far
+            self.peers.sort()
+            i = self.peers.index(self)
+            self._with_subclasses = self.check_any_subclass_in_peer_list(i)
+        return self._with_subclasses
+
     def compute_fn(self):
         if self.value is None:
             self.peers.sort()
             for i, peer in enumerate(self.peers):
                 assert peer.value is None or peer.value == i
                 peer.value = i
+                #
+                if peer._with_subclasses is False:
+                    if peer.check_any_subclass_in_peer_list(i):
+                        raise TooLateForNewSubclass
+                #
             assert self.value is not None
         return self.value
 
diff --git a/rpython/rtyper/rclass.py b/rpython/rtyper/rclass.py
--- a/rpython/rtyper/rclass.py
+++ b/rpython/rtyper/rclass.py
@@ -1007,14 +1007,11 @@
         v_obj, v_cls = hop.inputargs(instance_repr, class_repr)
         if isinstance(v_cls, Constant):
             cls = v_cls.value
-            # XXX re-implement the following optimization
-            #if cls.subclassrange_max == cls.subclassrange_min:
-            #    # a class with no subclass
-            #    return hop.gendirectcall(rclass.ll_isinstance_exact, v_obj, v_cls)
-            #else:
-            minid = hop.inputconst(Signed, cls.subclassrange_min)
-            maxid = hop.inputconst(Signed, cls.subclassrange_max)
-            return hop.gendirectcall(ll_isinstance_const, v_obj, minid, maxid)
+            llf, llf_nonnull = make_ll_isinstance(self.rtyper, cls)
+            if hop.args_s[0].can_be_None:
+                return hop.gendirectcall(llf, v_obj)
+            else:
+                return hop.gendirectcall(llf_nonnull, v_obj)
         else:
             return hop.gendirectcall(ll_isinstance, v_obj, v_cls)
 
@@ -1128,16 +1125,26 @@
     obj_cls = obj.typeptr
     return ll_issubclass(obj_cls, cls)
 
-def ll_isinstance_const(obj, minid, maxid):
-    if not obj:
-        return False
-    return ll_issubclass_const(obj.typeptr, minid, maxid)
-
-def ll_isinstance_exact(obj, cls):
-    if not obj:
-        return False
-    obj_cls = obj.typeptr
-    return obj_cls == cls
+def make_ll_isinstance(rtyper, cls):
+    try:
+        return rtyper.isinstance_helpers[cls._obj]
+    except KeyError:
+        minid = cls.subclassrange_min
+        maxid = cls.subclassrange_max
+        if minid.number_with_subclasses():
+            def ll_isinstance_const_nonnull(obj):
+                objid = obj.typeptr.subclassrange_min
+                return llop.int_between(Bool, minid, objid, maxid)
+        else:
+            def ll_isinstance_const_nonnull(obj):
+                return obj.typeptr == cls
+        def ll_isinstance_const(obj):
+            if not obj:
+                return False
+            return ll_isinstance_const_nonnull(obj)
+        result = (ll_isinstance_const, ll_isinstance_const_nonnull)
+        rtyper.isinstance_helpers[cls._obj] = result
+        return result
 
 def ll_runtime_type_info(obj):
     return obj.typeptr.rtti
diff --git a/rpython/rtyper/rtyper.py b/rpython/rtyper/rtyper.py
--- a/rpython/rtyper/rtyper.py
+++ b/rpython/rtyper/rtyper.py
@@ -59,6 +59,7 @@
         self.typererror_count = 0
         # make the primitive_to_repr constant mapping
         self.primitive_to_repr = {}
+        self.isinstance_helpers = {}
         self.exceptiondata = ExceptionData(self)
         self.custom_trace_funcs = []
 
diff --git a/rpython/rtyper/test/test_normalizecalls.py b/rpython/rtyper/test/test_normalizecalls.py
--- a/rpython/rtyper/test/test_normalizecalls.py
+++ b/rpython/rtyper/test/test_normalizecalls.py
@@ -6,6 +6,7 @@
 from rpython.rtyper.test.test_llinterp import interpret
 from rpython.rtyper.lltypesystem import lltype
 from rpython.rtyper.normalizecalls import TotalOrderSymbolic, MAX
+from rpython.rtyper.normalizecalls import TooLateForNewSubclass
 
 
 def test_TotalOrderSymbolic():
@@ -21,6 +22,49 @@
     assert t1 <= 5
     assert t1.value == 0
 
+def test_TotalOrderSymbolic_with_subclasses():
+    lst = []
+    t3 = TotalOrderSymbolic([3, 4, 2, MAX], lst)
+    t1 = TotalOrderSymbolic([3, 4], lst)
+    t2 = TotalOrderSymbolic([3, 4, 2], lst)
+    t4 = TotalOrderSymbolic([3, 4, MAX], lst)
+    assert t1.number_with_subclasses()
+    assert not t2.number_with_subclasses()
+    assert [t.compute_fn() for t in [t1, t2, t3, t4]] == range(4)
+    #
+    lst = []
+    t1 = TotalOrderSymbolic([3, 4], lst)
+    t3 = TotalOrderSymbolic([3, 4, 2, MAX], lst)
+    t4 = TotalOrderSymbolic([3, 4, MAX], lst)
+    t2 = TotalOrderSymbolic([3, 4, 2], lst)
+    assert not t2.number_with_subclasses()
+    assert t1.number_with_subclasses()
+    assert [t.compute_fn() for t in [t1, t2, t3, t4]] == range(4)
+    #
+    lst = []
+    t1 = TotalOrderSymbolic([3, 4], lst)
+    t4 = TotalOrderSymbolic([3, 4, MAX], lst)
+    assert not t1.number_with_subclasses()
+    t2 = TotalOrderSymbolic([3, 4, 2], lst)
+    t3 = TotalOrderSymbolic([3, 4, 2, MAX], lst)
+    py.test.raises(TooLateForNewSubclass, t2.compute_fn)
+    #
+    lst = []
+    t1 = TotalOrderSymbolic([3, 4], lst)
+    t4 = TotalOrderSymbolic([3, 4, MAX], lst)
+    assert not t1.number_with_subclasses()
+    t2 = TotalOrderSymbolic([1], lst)
+    t3 = TotalOrderSymbolic([1, MAX], lst)
+    assert [t.compute_fn() for t in [t2, t3, t1, t4]] == range(4)
+    #
+    lst = []
+    t1 = TotalOrderSymbolic([3, 4], lst)
+    t4 = TotalOrderSymbolic([3, 4, MAX], lst)
+    assert not t1.number_with_subclasses()
+    t2 = TotalOrderSymbolic([6], lst)
+    t3 = TotalOrderSymbolic([6, MAX], lst)
+    assert [t.compute_fn() for t in [t1, t4, t2, t3]] == range(4)
+
 # ____________________________________________________________
 
 class TestNormalize(object):

From noreply at buildbot.pypy.org  Mon Nov 24 09:54:13 2014
From: noreply at buildbot.pypy.org (fijal)
Date: Mon, 24 Nov 2014 09:54:13 +0100 (CET)
Subject: [pypy-commit] pypy optresult: fix test_executor
Message-ID: <20141124085413.89FF41C3140@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: optresult
Changeset: r74655:f82fe958c290
Date: 2014-11-24 10:53 +0200
http://bitbucket.org/pypy/pypy/changeset/f82fe958c290/

Log:	fix test_executor

diff --git a/rpython/jit/backend/test/runner_test.py b/rpython/jit/backend/test/runner_test.py
--- a/rpython/jit/backend/test/runner_test.py
+++ b/rpython/jit/backend/test/runner_test.py
@@ -5,9 +5,9 @@
                                          BasicFinalDescr,
                                          JitCellToken, TargetToken,
                                          ConstInt, ConstPtr,
-                                         ConstFloat)
+                                         ConstFloat, Const)
 from rpython.jit.metainterp.resoperation import ResOperation, rop, InputArgInt,\
-     InputArgFloat
+     InputArgFloat, opname
 from rpython.jit.metainterp.typesystem import deref
 from rpython.jit.codewriter.effectinfo import EffectInfo
 from rpython.jit.tool.oparser import parse
@@ -53,11 +53,11 @@
         self.cpu.compile_loop(inputargs, operations, looptoken)
         args = []
         for box in inputargs:
-            if isinstance(box, BoxInt):
+            if box.type == 'i':
                 args.append(box.getint())
-            elif isinstance(box, BoxPtr):
+            elif box.type == 'r':
                 args.append(box.getref_base())
-            elif isinstance(box, BoxFloat):
+            elif box.type == 'f':
                 args.append(box.getfloatstorage())
             else:
                 raise NotImplementedError(box)
@@ -67,11 +67,11 @@
         else:
             self.guard_failed = True
         if result_type == 'int':
-            return BoxInt(self.cpu.get_int_value(deadframe, 0))
+            return self.cpu.get_int_value(deadframe, 0)
         elif result_type == 'ref':
-            return BoxPtr(self.cpu.get_ref_value(deadframe, 0))
+            return self.cpu.get_ref_value(deadframe, 0)
         elif result_type == 'float':
-            return BoxFloat(self.cpu.get_float_value(deadframe, 0))
+            return self.cpu.get_float_value(deadframe, 0)
         elif result_type == 'void':
             return None
         else:
@@ -79,23 +79,12 @@
 
     def _get_single_operation_list(self, opnum, result_type, valueboxes,
                                    descr):
+        op0 = ResOperation(opnum, valueboxes)
         if result_type == 'void':
-            result = None
-        elif result_type == 'int':
-            result = BoxInt()
-        elif result_type == 'ref':
-            result = BoxPtr()
-        elif result_type == 'float':
-            result = BoxFloat()
+            op1 = ResOperation(rop.FINISH, [], descr=BasicFinalDescr(0))
         else:
-            raise ValueError(result_type)
-        if result is None:
-            results = []
-        else:
-            results = [result]
-        operations = [ResOperation(opnum, valueboxes, result),
-                      ResOperation(rop.FINISH, results, None,
-                                   descr=BasicFinalDescr(0))]
+            op1 = ResOperation(rop.FINISH, [op0], descr=BasicFinalDescr(0))
+        operations = [op0, op1]
         if operations[0].is_guard():
             operations[0].setfailargs([])
             if not descr:
@@ -104,7 +93,7 @@
             operations[0].setdescr(descr)
         inputargs = []
         for box in valueboxes:
-            if isinstance(box, Box) and box not in inputargs:
+            if not isinstance(box, Const) and box not in inputargs:
                 inputargs.append(box)
         return inputargs, operations
 
@@ -413,16 +402,13 @@
         for opnum, boxargs, retvalue in get_int_tests():
             print opnum
             res = self.execute_operation(opnum, boxargs, 'int')
-            assert res.value == retvalue
+            assert res == retvalue
 
     def test_float_operations(self):
         from rpython.jit.metainterp.test.test_executor import get_float_tests
         for opnum, boxargs, rettype, retvalue in get_float_tests(self.cpu):
             res = self.execute_operation(opnum, boxargs, rettype)
-            if isinstance(res, BoxFloat):
-                assert res.getfloat() == retvalue
-            else:
-                assert res.value == retvalue
+            assert res == retvalue
 
     def test_ovf_operations(self, reversed=False):
         minint = -sys.maxint-1
diff --git a/rpython/jit/metainterp/executor.py b/rpython/jit/metainterp/executor.py
--- a/rpython/jit/metainterp/executor.py
+++ b/rpython/jit/metainterp/executor.py
@@ -481,8 +481,8 @@
     return wrap_constant(_execute_nonspec(cpu, metainterp, opnum, argboxes,
                                           descr, type))
 
- at specialize.arg(5)
-def _execute_nonspec(cpu, metainterp, opnum, argboxes, descr=None, type='i'):
+ at specialize.arg(2)
+def _execute_nonspec(cpu, metainterp, opnum, argboxes, descr=None):
     arity = resoperation.oparity[opnum]
     assert arity == -1 or len(argboxes) == arity
     if resoperation.opwithdescr[opnum]:
diff --git a/rpython/jit/metainterp/resoperation.py b/rpython/jit/metainterp/resoperation.py
--- a/rpython/jit/metainterp/resoperation.py
+++ b/rpython/jit/metainterp/resoperation.py
@@ -320,6 +320,7 @@
         return self._resfloat
 
     getvalue = getfloatstorage
+    getfloat = getfloatstorage
 
     def setfloatstorage(self, floatval):
         self._resfloat = floatval
diff --git a/rpython/jit/metainterp/test/test_executor.py b/rpython/jit/metainterp/test/test_executor.py
--- a/rpython/jit/metainterp/test/test_executor.py
+++ b/rpython/jit/metainterp/test/test_executor.py
@@ -1,9 +1,10 @@
 import py
 import sys, random
 from rpython.rlib.rarithmetic import r_uint, intmask
-from rpython.jit.metainterp.executor import execute
+from rpython.jit.metainterp.executor import execute, wrap_constant
 from rpython.jit.metainterp.executor import execute_varargs, _execute_nonspec
-from rpython.jit.metainterp.resoperation import rop, opname, opclasses
+from rpython.jit.metainterp.resoperation import rop, opname, opclasses,\
+     InputArgInt, InputArgFloat, InputArgRef
 from rpython.jit.metainterp.history import ConstInt, ConstPtr, ConstFloat
 from rpython.jit.metainterp.history import AbstractDescr
 from rpython.jit.metainterp import history
@@ -63,7 +64,7 @@
         self.fakestrsetitem = (string, index, newvalue)
 
 def boxfloat(x):
-    return BoxFloat(longlong.getfloatstorage(x))
+    return InputArgFloat(longlong.getfloatstorage(x))
 
 def constfloat(x):
     return ConstFloat(longlong.getfloatstorage(x))
@@ -72,18 +73,18 @@
 def test_execute():
     cpu = FakeCPU()
     descr = FakeDescr()
-    box = execute(cpu, None, rop.INT_ADD, None, BoxInt(40), ConstInt(2))
-    assert box.value == 42
+    box = execute(cpu, None, rop.INT_ADD, None, InputArgInt(40), ConstInt(2))
+    assert box == 42
     box = execute(cpu, None, rop.NEW, descr)
-    assert box.value.fakeargs == ('new', descr)
+    assert box.fakeargs == ('new', descr)
 
 def test_execute_varargs():
     cpu = FakeCPU()
     descr = FakeCallDescr()
-    argboxes = [BoxInt(99999), BoxInt(321), constfloat(2.25), ConstInt(123),
-                BoxPtr(), boxfloat(5.5)]
-    box = execute_varargs(cpu, FakeMetaInterp(), rop.CALL, argboxes, descr)
-    assert box.getfloat() == 42.5
+    argboxes = [InputArgInt(99999), InputArgInt(321), constfloat(2.25), ConstInt(123),
+                InputArgRef(), boxfloat(5.5)]
+    box = execute_varargs(cpu, FakeMetaInterp(), rop.CALL_F, argboxes, descr)
+    assert box == 42.5
     assert cpu.fakecalled == (99999, [321, 123],
                               [ConstPtr.value],
                               [longlong.getfloatstorage(2.25),
@@ -95,39 +96,40 @@
     descr = FakeDescr()
     # cases with a descr
     # arity == -1
-    argboxes = [BoxInt(321), ConstInt(123)]
-    box = execute_nonspec(cpu, FakeMetaInterp(), rop.CALL,
-                          argboxes, FakeCallDescr())
-    assert box.getfloat() == 42.5
+    argboxes = [InputArgInt(321), ConstInt(123)]
+    box = _execute_nonspec(cpu, FakeMetaInterp(), rop.CALL_F,
+                           argboxes, FakeCallDescr())
+    assert box == 42.5
     # arity == 0
-    box = execute_nonspec(cpu, None, rop.NEW, [], descr)
-    assert box.value.fakeargs == ('new', descr)
+    box = _execute_nonspec(cpu, None, rop.NEW, [], descr)
+    assert box.fakeargs == ('new', descr)
     # arity == 1
-    box1 = BoxPtr()
-    box = execute_nonspec(cpu, None, rop.ARRAYLEN_GC, [box1], descr)
-    assert box.value == 55
+    box1 = InputArgRef()
+    box = _execute_nonspec(cpu, None, rop.ARRAYLEN_GC, [box1], descr)
+    assert box == 55
     # arity == 2
     box2 = boxfloat(222.2)
     fielddescr = FakeFieldDescr()
-    execute_nonspec(cpu, None, rop.SETFIELD_GC, [box1, box2], fielddescr)
-    assert cpu.fakesetfield == (box1.value, box2.value, fielddescr)
+    _execute_nonspec(cpu, None, rop.SETFIELD_GC, [box1, box2], fielddescr)
+    assert cpu.fakesetfield == (box1.getref_base(), box2.getfloat(),
+                                fielddescr)
     # arity == 3
-    box3 = BoxInt(33)
+    box3 = InputArgInt(33)
     arraydescr = FakeArrayDescr()
-    execute_nonspec(cpu, None, rop.SETARRAYITEM_GC, [box1, box3, box2],
+    _execute_nonspec(cpu, None, rop.SETARRAYITEM_GC, [box1, box3, box2],
                     arraydescr)
-    assert cpu.fakesetarrayitem == (box1.value, box3.value, box2.value,
-                                    arraydescr)
+    assert cpu.fakesetarrayitem == (box1.getref_base(), box3.getint(),
+                                    box2.getfloat(), arraydescr)
     # cases without descr
     # arity == 1
-    box = execute_nonspec(cpu, None, rop.INT_INVERT, [box3])
-    assert box.value == ~33
+    box = _execute_nonspec(cpu, None, rop.INT_INVERT, [box3])
+    assert box == ~33
     # arity == 2
-    box = execute_nonspec(cpu, None, rop.INT_LSHIFT, [box3, BoxInt(3)])
-    assert box.value == 33 << 3
+    box = _execute_nonspec(cpu, None, rop.INT_LSHIFT, [box3, InputArgInt(3)])
+    assert box == 33 << 3
     # arity == 3
-    execute_nonspec(cpu, None, rop.STRSETITEM, [box1, BoxInt(3), box3])
-    assert cpu.fakestrsetitem == (box1.value, 3, box3.value)
+    _execute_nonspec(cpu, None, rop.STRSETITEM, [box1, InputArgInt(3), box3])
+    assert cpu.fakestrsetitem == (box1.getref_base(), 3, box3.getint())
 
 # ints
 
@@ -230,21 +232,21 @@
             list(_int_binary_operations()) +
             list(_int_comparison_operations()) +
             list(_int_unary_operations())):
-        yield opnum, [BoxInt(x) for x in args], retvalue
+        yield opnum, [InputArgInt(x) for x in args], retvalue
         if len(args) > 1:
             assert len(args) == 2
-            yield opnum, [BoxInt(args[0]), ConstInt(args[1])], retvalue
-            yield opnum, [ConstInt(args[0]), BoxInt(args[1])], retvalue
+            yield opnum, [InputArgInt(args[0]), ConstInt(args[1])], retvalue
+            yield opnum, [ConstInt(args[0]), InputArgInt(args[1])], retvalue
             if args[0] == args[1]:
-                commonbox = BoxInt(args[0])
+                commonbox = InputArgInt(args[0])
                 yield opnum, [commonbox, commonbox], retvalue
 
 
 def test_int_ops():
     cpu = FakeCPU()
     for opnum, boxargs, retvalue in get_int_tests():
-        box = execute_nonspec(cpu, None, opnum, boxargs)
-        assert box.getint() == retvalue
+        r = _execute_nonspec(cpu, None, opnum, boxargs)
+        assert r == retvalue
 
 # floats
 
@@ -296,30 +298,25 @@
         boxargs = []
         for x in args:
             if isinstance(x, float):
-                boxargs.append(boxfloat(x))
+                boxargs.append(InputArgFloat(x))
             else:
-                boxargs.append(BoxInt(x))
+                boxargs.append(InputArgInt(x))
         yield opnum, boxargs, rettype, retvalue
         if len(args) > 1:
             assert len(args) == 2
-            yield opnum, [boxargs[0], boxargs[1].constbox()], rettype, retvalue
-            yield opnum, [boxargs[0].constbox(), boxargs[1]], rettype, retvalue
+            yield opnum, [boxargs[0], wrap_constant(boxargs[1].getvalue())], rettype, retvalue
+            yield opnum, [wrap_constant(boxargs[0].getvalue()), boxargs[1]], rettype, retvalue
             if (isinstance(args[0], float) and
                 isinstance(args[1], float) and
                 args[0] == args[1]):
-                commonbox = boxfloat(args[0])
+                commonbox = InputArgFloat(args[0])
                 yield opnum, [commonbox, commonbox], rettype, retvalue
 
 def test_float_ops():
     cpu = FakeCPU()
     for opnum, boxargs, rettype, retvalue in get_float_tests(cpu):
-        box = execute_nonspec(cpu, None, opnum, boxargs)
-        if rettype == 'float':
-            assert box.getfloat() == retvalue
-        elif rettype == 'int':
-            assert box.getint() == retvalue
-        else:
-            assert 0, "rettype is %r" % (rettype,)
+        res = _execute_nonspec(cpu, None, opnum, boxargs)
+        assert res == retvalue
 
 def make_args_for_op(op, a, b):
     n=opname[op]
@@ -350,7 +347,7 @@
                 arg1, arg2 = make_args_for_op(op1, a, b)
                 box1 = execute(cpu, None, op1, None, arg1, arg2)
                 box2 = execute(cpu, None, op2, None, arg1, arg2)
-                assert box1.value == (not box2.value)
+                assert box1 == (not box2)
 
 def test_opboolreflex():
     cpu = FakeCPU()
@@ -364,4 +361,4 @@
                 arg1, arg2 = make_args_for_op(op1, a, b)
                 box1 = execute(cpu, None, op1, None, arg1, arg2)
                 box2 = execute(cpu, None, op2, None, arg2, arg1)
-                assert box1.value == box2.value
+                assert box1 == box2

From noreply at buildbot.pypy.org  Mon Nov 24 10:39:28 2014
From: noreply at buildbot.pypy.org (fijal)
Date: Mon, 24 Nov 2014 10:39:28 +0100 (CET)
Subject: [pypy-commit] pypy optresult: Port 30% of runner_test
Message-ID: <20141124093928.56CBF1C06AE@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: optresult
Changeset: r74656:abd33e0a45d9
Date: 2014-11-24 11:39 +0200
http://bitbucket.org/pypy/pypy/changeset/abd33e0a45d9/

Log:	Port 30% of runner_test

diff --git a/rpython/jit/backend/llgraph/runner.py b/rpython/jit/backend/llgraph/runner.py
--- a/rpython/jit/backend/llgraph/runner.py
+++ b/rpython/jit/backend/llgraph/runner.py
@@ -1022,8 +1022,10 @@
             result = support.cast_to_floatstorage(result)
         return result
 
-    def execute_same_as(self, _, x):
+    def execute_same_as_i(self, _, x):
         return x
+    execute_same_as_f = execute_same_as_i
+    execute_same_as_r = execute_same_as_i
 
     def execute_debug_merge_point(self, descr, *args):
         from rpython.jit.metainterp.warmspot import get_stats
diff --git a/rpython/jit/backend/test/runner_test.py b/rpython/jit/backend/test/runner_test.py
--- a/rpython/jit/backend/test/runner_test.py
+++ b/rpython/jit/backend/test/runner_test.py
@@ -7,8 +7,9 @@
                                          ConstInt, ConstPtr,
                                          ConstFloat, Const)
 from rpython.jit.metainterp.resoperation import ResOperation, rop, InputArgInt,\
-     InputArgFloat, opname
+     InputArgFloat, opname, InputArgRef
 from rpython.jit.metainterp.typesystem import deref
+from rpython.jit.metainterp.executor import wrap_constant
 from rpython.jit.codewriter.effectinfo import EffectInfo
 from rpython.jit.tool.oparser import parse
 from rpython.rtyper.lltypesystem import lltype, llmemory, rstr, rffi
@@ -28,6 +29,16 @@
 def constfloat(x):
     return ConstFloat(longlong.getfloatstorage(x))
 
+def boxfloat(x):
+    return InputArgFloat(longlong.getfloatstorage(x))
+
+def clone(op):
+    if op.type == 'i':
+        return InputArgInt(op.getint())
+    elif op.type == 'r':
+        return InputArgRef(op.getref_base())
+    xxx
+
 def boxlonglong(ll):
     if longlong.is_64_bit:
         return BoxInt(ll)
@@ -424,28 +435,23 @@
                                (-2, -(minint/2), minint),
                                (minint/2, -2, boom)]),
             ]:
-            v1 = BoxInt(testcases[0][0])
-            v2 = BoxInt(testcases[0][1])
-            v_res = BoxInt()
+            v1 = InputArgInt(testcases[0][0])
+            v2 = InputArgInt(testcases[0][1])
             #
             if not reversed:
-                ops = [
-                    ResOperation(opnum, [v1, v2], v_res),
-                    ResOperation(rop.GUARD_NO_OVERFLOW, [], None,
-                                 descr=BasicFailDescr(1)),
-                    ResOperation(rop.FINISH, [v_res], None,
-                                 descr=BasicFinalDescr(2)),
-                    ]
-                ops[1].setfailargs([])
+                op0 = ResOperation(opnum, [v1, v2])
+                op1 = ResOperation(rop.GUARD_NO_OVERFLOW, [],
+                                   descr=BasicFailDescr(1))
+                op2 = ResOperation(rop.FINISH, [op0], descr=BasicFinalDescr(2))
+                ops = [op0, op1, op2]
+                op1.setfailargs([])
             else:
-                v_exc = self.cpu.ts.BoxRef()
-                ops = [
-                    ResOperation(opnum, [v1, v2], v_res),
-                    ResOperation(rop.GUARD_OVERFLOW, [], None,
-                                 descr=BasicFailDescr(1)),
-                    ResOperation(rop.FINISH, [], None, descr=BasicFinalDescr(2)),
-                    ]
-                ops[1].setfailargs([v_res])
+                op0 = ResOperation(opnum, [v1, v2])
+                op1 = ResOperation(rop.GUARD_OVERFLOW, [],
+                                   descr=BasicFailDescr(1))
+                op2 = ResOperation(rop.FINISH, [], descr=BasicFinalDescr(2))
+                ops = [op0, op1, op2]
+                op1.setfailargs([op0])
             #
             looptoken = JitCellToken()
             self.cpu.compile_loop([v1, v2], ops, looptoken)
@@ -516,24 +522,27 @@
             # first, try it with the "normal" calldescr
             calldescr = cpu.calldescrof(FUNC, FUNC.ARGS, FUNC.RESULT,
                                         EffectInfo.MOST_GENERAL)
-            res = self.execute_operation(rop.CALL,
-                                         [funcbox, BoxInt(num), BoxInt(num)],
+            res = self.execute_operation(rop.CALL_I,
+                                         [funcbox, InputArgInt(num),
+                                          InputArgInt(num)],
                                          'int', descr=calldescr)
-            assert res.value == 2 * num
+            assert res == 2 * num
             # then, try it with the dynamic calldescr
             dyn_calldescr = cpu._calldescr_dynamic_for_tests(
                 [ffi_type, ffi_type], ffi_type)
-            res = self.execute_operation(rop.CALL,
-                                         [funcbox, BoxInt(num), BoxInt(num)],
+            res = self.execute_operation(rop.CALL_I,
+                                         [funcbox, InputArgInt(num),
+                                          InputArgInt(num)],
                                          'int', descr=dyn_calldescr)
-            assert res.value == 2 * num
+            assert res == 2 * num
 
             # last, try it with one constant argument
             calldescr = cpu.calldescrof(FUNC, FUNC.ARGS, FUNC.RESULT, EffectInfo.MOST_GENERAL)
-            res = self.execute_operation(rop.CALL,
-                                         [funcbox, ConstInt(num), BoxInt(num)],
+            res = self.execute_operation(rop.CALL_I,
+                                         [funcbox, ConstInt(num),
+                                          InputArgInt(num)],
                                          'int', descr=calldescr)
-            assert res.value == 2 * num
+            assert res == 2 * num
 
         if cpu.supports_floats:
             def func(f0, f1, f2, f3, f4, f5, f6, i0, f7, i1, f8, f9):
@@ -547,12 +556,12 @@
                                         EffectInfo.MOST_GENERAL)
             funcbox = self.get_funcbox(cpu, func_ptr)
             args = ([boxfloat(.1) for i in range(7)] +
-                    [BoxInt(1), boxfloat(.2), BoxInt(2), boxfloat(.3),
+                    [InputArgInt(1), boxfloat(.2), InputArgInt(2), boxfloat(.3),
                      boxfloat(.4)])
-            res = self.execute_operation(rop.CALL,
+            res = self.execute_operation(rop.CALL_F,
                                          [funcbox] + args,
                                          'float', descr=calldescr)
-            assert abs(res.getfloat() - 4.6) < 0.0001
+            assert abs(res - 4.6) < 0.0001
 
     def test_call_many_arguments(self):
         # Test calling a function with a large number of arguments (more than
@@ -571,8 +580,8 @@
         func_ptr = llhelper(FPTR, func)
         args = range(16)
         funcbox = self.get_funcbox(self.cpu, func_ptr)
-        res = self.execute_operation(rop.CALL, [funcbox] + map(BoxInt, args), 'int', descr=calldescr)
-        assert res.value == func(*args)
+        res = self.execute_operation(rop.CALL_I, [funcbox] + map(InputArgInt, args), 'int', descr=calldescr)
+        assert res == func(*args)
 
     def test_call_box_func(self):
         def a(a1, a2):
@@ -588,13 +597,14 @@
             func_ptr = llhelper(FPTR, f)
             FUNC = deref(FPTR)
             funcconst = self.get_funcbox(self.cpu, func_ptr)
-            funcbox = funcconst.clonebox()
+            funcbox = InputArgInt(funcconst.getint())
             calldescr = self.cpu.calldescrof(FUNC, FUNC.ARGS, FUNC.RESULT,
                                         EffectInfo.MOST_GENERAL)
-            res = self.execute_operation(rop.CALL,
-                                         [funcbox, BoxInt(arg1), BoxInt(arg2)],
+            res = self.execute_operation(rop.CALL_I,
+                                         [funcbox, InputArgInt(arg1),
+                                          InputArgInt(arg2)],
                                          'int', descr=calldescr)
-            assert res.getint() == f(arg1, arg2)
+            assert res == f(arg1, arg2)
 
     def test_call_stack_alignment(self):
         # test stack alignment issues, notably for Mac OS/X.
@@ -616,10 +626,10 @@
                                         EffectInfo.MOST_GENERAL)
             funcbox = self.get_funcbox(cpu, func_ptr)
             args = [280-24*i for i in range(nb_args)]
-            res = self.execute_operation(rop.CALL,
-                                         [funcbox] + map(BoxInt, args),
+            res = self.execute_operation(rop.CALL_I,
+                                         [funcbox] + map(InputArgInt, args),
                                          'int', descr=calldescr)
-            assert res.value == func_ints(*args)
+            assert res == func_ints(*args)
 
     def test_call_with_const_floats(self):
         if not self.cpu.supports_floats:
@@ -633,10 +643,10 @@
                                          EffectInfo.MOST_GENERAL)
         func_ptr = llhelper(FPTR, func)
         funcbox = self.get_funcbox(self.cpu, func_ptr)
-        res = self.execute_operation(rop.CALL, [funcbox, constfloat(1.5),
+        res = self.execute_operation(rop.CALL_F, [funcbox, constfloat(1.5),
                                                 constfloat(2.5)], 'float',
                                      descr=calldescr)
-        assert res.getfloat() == 4.0
+        assert res == 4.0
 
 
     def test_field_basic(self):
@@ -644,31 +654,31 @@
         fielddescr = self.cpu.fielddescrof(self.S, 'value')
         assert not fielddescr.is_pointer_field()
         #
-        res = self.execute_operation(rop.SETFIELD_GC, [t_box, BoxInt(39082)],
+        res = self.execute_operation(rop.SETFIELD_GC, [t_box, InputArgInt(39082)],
                                      'void', descr=fielddescr)
         assert res is None
-        res = self.execute_operation(rop.GETFIELD_GC, [t_box],
+        res = self.execute_operation(rop.GETFIELD_GC_I, [t_box],
                                      'int', descr=fielddescr)
-        assert res.value == 39082
+        assert res == 39082
         #
         fielddescr1 = self.cpu.fielddescrof(self.S, 'chr1')
         fielddescr2 = self.cpu.fielddescrof(self.S, 'chr2')
         shortdescr = self.cpu.fielddescrof(self.S, 'short')
-        self.execute_operation(rop.SETFIELD_GC, [t_box, BoxInt(250)],
+        self.execute_operation(rop.SETFIELD_GC, [t_box, InputArgInt(250)],
                                'void', descr=fielddescr2)
-        self.execute_operation(rop.SETFIELD_GC, [t_box, BoxInt(133)],
+        self.execute_operation(rop.SETFIELD_GC, [t_box, InputArgInt(133)],
                                'void', descr=fielddescr1)
-        self.execute_operation(rop.SETFIELD_GC, [t_box, BoxInt(1331)],
+        self.execute_operation(rop.SETFIELD_GC, [t_box, InputArgInt(1331)],
                                'void', descr=shortdescr)
-        res = self.execute_operation(rop.GETFIELD_GC, [t_box],
+        res = self.execute_operation(rop.GETFIELD_GC_I, [t_box],
                                      'int', descr=fielddescr2)
-        assert res.value == 250
-        res = self.execute_operation(rop.GETFIELD_GC, [t_box],
+        assert res == 250
+        res = self.execute_operation(rop.GETFIELD_GC_I, [t_box],
                                      'int', descr=fielddescr1)
-        assert res.value == 133
-        res = self.execute_operation(rop.GETFIELD_GC, [t_box],
+        assert res == 133
+        res = self.execute_operation(rop.GETFIELD_GC_I, [t_box],
                                      'int', descr=shortdescr)
-        assert res.value == 1331
+        assert res == 1331
 
         #
         u_box, U_box = self.alloc_instance(self.U)
@@ -677,38 +687,38 @@
         res = self.execute_operation(rop.SETFIELD_GC, [t_box, u_box],
                                      'void', descr=fielddescr2)
         assert res is None
-        res = self.execute_operation(rop.GETFIELD_GC, [t_box],
+        res = self.execute_operation(rop.GETFIELD_GC_R, [t_box],
                                      'ref', descr=fielddescr2)
-        assert res.value == u_box.value
+        assert res == u_box.getref_base()
         #
-        null_const = self.null_instance().constbox()
+        null_const = wrap_constant(self.null_instance().getref_base())
         res = self.execute_operation(rop.SETFIELD_GC, [t_box, null_const],
                                      'void', descr=fielddescr2)
         assert res is None
-        res = self.execute_operation(rop.GETFIELD_GC, [t_box],
+        res = self.execute_operation(rop.GETFIELD_GC_R, [t_box],
                                      'ref', descr=fielddescr2)
-        assert res.value == null_const.value
+        assert not res
         if self.cpu.supports_floats:
             floatdescr = self.cpu.fielddescrof(self.S, 'float')
             self.execute_operation(rop.SETFIELD_GC, [t_box, boxfloat(3.4)],
                                    'void', descr=floatdescr)
-            res = self.execute_operation(rop.GETFIELD_GC, [t_box],
+            res = self.execute_operation(rop.GETFIELD_GC_F, [t_box],
                                          'float', descr=floatdescr)
-            assert res.getfloat() == 3.4
+            assert res == 3.4
             #
             self.execute_operation(rop.SETFIELD_GC, [t_box, constfloat(-3.6)],
                                    'void', descr=floatdescr)
-            res = self.execute_operation(rop.GETFIELD_GC, [t_box],
+            res = self.execute_operation(rop.GETFIELD_GC_F, [t_box],
                                          'float', descr=floatdescr)
-            assert res.getfloat() == -3.6
+            assert res == -3.6
 
 
     def test_passing_guards(self):
         t_box, T_box = self.alloc_instance(self.T)
         nullbox = self.null_instance()
-        all = [(rop.GUARD_TRUE, [BoxInt(1)]),
-               (rop.GUARD_FALSE, [BoxInt(0)]),
-               (rop.GUARD_VALUE, [BoxInt(42), ConstInt(42)]),
+        all = [(rop.GUARD_TRUE, [InputArgInt(1)]),
+               (rop.GUARD_FALSE, [InputArgInt(0)]),
+               (rop.GUARD_VALUE, [InputArgInt(42), ConstInt(42)]),
                ]
         if not self.avoid_instances:
             all.extend([
@@ -733,9 +743,9 @@
     def test_failing_guards(self):
         t_box, T_box = self.alloc_instance(self.T)
         nullbox = self.null_instance()
-        all = [(rop.GUARD_TRUE, [BoxInt(0)]),
-               (rop.GUARD_FALSE, [BoxInt(1)]),
-               (rop.GUARD_VALUE, [BoxInt(42), ConstInt(41)]),
+        all = [(rop.GUARD_TRUE, [InputArgInt(0)]),
+               (rop.GUARD_FALSE, [InputArgInt(1)]),
+               (rop.GUARD_VALUE, [InputArgInt(42), ConstInt(41)]),
                ]
         if not self.avoid_instances:
             all.extend([
@@ -764,38 +774,38 @@
         u1_box, U_box = self.alloc_instance(self.U)
         u2_box, U_box = self.alloc_instance(self.U)
         r = self.execute_operation(rop.PTR_EQ, [u1_box,
-                                                u1_box.clonebox()], 'int')
-        assert r.value == 1
+                                                clone(u1_box)], 'int')
+        assert r == 1
         r = self.execute_operation(rop.PTR_NE, [u2_box,
-                                                u2_box.clonebox()], 'int')
-        assert r.value == 0
+                                                clone(u2_box)], 'int')
+        assert r == 0
         r = self.execute_operation(rop.PTR_EQ, [u1_box, u2_box], 'int')
-        assert r.value == 0
+        assert r == 0
         r = self.execute_operation(rop.PTR_NE, [u2_box, u1_box], 'int')
-        assert r.value == 1
+        assert r == 1
         #
         null_box = self.null_instance()
         r = self.execute_operation(rop.PTR_EQ, [null_box,
-                                                null_box.clonebox()], 'int')
-        assert r.value == 1
+                                                clone(null_box)], 'int')
+        assert r == 1
         r = self.execute_operation(rop.PTR_EQ, [u1_box, null_box], 'int')
-        assert r.value == 0
+        assert r == 0
         r = self.execute_operation(rop.PTR_EQ, [null_box, u2_box], 'int')
-        assert r.value == 0
+        assert r == 0
         r = self.execute_operation(rop.PTR_NE, [null_box,
-                                                null_box.clonebox()], 'int')
-        assert r.value == 0
+                                                clone(null_box)], 'int')
+        assert r == 0
         r = self.execute_operation(rop.PTR_NE, [u2_box, null_box], 'int')
-        assert r.value == 1
+        assert r == 1
         r = self.execute_operation(rop.PTR_NE, [null_box, u1_box], 'int')
-        assert r.value == 1
+        assert r == 1
 
         # These operations are supposed to be the same as PTR_EQ/PTR_NE
         # just checking that the operations are defined in the backend.
         r = self.execute_operation(rop.INSTANCE_PTR_EQ, [u1_box, u2_box], 'int')
-        assert r.value == 0
+        assert r == 0
         r = self.execute_operation(rop.INSTANCE_PTR_NE, [u2_box, u1_box], 'int')
-        assert r.value == 1
+        assert r == 1
 
     def test_array_basic(self):
         a_box, A = self.alloc_array_of(rffi.SHORT, 342)
@@ -804,14 +814,14 @@
         #
         r = self.execute_operation(rop.ARRAYLEN_GC, [a_box],
                                    'int', descr=arraydescr)
-        assert r.value == 342
-        r = self.execute_operation(rop.SETARRAYITEM_GC, [a_box, BoxInt(310),
-                                                         BoxInt(744)],
+        assert r == 342
+        r = self.execute_operation(rop.SETARRAYITEM_GC, [a_box, InputArgInt(310),
+                                                         InputArgInt(744)],
                                    'void', descr=arraydescr)
         assert r is None
-        r = self.execute_operation(rop.GETARRAYITEM_GC, [a_box, BoxInt(310)],
+        r = self.execute_operation(rop.GETARRAYITEM_GC_I, [a_box, InputArgInt(310)],
                                    'int', descr=arraydescr)
-        assert r.value == 744
+        assert r == 744
 
         a_box, A = self.alloc_array_of(lltype.Signed, 342)
         arraydescr = self.cpu.arraydescrof(A)
@@ -819,35 +829,35 @@
         #
         r = self.execute_operation(rop.ARRAYLEN_GC, [a_box],
                                    'int', descr=arraydescr)
-        assert r.value == 342
-        r = self.execute_operation(rop.SETARRAYITEM_GC, [a_box, BoxInt(310),
-                                                         BoxInt(7441)],
+        assert r == 342
+        r = self.execute_operation(rop.SETARRAYITEM_GC, [a_box, InputArgInt(310),
+                                                         InputArgInt(7441)],
                                    'void', descr=arraydescr)
         assert r is None
-        r = self.execute_operation(rop.GETARRAYITEM_GC, [a_box, BoxInt(310)],
+        r = self.execute_operation(rop.GETARRAYITEM_GC_I, [a_box, InputArgInt(310)],
                                    'int', descr=arraydescr)
-        assert r.value == 7441
+        assert r == 7441
         #
         a_box, A = self.alloc_array_of(lltype.Char, 11)
         arraydescr = self.cpu.arraydescrof(A)
         assert not arraydescr.is_array_of_pointers()
         r = self.execute_operation(rop.ARRAYLEN_GC, [a_box],
                                    'int', descr=arraydescr)
-        assert r.value == 11
-        r = self.execute_operation(rop.SETARRAYITEM_GC, [a_box, BoxInt(4),
-                                                         BoxInt(150)],
+        assert r == 11
+        r = self.execute_operation(rop.SETARRAYITEM_GC, [a_box, InputArgInt(4),
+                                                         InputArgInt(150)],
                                    'void', descr=arraydescr)
         assert r is None
-        r = self.execute_operation(rop.SETARRAYITEM_GC, [a_box, BoxInt(3),
-                                                         BoxInt(160)],
+        r = self.execute_operation(rop.SETARRAYITEM_GC, [a_box, InputArgInt(3),
+                                                         InputArgInt(160)],
                                    'void', descr=arraydescr)
         assert r is None
-        r = self.execute_operation(rop.GETARRAYITEM_GC, [a_box, BoxInt(4)],
+        r = self.execute_operation(rop.GETARRAYITEM_GC_I, [a_box, InputArgInt(4)],
                                    'int', descr=arraydescr)
-        assert r.value == 150
-        r = self.execute_operation(rop.GETARRAYITEM_GC, [a_box, BoxInt(3)],
+        assert r == 150
+        r = self.execute_operation(rop.GETARRAYITEM_GC_I, [a_box, InputArgInt(3)],
                                    'int', descr=arraydescr)
-        assert r.value == 160
+        assert r == 160
 
         #
         if isinstance(A, lltype.GcArray):
@@ -857,14 +867,14 @@
         assert arraydescr.is_array_of_pointers()
         r = self.execute_operation(rop.ARRAYLEN_GC, [b_box],
                                    'int', descr=arraydescr)
-        assert r.value == 3
-        r = self.execute_operation(rop.SETARRAYITEM_GC, [b_box, BoxInt(1),
+        assert r == 3
+        r = self.execute_operation(rop.SETARRAYITEM_GC, [b_box, InputArgInt(1),
                                                          a_box],
                                    'void', descr=arraydescr)
         assert r is None
-        r = self.execute_operation(rop.GETARRAYITEM_GC, [b_box, BoxInt(1)],
+        r = self.execute_operation(rop.GETARRAYITEM_GC_R, [b_box, InputArgInt(1)],
                                    'ref', descr=arraydescr)
-        assert r.value == a_box.value
+        assert r == a_box.getvalue()
         #
         # Unsigned should work the same as Signed
         a_box, A = self.alloc_array_of(lltype.Unsigned, 342)
@@ -872,14 +882,14 @@
         assert not arraydescr.is_array_of_pointers()
         r = self.execute_operation(rop.ARRAYLEN_GC, [a_box],
                                    'int', descr=arraydescr)
-        assert r.value == 342
-        r = self.execute_operation(rop.SETARRAYITEM_GC, [a_box, BoxInt(310),
-                                                         BoxInt(7441)],
+        assert r == 342
+        r = self.execute_operation(rop.SETARRAYITEM_GC, [a_box, InputArgInt(310),
+                                                         InputArgInt(7441)],
                                    'void', descr=arraydescr)
         assert r is None
-        r = self.execute_operation(rop.GETARRAYITEM_GC, [a_box, BoxInt(310)],
+        r = self.execute_operation(rop.GETARRAYITEM_GC_I, [a_box, InputArgInt(310)],
                                    'int', descr=arraydescr)
-        assert r.value == 7441
+        assert r == 7441
         #
         # Bool should work the same as Char
         a_box, A = self.alloc_array_of(lltype.Bool, 311)
@@ -887,44 +897,44 @@
         assert not arraydescr.is_array_of_pointers()
         r = self.execute_operation(rop.ARRAYLEN_GC, [a_box],
                                    'int', descr=arraydescr)
-        assert r.value == 311
-        r = self.execute_operation(rop.SETARRAYITEM_GC, [a_box, BoxInt(304),
-                                                         BoxInt(1)],
+        assert r == 311
+        r = self.execute_operation(rop.SETARRAYITEM_GC, [a_box, InputArgInt(304),
+                                                         InputArgInt(1)],
                                    'void', descr=arraydescr)
         assert r is None
-        r = self.execute_operation(rop.SETARRAYITEM_GC, [a_box, BoxInt(303),
-                                                         BoxInt(0)],
+        r = self.execute_operation(rop.SETARRAYITEM_GC, [a_box, InputArgInt(303),
+                                                         InputArgInt(0)],
                                    'void', descr=arraydescr)
         assert r is None
-        r = self.execute_operation(rop.SETARRAYITEM_GC, [a_box, BoxInt(302),
-                                                         BoxInt(1)],
+        r = self.execute_operation(rop.SETARRAYITEM_GC, [a_box, InputArgInt(302),
+                                                         InputArgInt(1)],
                                    'void', descr=arraydescr)
         assert r is None
-        r = self.execute_operation(rop.GETARRAYITEM_GC, [a_box, BoxInt(304)],
+        r = self.execute_operation(rop.GETARRAYITEM_GC_I, [a_box, InputArgInt(304)],
                                    'int', descr=arraydescr)
-        assert r.value == 1
-        r = self.execute_operation(rop.GETARRAYITEM_GC, [a_box, BoxInt(303)],
+        assert r == 1
+        r = self.execute_operation(rop.GETARRAYITEM_GC_I, [a_box, InputArgInt(303)],
                                    'int', descr=arraydescr)
-        assert r.value == 0
-        r = self.execute_operation(rop.GETARRAYITEM_GC, [a_box, BoxInt(302)],
+        assert r == 0
+        r = self.execute_operation(rop.GETARRAYITEM_GC_I, [a_box, InputArgInt(302)],
                                    'int', descr=arraydescr)
-        assert r.value == 1
+        assert r == 1
 
         if self.cpu.supports_floats:
             a_box, A = self.alloc_array_of(lltype.Float, 31)
             arraydescr = self.cpu.arraydescrof(A)
-            self.execute_operation(rop.SETARRAYITEM_GC, [a_box, BoxInt(1),
+            self.execute_operation(rop.SETARRAYITEM_GC, [a_box, InputArgInt(1),
                                                          boxfloat(3.5)],
                                    'void', descr=arraydescr)
-            self.execute_operation(rop.SETARRAYITEM_GC, [a_box, BoxInt(2),
+            self.execute_operation(rop.SETARRAYITEM_GC, [a_box, InputArgInt(2),
                                                          constfloat(4.5)],
                                    'void', descr=arraydescr)
-            r = self.execute_operation(rop.GETARRAYITEM_GC, [a_box, BoxInt(1)],
+            r = self.execute_operation(rop.GETARRAYITEM_GC_F, [a_box, InputArgInt(1)],
                                        'float', descr=arraydescr)
-            assert r.getfloat() == 3.5
-            r = self.execute_operation(rop.GETARRAYITEM_GC, [a_box, BoxInt(2)],
+            assert r == 3.5
+            r = self.execute_operation(rop.GETARRAYITEM_GC_F, [a_box, InputArgInt(2)],
                                        'float', descr=arraydescr)
-            assert r.getfloat() == 4.5
+            assert r == 4.5
 
         # For platforms where sizeof(INT) != sizeof(Signed) (ie, x86-64)
         a_box, A = self.alloc_array_of(rffi.INT, 342)
@@ -932,14 +942,14 @@
         assert not arraydescr.is_array_of_pointers()
         r = self.execute_operation(rop.ARRAYLEN_GC, [a_box],
                                    'int', descr=arraydescr)
-        assert r.value == 342
-        r = self.execute_operation(rop.SETARRAYITEM_GC, [a_box, BoxInt(310),
-                                                         BoxInt(7441)],
+        assert r == 342
+        r = self.execute_operation(rop.SETARRAYITEM_GC, [a_box, InputArgInt(310),
+                                                         InputArgInt(7441)],
                                    'void', descr=arraydescr)
         assert r is None
-        r = self.execute_operation(rop.GETARRAYITEM_GC, [a_box, BoxInt(310)],
+        r = self.execute_operation(rop.GETARRAYITEM_GC_I, [a_box, InputArgInt(310)],
                                    'int', descr=arraydescr)
-        assert r.value == 7441
+        assert r == 7441
 
     def test_array_of_structs(self):
         TP = lltype.GcStruct('x')
@@ -959,15 +969,15 @@
         vsdescr = self.cpu.interiorfielddescrof(A, 'vs')
         kdescr = self.cpu.interiorfielddescrof(A, 'k')
         pdescr = self.cpu.interiorfielddescrof(A, 'p')
-        self.execute_operation(rop.SETINTERIORFIELD_GC, [a_box, BoxInt(3),
+        self.execute_operation(rop.SETINTERIORFIELD_GC, [a_box, InputArgInt(3),
                                                          boxfloat(1.5)],
                                'void', descr=kdescr)
         f = self.cpu.bh_getinteriorfield_gc_f(a_box.getref_base(), 3, kdescr)
         assert longlong.getrealfloat(f) == 1.5
         self.cpu.bh_setinteriorfield_gc_f(a_box.getref_base(), 3, longlong.getfloatstorage(2.5), kdescr)
-        r = self.execute_operation(rop.GETINTERIORFIELD_GC, [a_box, BoxInt(3)],
+        r = self.execute_operation(rop.GETINTERIORFIELD_GC_F, [a_box, InputArgInt(3)],
                                    'float', descr=kdescr)
-        assert r.getfloat() == 2.5
+        assert r == 2.5
         #
         NUMBER_FIELDS = [('vs', lltype.Signed),
                          ('vu', lltype.Unsigned),
@@ -979,8 +989,8 @@
                          ('vui', rffi.UINT)]
         for name, TYPE in NUMBER_FIELDS[::-1]:
             vdescr = self.cpu.interiorfielddescrof(A, name)
-            self.execute_operation(rop.SETINTERIORFIELD_GC, [a_box, BoxInt(3),
-                                                             BoxInt(-15)],
+            self.execute_operation(rop.SETINTERIORFIELD_GC, [a_box, InputArgInt(3),
+                                                             InputArgInt(-15)],
                                    'void', descr=vdescr)
         for name, TYPE in NUMBER_FIELDS:
             vdescr = self.cpu.interiorfielddescrof(A, name)
@@ -993,24 +1003,24 @@
                                               -25, vdescr)
         for name, TYPE in NUMBER_FIELDS:
             vdescr = self.cpu.interiorfielddescrof(A, name)
-            r = self.execute_operation(rop.GETINTERIORFIELD_GC,
-                                       [a_box, BoxInt(3)],
+            r = self.execute_operation(rop.GETINTERIORFIELD_GC_I,
+                                       [a_box, InputArgInt(3)],
                                        'int', descr=vdescr)
-            assert r.getint() == rffi.cast(lltype.Signed, rffi.cast(TYPE, -25))
+            assert r == rffi.cast(lltype.Signed, rffi.cast(TYPE, -25))
         #
-        self.execute_operation(rop.SETINTERIORFIELD_GC, [a_box, BoxInt(4),
+        self.execute_operation(rop.SETINTERIORFIELD_GC, [a_box, InputArgInt(4),
                                                          s_box],
                                'void', descr=pdescr)
         r = self.cpu.bh_getinteriorfield_gc_r(a_box.getref_base(), 4, pdescr)
         assert r == s_box.getref_base()
         self.cpu.bh_setinteriorfield_gc_r(a_box.getref_base(), 3,
                                           s_box.getref_base(), pdescr)
-        r = self.execute_operation(rop.GETINTERIORFIELD_GC, [a_box, BoxInt(3)],
+        r = self.execute_operation(rop.GETINTERIORFIELD_GC_R, [a_box, InputArgInt(3)],
                                    'ref', descr=pdescr)
-        assert r.getref_base() == s_box.getref_base()
+        assert r == s_box.getref_base()
         #
         # test a corner case that used to fail on x86
-        i4 = BoxInt(4)
+        i4 = InputArgInt(4)
         self.execute_operation(rop.SETINTERIORFIELD_GC, [a_box, i4, i4],
                                'void', descr=vsdescr)
         r = self.cpu.bh_getinteriorfield_gc_i(a_box.getref_base(), 4, vsdescr)
@@ -1022,17 +1032,16 @@
         for length in range(1, 75):
             ITEM = lltype.FixedSizeArray(lltype.Char, length)
             a_box, A = self.alloc_array_of(ITEM, 5)
-            a = a_box.getref(lltype.Ptr(A))
+            a = lltype.cast_opaque_ptr(lltype.Ptr(A), a_box.getref_base())
             middle = length // 2
             a[3][middle] = chr(65 + length)
             fdescr = self.cpu.interiorfielddescrof(A, 'item%d' % middle)
-            r = self.execute_operation(rop.GETINTERIORFIELD_GC,
-                                       [a_box, BoxInt(3)],
+            r = self.execute_operation(rop.GETINTERIORFIELD_GC_I,
+                                       [a_box, InputArgInt(3)],
                                        'int', descr=fdescr)
-            r = r.getint()
             assert r == 65 + length
             self.execute_operation(rop.SETINTERIORFIELD_GC,
-                                   [a_box, BoxInt(2), BoxInt(r + 1)],
+                                   [a_box, InputArgInt(2), InputArgInt(r + 1)],
                                    'void', descr=fdescr)
             r1 = self.cpu.bh_getinteriorfield_gc_i(a_box.getref_base(), 2,
                                                   fdescr)
@@ -1041,27 +1050,27 @@
     def test_string_basic(self):
         s_box = self.alloc_string("hello\xfe")
         r = self.execute_operation(rop.STRLEN, [s_box], 'int')
-        assert r.value == 6
-        r = self.execute_operation(rop.STRGETITEM, [s_box, BoxInt(5)], 'int')
-        assert r.value == 254
-        r = self.execute_operation(rop.STRSETITEM, [s_box, BoxInt(4),
-                                                    BoxInt(153)], 'void')
+        assert r == 6
+        r = self.execute_operation(rop.STRGETITEM, [s_box, InputArgInt(5)], 'int')
+        assert r == 254
+        r = self.execute_operation(rop.STRSETITEM, [s_box, InputArgInt(4),
+                                                    InputArgInt(153)], 'void')
         assert r is None
-        r = self.execute_operation(rop.STRGETITEM, [s_box, BoxInt(5)], 'int')
-        assert r.value == 254
-        r = self.execute_operation(rop.STRGETITEM, [s_box, BoxInt(4)], 'int')
-        assert r.value == 153
+        r = self.execute_operation(rop.STRGETITEM, [s_box, InputArgInt(5)], 'int')
+        assert r == 254
+        r = self.execute_operation(rop.STRGETITEM, [s_box, InputArgInt(4)], 'int')
+        assert r == 153
 
     def test_copystrcontent(self):
         s_box = self.alloc_string("abcdef")
-        for s_box in [s_box, s_box.constbox()]:
-            for srcstart_box in [BoxInt(2), ConstInt(2)]:
-                for dststart_box in [BoxInt(3), ConstInt(3)]:
-                    for length_box in [BoxInt(4), ConstInt(4)]:
+        for s_box in [s_box, wrap_constant(s_box.getref_base())]:
+            for srcstart_box in [InputArgInt(2), ConstInt(2)]:
+                for dststart_box in [InputArgInt(3), ConstInt(3)]:
+                    for length_box in [InputArgInt(4), ConstInt(4)]:
                         for r_box_is_const in [False, True]:
                             r_box = self.alloc_string("!???????!")
                             if r_box_is_const:
-                                r_box = r_box.constbox()
+                                r_box = wrap_constant(r_box.getref_base())
                                 self.execute_operation(rop.COPYSTRCONTENT,
                                                        [s_box, r_box,
                                                         srcstart_box,
@@ -1071,14 +1080,14 @@
 
     def test_copyunicodecontent(self):
         s_box = self.alloc_unicode(u"abcdef")
-        for s_box in [s_box, s_box.constbox()]:
-            for srcstart_box in [BoxInt(2), ConstInt(2)]:
-                for dststart_box in [BoxInt(3), ConstInt(3)]:
-                    for length_box in [BoxInt(4), ConstInt(4)]:
+        for s_box in [s_box, wrap_constant(s_box.getref_base())]:
+            for srcstart_box in [InputArgInt(2), ConstInt(2)]:
+                for dststart_box in [InputArgInt(3), ConstInt(3)]:
+                    for length_box in [InputArgInt(4), ConstInt(4)]:
                         for r_box_is_const in [False, True]:
                             r_box = self.alloc_unicode(u"!???????!")
                             if r_box_is_const:
-                                r_box = r_box.constbox()
+                                r_box = wrap_constant(r_box.getref_base())
                                 self.execute_operation(rop.COPYUNICODECONTENT,
                                                        [s_box, r_box,
                                                         srcstart_box,
@@ -1095,36 +1104,36 @@
     def test_unicode_basic(self):
         u_box = self.alloc_unicode(u"hello\u1234")
         r = self.execute_operation(rop.UNICODELEN, [u_box], 'int')
-        assert r.value == 6
-        r = self.execute_operation(rop.UNICODEGETITEM, [u_box, BoxInt(5)],
+        assert r == 6
+        r = self.execute_operation(rop.UNICODEGETITEM, [u_box, InputArgInt(5)],
                                    'int')
-        assert r.value == 0x1234
-        r = self.execute_operation(rop.UNICODESETITEM, [u_box, BoxInt(4),
-                                                        BoxInt(31313)], 'void')
+        assert r == 0x1234
+        r = self.execute_operation(rop.UNICODESETITEM, [u_box, InputArgInt(4),
+                                                        InputArgInt(31313)], 'void')
         assert r is None
-        r = self.execute_operation(rop.UNICODEGETITEM, [u_box, BoxInt(5)],
+        r = self.execute_operation(rop.UNICODEGETITEM, [u_box, InputArgInt(5)],
                                    'int')
-        assert r.value == 0x1234
-        r = self.execute_operation(rop.UNICODEGETITEM, [u_box, BoxInt(4)],
+        assert r == 0x1234
+        r = self.execute_operation(rop.UNICODEGETITEM, [u_box, InputArgInt(4)],
                                    'int')
-        assert r.value == 31313
+        assert r == 31313
 
     def test_same_as(self):
-        r = self.execute_operation(rop.SAME_AS, [ConstInt(5)], 'int')
-        assert r.value == 5
-        r = self.execute_operation(rop.SAME_AS, [BoxInt(5)], 'int')
-        assert r.value == 5
+        r = self.execute_operation(rop.SAME_AS_I, [ConstInt(5)], 'int')
+        assert r == 5
+        r = self.execute_operation(rop.SAME_AS_I, [InputArgInt(5)], 'int')
+        assert r == 5
         u_box = self.alloc_unicode(u"hello\u1234")
-        r = self.execute_operation(rop.SAME_AS, [u_box.constbox()], 'ref')
-        assert r.value == u_box.value
-        r = self.execute_operation(rop.SAME_AS, [u_box], 'ref')
-        assert r.value == u_box.value
+        r = self.execute_operation(rop.SAME_AS_R, [wrap_constant(u_box.getref_base())], 'ref')
+        assert r == u_box.getref_base()
+        r = self.execute_operation(rop.SAME_AS_R, [u_box], 'ref')
+        assert r == u_box.getref_base()
 
         if self.cpu.supports_floats:
-            r = self.execute_operation(rop.SAME_AS, [constfloat(5.5)], 'float')
-            assert r.getfloat() == 5.5
-            r = self.execute_operation(rop.SAME_AS, [boxfloat(5.5)], 'float')
-            assert r.getfloat() == 5.5
+            r = self.execute_operation(rop.SAME_AS_F, [constfloat(5.5)], 'float')
+            assert r == 5.5
+            r = self.execute_operation(rop.SAME_AS_F, [boxfloat(5.5)], 'float')
+            assert r == 5.5
 
     def test_virtual_ref(self):
         pass   # VIRTUAL_REF must not reach the backend nowadays
@@ -1151,10 +1160,10 @@
             for k in range(nb_args):
                 kind = r.randrange(0, numkinds)
                 if kind == 0:
-                    inputargs.append(BoxInt())
+                    inputargs.append(InputArgInt())
                     values.append(r.randrange(-100000, 100000))
                 else:
-                    inputargs.append(BoxFloat())
+                    inputargs.append(InputArgFloat())
                     values.append(longlong.getfloatstorage(r.random()))
             #
             looptoken = JitCellToken()
@@ -1167,33 +1176,29 @@
             ks = range(nb_args)
             random.shuffle(ks)
             for k in ks:
-                if isinstance(inputargs[k], BoxInt):
-                    newbox = BoxInt()
+                if isinstance(inputargs[k], InputArgInt):
                     x = r.randrange(-100000, 100000)
                     operations.append(
                         ResOperation(rop.INT_ADD, [inputargs[k],
-                                                   ConstInt(x)], newbox)
+                                                   ConstInt(x)])
                         )
                     y = values[k] + x
                 else:
-                    newbox = BoxFloat()
                     x = r.random()
                     operations.append(
                         ResOperation(rop.FLOAT_ADD, [inputargs[k],
-                                                     constfloat(x)], newbox)
+                                                     constfloat(x)])
                         )
                     y = longlong.getrealfloat(values[k]) + x
                     y = longlong.getfloatstorage(y)
                 kk = r.randrange(0, len(retboxes)+1)
-                retboxes.insert(kk, newbox)
+                retboxes.insert(kk, operations[-1])
                 retvalues.insert(kk, y)
             #
-            zero = BoxInt()
-            operations.extend([
-                ResOperation(rop.SAME_AS, [ConstInt(0)], zero),
-                ResOperation(rop.GUARD_TRUE, [zero], None, descr=guarddescr),
-                ResOperation(rop.FINISH, [], None, descr=faildescr)
-                ])
+            op0 = ResOperation(rop.SAME_AS_I, [ConstInt(0)])
+            op1 = ResOperation(rop.GUARD_TRUE, [op0], descr=guarddescr)
+            op2 = ResOperation(rop.FINISH, [], descr=faildescr)
+            operations += [op0, op1, op2]
             operations[-2].setfailargs(retboxes)
             print inputargs
             for op in operations:
@@ -1205,7 +1210,7 @@
             assert fail.identifier == 42
             #
             for k in range(len(retvalues)):
-                if isinstance(retboxes[k], BoxInt):
+                if retboxes[k].type == 'i':
                     got = self.cpu.get_int_value(deadframe, k)
                 else:
                     got = self.cpu.get_float_value(deadframe, k)
@@ -1229,11 +1234,11 @@
             for k in range(nb_args):
                 kind = r.randrange(0, numkinds)
                 if kind == 0:
-                    inputargs.append(BoxInt())
+                    inputargs.append(InputArgInt())
                 elif kind == 1:
-                    inputargs.append(BoxPtr())
+                    inputargs.append(InputArgRef())
                 else:
-                    inputargs.append(BoxFloat())
+                    inputargs.append(InputArgFloat())
             jumpargs = []
             remixing = []
             for srcbox in inputargs:
@@ -1246,22 +1251,19 @@
                 jumpargs.append(otherbox)
             #
             index_counter = r.randrange(0, len(inputargs)+1)
-            i0 = BoxInt()
-            i1 = BoxInt()
-            i2 = BoxInt()
+            i0 = InputArgInt()
             inputargs.insert(index_counter, i0)
-            jumpargs.insert(index_counter, i1)
             #
             looptoken = JitCellToken()
             targettoken = TargetToken()
             faildescr = BasicFailDescr(15)
-            operations = [
-                ResOperation(rop.LABEL, inputargs, None, descr=targettoken),
-                ResOperation(rop.INT_SUB, [i0, ConstInt(1)], i1),
-                ResOperation(rop.INT_GE, [i1, ConstInt(0)], i2),
-                ResOperation(rop.GUARD_TRUE, [i2], None),
-                ResOperation(rop.JUMP, jumpargs, None, descr=targettoken),
-                ]
+            op0 = ResOperation(rop.LABEL, inputargs, descr=targettoken)
+            op1 = ResOperation(rop.INT_SUB, [i0, ConstInt(1)])
+            op2 = ResOperation(rop.INT_GE, [op1, ConstInt(0)])
+            op3 = ResOperation(rop.GUARD_TRUE, [op2])
+            jumpargs.insert(index_counter, op1)
+            op4 = ResOperation(rop.JUMP, jumpargs, descr=targettoken)
+            operations = [op0, op1, op2, op3, op4]
             operations[3].setfailargs(inputargs[:])
             operations[3].setdescr(faildescr)
             #
@@ -1270,12 +1272,12 @@
             values = []
             S = lltype.GcStruct('S')
             for box in inputargs:
-                if isinstance(box, BoxInt):
+                if box.type == 'i':
                     values.append(r.randrange(-10000, 10000))
-                elif isinstance(box, BoxPtr):
+                elif box.type == 'r':
                     p = lltype.malloc(S)
                     values.append(lltype.cast_opaque_ptr(llmemory.GCREF, p))
-                elif isinstance(box, BoxFloat):
+                elif box.type == 'f':
                     values.append(longlong.getfloatstorage(r.random()))
                 else:
                     assert 0
@@ -1296,11 +1298,11 @@
             assert dstvalues[index_counter] == 11
             dstvalues[index_counter] = 0
             for i, (box, val) in enumerate(zip(inputargs, dstvalues)):
-                if isinstance(box, BoxInt):
+                if box.type == 'i':
                     got = self.cpu.get_int_value(deadframe, i)
-                elif isinstance(box, BoxPtr):
+                elif box.type == 'r':
                     got = self.cpu.get_ref_value(deadframe, i)
-                elif isinstance(box, BoxFloat):
+                elif box.type == 'f':
                     got = self.cpu.get_float_value(deadframe, i)
                 else:
                     assert 0
@@ -1311,7 +1313,7 @@
         if not self.cpu.supports_floats:
             py.test.skip("requires floats")
         fboxes = [BoxFloat() for i in range(12)]
-        i2 = BoxInt()
+        i2 = InputArgInt()
         targettoken = TargetToken()
         faildescr1 = BasicFailDescr(1)
         faildescr2 = BasicFailDescr(2)
@@ -1381,7 +1383,7 @@
         assert longlong.getrealfloat(f2) == 0.75
         assert longlong.getrealfloat(f3) == 133.0
 
-        zero = BoxInt()
+        zero = InputArgInt()
         bridgeops = [
             ResOperation(rop.SAME_AS, [ConstInt(0)], zero),
             ResOperation(rop.GUARD_TRUE, [zero], None, descr=faildescr1),
@@ -1411,8 +1413,8 @@
                 (rop.GUARD_FALSE, False),
                 (rop.GUARD_TRUE,  True),
                 ]:
-                box = BoxInt()
-                res = BoxInt()
+                box = InputArgInt()
+                res = InputArgInt()
                 faildescr1 = BasicFailDescr(1)
                 faildescr2 = BasicFinalDescr(2)
                 inputargs = [box]
@@ -1449,18 +1451,18 @@
                 for combinaison in ["bb", "bc", "cb"]:
                     #
                     if combinaison[0] == 'b':
-                        ibox1 = BoxInt()
+                        ibox1 = InputArgInt()
                     else:
                         ibox1 = ConstInt(-42)
                     if combinaison[1] == 'b':
-                        ibox2 = BoxInt()
+                        ibox2 = InputArgInt()
                     else:
                         ibox2 = ConstInt(-42)
-                    b1 = BoxInt()
+                    b1 = InputArgInt()
                     faildescr1 = BasicFailDescr(1)
                     faildescr2 = BasicFinalDescr(2)
                     inputargs = [ib for ib in [ibox1, ibox2]
-                                    if isinstance(ib, BoxInt)]
+                                    if isinstance(ib, InputArgInt)]
                     operations = [
                         ResOperation(opname, [ibox1, ibox2], b1),
                         ResOperation(opguard, [b1], None, descr=faildescr1),
@@ -1501,18 +1503,18 @@
                 for combinaison in ["bb", "bc", "cb"]:
                     #
                     if combinaison[0] == 'b':
-                        ibox1 = BoxInt()
+                        ibox1 = InputArgInt()
                     else:
                         ibox1 = ConstInt(42)
                     if combinaison[1] == 'b':
-                        ibox2 = BoxInt()
+                        ibox2 = InputArgInt()
                     else:
                         ibox2 = ConstInt(42)
-                    b1 = BoxInt()
+                    b1 = InputArgInt()
                     faildescr1 = BasicFailDescr(1)
                     faildescr2 = BasicFinalDescr(2)
                     inputargs = [ib for ib in [ibox1, ibox2]
-                                    if isinstance(ib, BoxInt)]
+                                    if isinstance(ib, InputArgInt)]
                     operations = [
                         ResOperation(opname, [ibox1, ibox2], b1),
                         ResOperation(opguard, [b1], None, descr=faildescr1),
@@ -1564,7 +1566,7 @@
                         fbox2 = BoxFloat()
                     else:
                         fbox2 = constfloat(-4.5)
-                    b1 = BoxInt()
+                    b1 = InputArgInt()
                     faildescr1 = BasicFailDescr(1)
                     faildescr2 = BasicFinalDescr(2)
                     inputargs = [fb for fb in [fbox1, fbox2]
@@ -1622,7 +1624,7 @@
         for opnum, boxargs, rettype, retvalue in tests:
             inputargs += [box for box in boxargs if isinstance(box, Box)]
             if rettype == 'int':
-                boxres = BoxInt()
+                boxres = InputArgInt()
             elif rettype == 'float':
                 boxres = BoxFloat()
             else:
@@ -1639,7 +1641,7 @@
         #
         args = []
         for box in inputargs:
-            if isinstance(box, BoxInt):
+            if isinstance(box, InputArgInt):
                 args.append(box.getint())
             elif isinstance(box, BoxFloat):
                 args.append(box.getfloatstorage())
@@ -1697,7 +1699,7 @@
                 if isinstance(expected, bool):
                     for guard_opnum, expected_id in [(rop.GUARD_TRUE, 1),
                                                      (rop.GUARD_FALSE, 0)]:
-                        box = BoxInt()
+                        box = InputArgInt()
                         operations = [
                             ResOperation(opnum, list(testcase), box),
                             ResOperation(guard_opnum, [box], None,
@@ -1836,24 +1838,24 @@
             t.parent.parent.typeptr = vtable_for_T
         elif T == self.U:
             t.parent.parent.parent.typeptr = vtable_for_T
-        t_box = BoxPtr(lltype.cast_opaque_ptr(llmemory.GCREF, t))
+        t_box = InputArgRef(lltype.cast_opaque_ptr(llmemory.GCREF, t))
         T_box = ConstInt(heaptracker.adr2int(vtable_for_T_addr))
         return t_box, T_box
 
     def null_instance(self):
-        return BoxPtr(lltype.nullptr(llmemory.GCREF.TO))
+        return InputArgRef(lltype.nullptr(llmemory.GCREF.TO))
 
     def alloc_array_of(self, ITEM, length):
         A = lltype.GcArray(ITEM)
         a = lltype.malloc(A, length)
-        a_box = BoxPtr(lltype.cast_opaque_ptr(llmemory.GCREF, a))
+        a_box = InputArgRef(lltype.cast_opaque_ptr(llmemory.GCREF, a))
         return a_box, A
 
     def alloc_string(self, string):
         s = rstr.mallocstr(len(string))
         for i in range(len(string)):
             s.chars[i] = string[i]
-        s_box = BoxPtr(lltype.cast_opaque_ptr(llmemory.GCREF, s))
+        s_box = InputArgRef(lltype.cast_opaque_ptr(llmemory.GCREF, s))
         return s_box
 
     def look_string(self, string_box):
@@ -1864,7 +1866,7 @@
         u = rstr.mallocunicode(len(unicode))
         for i in range(len(unicode)):
             u.chars[i] = unicode[i]
-        u_box = BoxPtr(lltype.cast_opaque_ptr(llmemory.GCREF, u))
+        u_box = InputArgRef(lltype.cast_opaque_ptr(llmemory.GCREF, u))
         return u_box
 
     def look_unicode(self, unicode_box):
@@ -1874,7 +1876,7 @@
 
     def test_cast_int_to_ptr(self):
         res = self.execute_operation(rop.CAST_INT_TO_PTR,
-                                     [BoxInt(-17)],  'ref').value
+                                     [InputArgInt(-17)],  'ref').value
         assert lltype.cast_ptr_to_int(res) == -17
 
     def test_cast_ptr_to_int(self):
@@ -1888,7 +1890,7 @@
             py.test.skip("requires floats")
         for x in [-10, -1, 0, 3, 42, sys.maxint-1]:
             res = self.execute_operation(rop.CAST_INT_TO_FLOAT,
-                                         [BoxInt(x)],  'float').value
+                                         [InputArgInt(x)],  'float').value
             assert longlong.getrealfloat(res) == float(x)
             # --- the front-end never generates CAST_INT_TO_FLOAT(Const)
             #res = self.execute_operation(rop.CAST_INT_TO_FLOAT,
@@ -1926,9 +1928,9 @@
     def test_ooops_non_gc(self):
         x = lltype.malloc(lltype.Struct('x'), flavor='raw')
         v = heaptracker.adr2int(llmemory.cast_ptr_to_adr(x))
-        r = self.execute_operation(rop.PTR_EQ, [BoxInt(v), BoxInt(v)], 'int')
+        r = self.execute_operation(rop.PTR_EQ, [InputArgInt(v), InputArgInt(v)], 'int')
         assert r.value == 1
-        r = self.execute_operation(rop.PTR_NE, [BoxInt(v), BoxInt(v)], 'int')
+        r = self.execute_operation(rop.PTR_NE, [InputArgInt(v), InputArgInt(v)], 'int')
         assert r.value == 0
         lltype.free(x, flavor='raw')
 
@@ -1941,9 +1943,9 @@
         assert r1.value != r2.value
         xdescr = cpu.fielddescrof(S, 'x')
         ydescr = cpu.fielddescrof(S, 'y')
-        self.execute_operation(rop.SETFIELD_GC, [r1, BoxInt(150)],
+        self.execute_operation(rop.SETFIELD_GC, [r1, InputArgInt(150)],
                                'void', descr=ydescr)
-        self.execute_operation(rop.SETFIELD_GC, [r1, BoxInt(190)],
+        self.execute_operation(rop.SETFIELD_GC, [r1, InputArgInt(190)],
                                'void', descr=xdescr)
         s = lltype.cast_opaque_ptr(lltype.Ptr(S), r1.value)
         assert s.x == chr(190)
@@ -1961,11 +1963,11 @@
         descr1 = cpu.fielddescrof(self.S, 'chr1')
         descr2 = cpu.fielddescrof(self.S, 'chr2')
         descrshort = cpu.fielddescrof(self.S, 'short')
-        self.execute_operation(rop.SETFIELD_GC, [r1, BoxInt(150)],
+        self.execute_operation(rop.SETFIELD_GC, [r1, InputArgInt(150)],
                                'void', descr=descr2)
-        self.execute_operation(rop.SETFIELD_GC, [r1, BoxInt(190)],
+        self.execute_operation(rop.SETFIELD_GC, [r1, InputArgInt(190)],
                                'void', descr=descr1)
-        self.execute_operation(rop.SETFIELD_GC, [r1, BoxInt(1313)],
+        self.execute_operation(rop.SETFIELD_GC, [r1, InputArgInt(1313)],
                                'void', descr=descrshort)
         s = lltype.cast_opaque_ptr(lltype.Ptr(self.T), r1.value)
         assert s.parent.chr1 == chr(190)
@@ -1984,9 +1986,9 @@
     def test_new_array(self):
         A = lltype.GcArray(lltype.Signed)
         arraydescr = self.cpu.arraydescrof(A)
-        r1 = self.execute_operation(rop.NEW_ARRAY, [BoxInt(342)],
+        r1 = self.execute_operation(rop.NEW_ARRAY, [InputArgInt(342)],
                                     'ref', descr=arraydescr)
-        r2 = self.execute_operation(rop.NEW_ARRAY, [BoxInt(342)],
+        r2 = self.execute_operation(rop.NEW_ARRAY, [InputArgInt(342)],
                                     'ref', descr=arraydescr)
         assert r1.value != r2.value
         a = lltype.cast_opaque_ptr(lltype.Ptr(A), r1.value)
@@ -1995,22 +1997,22 @@
     def test_new_array_clear(self):
         A = lltype.GcArray(lltype.Signed)
         arraydescr = self.cpu.arraydescrof(A)
-        r1 = self.execute_operation(rop.NEW_ARRAY_CLEAR, [BoxInt(342)],
+        r1 = self.execute_operation(rop.NEW_ARRAY_CLEAR, [InputArgInt(342)],
                                     'ref', descr=arraydescr)
         a = lltype.cast_opaque_ptr(lltype.Ptr(A), r1.value)
         assert a[0] == 0
         assert len(a) == 342
 
     def test_new_string(self):
-        r1 = self.execute_operation(rop.NEWSTR, [BoxInt(342)], 'ref')
-        r2 = self.execute_operation(rop.NEWSTR, [BoxInt(342)], 'ref')
+        r1 = self.execute_operation(rop.NEWSTR, [InputArgInt(342)], 'ref')
+        r2 = self.execute_operation(rop.NEWSTR, [InputArgInt(342)], 'ref')
         assert r1.value != r2.value
         a = lltype.cast_opaque_ptr(lltype.Ptr(rstr.STR), r1.value)
         assert len(a.chars) == 342
 
     def test_new_unicode(self):
-        r1 = self.execute_operation(rop.NEWUNICODE, [BoxInt(342)], 'ref')
-        r2 = self.execute_operation(rop.NEWUNICODE, [BoxInt(342)], 'ref')
+        r1 = self.execute_operation(rop.NEWUNICODE, [InputArgInt(342)], 'ref')
+        r2 = self.execute_operation(rop.NEWUNICODE, [InputArgInt(342)], 'ref')
         assert r1.value != r2.value
         a = lltype.cast_opaque_ptr(lltype.Ptr(rstr.UNICODE), r1.value)
         assert len(a.chars) == 342
@@ -2197,7 +2199,7 @@
             def get_write_barrier_from_array_fn(self, cpu):
                 return funcbox.getint()
         #
-        for BoxIndexCls in [BoxInt, ConstInt]*3:
+        for BoxIndexCls in [InputArgInt, ConstInt]*3:
             for cond in [-1, 0, 1, 2]:
                 # cond=-1:GCFLAG_TRACK_YOUNG_PTRS, GCFLAG_CARDS_SET are not set
                 # cond=0: GCFLAG_CARDS_SET is never set
@@ -2297,8 +2299,8 @@
         calldescr = self.cpu.calldescrof(FUNC, FUNC.ARGS, FUNC.RESULT,
                                          EffectInfo.MOST_GENERAL)
         cpu = self.cpu
-        i0 = BoxInt()
-        i1 = BoxInt()
+        i0 = InputArgInt()
+        i1 = InputArgInt()
         tok = BoxPtr()
         faildescr = BasicFailDescr(1)
         ops = [
@@ -2342,9 +2344,9 @@
         calldescr = self.cpu.calldescrof(FUNC, FUNC.ARGS, FUNC.RESULT,
                                          EffectInfo.MOST_GENERAL)
         cpu = self.cpu
-        i0 = BoxInt()
-        i1 = BoxInt()
-        i2 = BoxInt()
+        i0 = InputArgInt()
+        i1 = InputArgInt()
+        i2 = InputArgInt()
         tok = BoxPtr()
         faildescr = BasicFailDescr(1)
         ops = [
@@ -2390,8 +2392,8 @@
         calldescr = self.cpu.calldescrof(FUNC, FUNC.ARGS, FUNC.RESULT,
                                          EffectInfo.MOST_GENERAL)
         cpu = self.cpu
-        i0 = BoxInt()
-        i1 = BoxInt()
+        i0 = InputArgInt()
+        i1 = InputArgInt()
         f2 = BoxFloat()
         tok = BoxPtr()
         faildescr = BasicFailDescr(1)
@@ -2424,8 +2426,8 @@
 
     def test_guard_not_forced_2(self):
         cpu = self.cpu
-        i0 = BoxInt()
-        i1 = BoxInt()
+        i0 = InputArgInt()
+        i1 = InputArgInt()
         tok = BoxPtr()
         faildescr = BasicFailDescr(1)
         ops = [
@@ -2460,9 +2462,9 @@
         func_adr = llmemory.cast_ptr_to_adr(c_tolower.funcsym)
         funcbox = ConstInt(heaptracker.adr2int(func_adr))
         calldescr = cpu._calldescr_dynamic_for_tests([types.uchar], types.sint)
-        i1 = BoxInt()
-        i2 = BoxInt()
-        tok = BoxInt()
+        i1 = InputArgInt()
+        i2 = InputArgInt()
+        tok = InputArgInt()
         faildescr = BasicFailDescr(1)
         ops = [
         ResOperation(rop.CALL_RELEASE_GIL, [funcbox, i1], i2,
@@ -2516,11 +2518,11 @@
         calldescr = cpu._calldescr_dynamic_for_tests(
             [types.pointer, types_size_t, types_size_t, types.pointer],
             types.void)
-        i0 = BoxInt()
-        i1 = BoxInt()
-        i2 = BoxInt()
-        i3 = BoxInt()
-        tok = BoxInt()
+        i0 = InputArgInt()
+        i1 = InputArgInt()
+        i2 = InputArgInt()
+        i3 = InputArgInt()
+        tok = InputArgInt()
         faildescr = BasicFailDescr(1)
         ops = [
         ResOperation(rop.CALL_RELEASE_GIL, [funcbox, i0, i1, i2, i3], None,
@@ -2569,14 +2571,14 @@
             [types.ulong, types.pointer],
             types.ulong,
             abiname='FFI_STDCALL')
-        i1 = BoxInt()
-        i2 = BoxInt()
+        i1 = InputArgInt()
+        i2 = InputArgInt()
         faildescr = BasicFailDescr(1)
         # if the stdcall convention is ignored, then ESP is wrong after the
         # call: 8 bytes too much.  If we repeat the call often enough, crash.
         ops = []
         for i in range(50):
-            i3 = BoxInt()
+            i3 = InputArgInt()
             ops += [
                 ResOperation(rop.CALL_RELEASE_GIL, [funcbox, i1, i2], i3,
                              descr=calldescr),
@@ -2643,7 +2645,7 @@
             faildescr = BasicFailDescr(1)
             kind = types.getkind(ffitype)
             if kind in 'uis':
-                b3 = BoxInt()
+                b3 = InputArgInt()
             elif kind in 'fUI':
                 b3 = BoxFloat()
             else:
@@ -2662,7 +2664,7 @@
             deadframe = self.cpu.execute_token(looptoken)
             fail = self.cpu.get_latest_descr(deadframe)
             assert fail.identifier == 0
-            if isinstance(b3, BoxInt):
+            if isinstance(b3, InputArgInt):
                 r = self.cpu.get_int_value(deadframe, 0)
                 if isinstance(result, r_singlefloat):
                     assert -sys.maxint-1 <= r <= 0xFFFFFFFF
@@ -2782,13 +2784,13 @@
             calldescr = cpu._calldescr_dynamic_for_tests(ffitypes, types.void)
             faildescr = BasicFailDescr(1)
             #
-            argboxes = [BoxInt()]   # for the function to call
+            argboxes = [InputArgInt()]   # for the function to call
             codes = ['X']
             for ffitype in ffitypes:
                 kind = types.getkind(ffitype)
                 codes.append(kind)
                 if kind in 'uis':
-                    b1 = BoxInt()
+                    b1 = InputArgInt()
                 elif kind in 'fUI':
                     b1 = BoxFloat()
                 else:
@@ -2872,8 +2874,8 @@
 
     def test_guard_not_invalidated(self):
         cpu = self.cpu
-        i0 = BoxInt()
-        i1 = BoxInt()
+        i0 = InputArgInt()
+        i1 = InputArgInt()
         faildescr = BasicFailDescr(1)
         ops = [
             ResOperation(rop.GUARD_NOT_INVALIDATED, [], None, descr=faildescr),
@@ -2901,7 +2903,7 @@
         print '-'*79
 
         # attach a bridge
-        i2 = BoxInt()
+        i2 = InputArgInt()
         faildescr2 = BasicFailDescr(2)
         ops = [
             ResOperation(rop.GUARD_NOT_INVALIDATED, [],None, descr=faildescr2),
@@ -2931,7 +2933,7 @@
         # the label.  If it doesn't, then in this example after we invalidate
         # the guard, jumping to the label will hit the invalidation code too
         cpu = self.cpu
-        i0 = BoxInt()
+        i0 = InputArgInt()
         faildescr = BasicFailDescr(1)
         labeldescr = TargetToken()
         ops = [
@@ -2945,7 +2947,7 @@
         # mark as failing
         self.cpu.invalidate_loop(looptoken)
         # attach a bridge
-        i2 = BoxInt()
+        i2 = InputArgInt()
         ops2 = [
             ResOperation(rop.JUMP, [ConstInt(333)], None, descr=labeldescr),
         ]
@@ -3055,14 +3057,14 @@
         #descrfld_ry = cpu.fielddescrof(RS, 'y')
         #rs.y = a
         #x = cpu.do_getfield_raw(
-        #    BoxInt(cpu.cast_adr_to_int(llmemory.cast_ptr_to_adr(rs))),
+        #    InputArgInt(cpu.cast_adr_to_int(llmemory.cast_ptr_to_adr(rs))),
         #    descrfld_ry)
         #assert isinstance(x, BoxPtr)
         #assert x.getref(lltype.Ptr(A)) == a
         #
         #rs.y = lltype.nullptr(A)
         #cpu.do_setfield_raw(
-        #    BoxInt(cpu.cast_adr_to_int(llmemory.cast_ptr_to_adr(rs))), x,
+        #    InputArgInt(cpu.cast_adr_to_int(llmemory.cast_ptr_to_adr(rs))), x,
         #    descrfld_ry)
         #assert rs.y == a
         #
@@ -3111,12 +3113,12 @@
     def test_guards_nongc(self):
         x = lltype.malloc(lltype.Struct('x'), flavor='raw')
         v = heaptracker.adr2int(llmemory.cast_ptr_to_adr(x))
-        vbox = BoxInt(v)
+        vbox = InputArgInt(v)
         ops = [
             (rop.GUARD_NONNULL, vbox, False),
             (rop.GUARD_ISNULL, vbox, True),
-            (rop.GUARD_NONNULL, BoxInt(0), True),
-            (rop.GUARD_ISNULL, BoxInt(0), False),
+            (rop.GUARD_NONNULL, InputArgInt(0), True),
+            (rop.GUARD_ISNULL, InputArgInt(0), False),
             ]
         for opname, arg, res in ops:
             self.execute_operation(opname, [arg], 'void')
@@ -3315,8 +3317,8 @@
         a = lltype.malloc(ARRAY, 10, flavor='raw')
         a[7] = -4242
         addr = llmemory.cast_ptr_to_adr(a)
-        abox = BoxInt(heaptracker.adr2int(addr))
-        r1 = self.execute_operation(rop.GETARRAYITEM_RAW, [abox, BoxInt(7)],
+        abox = InputArgInt(heaptracker.adr2int(addr))
+        r1 = self.execute_operation(rop.GETARRAYITEM_RAW, [abox, InputArgInt(7)],
                                     'int', descr=descr)
         assert r1.getint() == -4242
         lltype.free(a, flavor='raw')
@@ -3326,9 +3328,9 @@
         descr = self.cpu.arraydescrof(ARRAY)
         a = lltype.malloc(ARRAY, 10, flavor='raw')
         addr = llmemory.cast_ptr_to_adr(a)
-        abox = BoxInt(heaptracker.adr2int(addr))
-        self.execute_operation(rop.SETARRAYITEM_RAW, [abox, BoxInt(5),
-                                                      BoxInt(12345)],
+        abox = InputArgInt(heaptracker.adr2int(addr))
+        self.execute_operation(rop.SETARRAYITEM_RAW, [abox, InputArgInt(5),
+                                                      InputArgInt(12345)],
                                'void', descr=descr)
         assert a[5] == 12345
         lltype.free(a, flavor='raw')
@@ -3492,7 +3494,7 @@
             a[3] = rffi.cast(RESTYPE, value)
             a_gcref = lltype.cast_opaque_ptr(llmemory.GCREF, a)
             res = self.execute_operation(rop.GETARRAYITEM_GC,
-                                         [BoxPtr(a_gcref), BoxInt(3)],
+                                         [BoxPtr(a_gcref), InputArgInt(3)],
                                          'int', descr=descrarray)
             assert res.value == expected, (
                 "%r: got %r, expected %r" % (RESTYPE, res.value, expected))
@@ -3535,7 +3537,7 @@
             a[3] = rffi.cast(RESTYPE, value)
             a_rawint = heaptracker.adr2int(llmemory.cast_ptr_to_adr(a))
             res = self.execute_operation(rop.GETARRAYITEM_RAW,
-                                         [BoxInt(a_rawint), BoxInt(3)],
+                                         [InputArgInt(a_rawint), InputArgInt(3)],
                                          'int', descr=descrarray)
             assert res.value == expected, (
                 "%r: got %r, expected %r" % (RESTYPE, res.value, expected))
@@ -3603,7 +3605,7 @@
             calldescr = self.cpu.calldescrof(FUNC, FUNC.ARGS, FUNC.RESULT,
                                              EffectInfo.MOST_GENERAL)
             funcbox = self.get_funcbox(self.cpu, f)
-            res = self.execute_operation(rop.CALL, [funcbox, BoxInt(value)],
+            res = self.execute_operation(rop.CALL, [funcbox, InputArgInt(value)],
                                          'int', descr=calldescr)
             assert res.value == expected, (
                 "%r: got %r, expected %r" % (RESTYPE, res.value, expected))
@@ -3723,7 +3725,7 @@
         funcbox = self.get_funcbox(self.cpu, f)
         ivalue = longlong.singlefloat2int(value)
         iexpected = longlong.singlefloat2int(expected)
-        res = self.execute_operation(rop.CALL, [funcbox, BoxInt(ivalue)],
+        res = self.execute_operation(rop.CALL, [funcbox, InputArgInt(ivalue)],
                                      'int', descr=calldescr)
         assert res.value == iexpected
 
@@ -3746,7 +3748,7 @@
         self.cpu.propagate_exception_descr = excdescr
         self.cpu.setup_once()    # xxx redo it, because we added
                                  # propagate_exception
-        i0 = BoxInt()
+        i0 = InputArgInt()
         p0 = BoxPtr()
         operations = [
             ResOperation(rop.NEWUNICODE, [i0], p0),
@@ -3787,10 +3789,10 @@
             assert res.getfloat() == expected
 
     def test_compile_loop_with_target(self):
-        i0 = BoxInt()
-        i1 = BoxInt()
-        i2 = BoxInt()
-        i3 = BoxInt()
+        i0 = InputArgInt()
+        i1 = InputArgInt()
+        i2 = InputArgInt()
+        i3 = InputArgInt()
         looptoken = JitCellToken()
         targettoken1 = TargetToken()
         targettoken2 = TargetToken()
@@ -3904,8 +3906,8 @@
         # It catches a case in which we underestimate the needed frame_depth across
         # the cross-loop JUMP, because we estimate it based on the frame_depth stored
         # in the original loop.
-        i0 = BoxInt()
-        i1 = BoxInt()
+        i0 = InputArgInt()
+        i1 = InputArgInt()
         looptoken1 = JitCellToken()
         targettoken1 = TargetToken()
         faildescr1 = BasicFailDescr(2)
@@ -3934,11 +3936,11 @@
                                     EffectInfo.MOST_GENERAL)
         funcbox = self.get_funcbox(cpu, func_ptr)
 
-        i0 = BoxInt(); i1 = BoxInt(); i2 = BoxInt(); i3 = BoxInt(); i4 = BoxInt()
-        i5 = BoxInt(); i6 = BoxInt(); i7 = BoxInt(); i8 = BoxInt(); i9 = BoxInt()
-        i10 = BoxInt(); i11 = BoxInt(); i12 = BoxInt(); i13 = BoxInt(); i14 = BoxInt()
-        i15 = BoxInt(); i16 = BoxInt(); i17 = BoxInt(); i18 = BoxInt(); i19 = BoxInt()
-        i20 = BoxInt()
+        i0 = InputArgInt(); i1 = InputArgInt(); i2 = InputArgInt(); i3 = InputArgInt(); i4 = InputArgInt()
+        i5 = InputArgInt(); i6 = InputArgInt(); i7 = InputArgInt(); i8 = InputArgInt(); i9 = InputArgInt()
+        i10 = InputArgInt(); i11 = InputArgInt(); i12 = InputArgInt(); i13 = InputArgInt(); i14 = InputArgInt()
+        i15 = InputArgInt(); i16 = InputArgInt(); i17 = InputArgInt(); i18 = InputArgInt(); i19 = InputArgInt()
+        i20 = InputArgInt()
         inputargs = [i0]
         operations2 = [
             ResOperation(rop.LABEL, [i0], None, descr=targettoken1),
@@ -3973,7 +3975,7 @@
         self.cpu.compile_bridge(faildescr1, inputargs, operations2, looptoken1)
 
         looptoken2 = JitCellToken()
-        inputargs = [BoxInt()]
+        inputargs = [InputArgInt()]
         operations3 = [
             ResOperation(rop.JUMP, [ConstInt(0)], None, descr=targettoken1),
             ]
@@ -4205,9 +4207,9 @@
         funcbox = self.get_funcbox(self.cpu, func_ptr).constbox()
         calldescr = self.cpu.calldescrof(FUNC, FUNC.ARGS, FUNC.RESULT,
                                          EffectInfo.MOST_GENERAL)
-        i0 = BoxInt()
-        i1 = BoxInt()
-        i2 = BoxInt()
+        i0 = InputArgInt()
+        i1 = InputArgInt()
+        i2 = InputArgInt()
         tok = BoxPtr()
         faildescr = BasicFailDescr(23)
         ops = [
@@ -4459,8 +4461,8 @@
             for (start, length) in [(0, 100), (49, 49), (1, 98),
                                     (15, 9), (10, 10), (47, 0),
                                     (0, 4)]:
-                for cls1 in [ConstInt, BoxInt]:
-                    for cls2 in [ConstInt, BoxInt]:
+                for cls1 in [ConstInt, InputArgInt]:
+                    for cls2 in [ConstInt, InputArgInt]:
                         print 'a_int:', a_int
                         print 'of:', OF
                         print 'start:', cls1.__name__, start

From noreply at buildbot.pypy.org  Mon Nov 24 11:28:37 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 24 Nov 2014 11:28:37 +0100 (CET)
Subject: [pypy-commit] pypy default: Fix test_pypy_c/model to be a bit less
 lenient in matching vars. For
Message-ID: <20141124102837.103BB1D2DFB@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r74657:736da94c7b02
Date: 2014-11-24 11:28 +0100
http://bitbucket.org/pypy/pypy/changeset/736da94c7b02/

Log:	Fix test_pypy_c/model to be a bit less lenient in matching vars. For
	example, it doesn't make sense that an expected variable matches
	what is really a numeric constant or vice-versa. This forces you to
	use '_' to mean 'one argument of any value', and '#' to mean 'one
	integer or float constant'.

diff --git a/pypy/module/pypyjit/test_pypy_c/model.py b/pypy/module/pypyjit/test_pypy_c/model.py
--- a/pypy/module/pypyjit/test_pypy_c/model.py
+++ b/pypy/module/pypyjit/test_pypy_c/model.py
@@ -275,9 +275,19 @@
 
     @classmethod
     def parse_op(cls, line):
-        # strip comment
+        # strip comment after '#', but not if it appears inside parentheses
         if '#' in line:
-            line = line[:line.index('#')]
+            nested = 0
+            for i, c in enumerate(line):
+                if c == '(':
+                    nested += 1
+                elif c == ')':
+                    assert nested > 0, "more ')' than '(' in %r" % (line,)
+                    nested -= 1
+                elif c == '#' and nested == 0:
+                    line = line[:i]
+                    break
+        #
         if line.strip() == 'guard_not_invalidated?':
             return 'guard_not_invalidated', None, [], '...', False
         # find the resvar, if any
@@ -314,7 +324,7 @@
         # to repeat it every time
         ticker_check = """
             guard_not_invalidated?
-            ticker0 = getfield_raw(ticker_address, descr=)
+            ticker0 = getfield_raw(#, descr=)
             ticker_cond0 = int_lt(ticker0, 0)
             guard_false(ticker_cond0, descr=...)
         """
@@ -323,9 +333,9 @@
         # this is the ticker check generated if we have threads
         thread_ticker_check = """
             guard_not_invalidated?
-            ticker0 = getfield_raw(ticker_address, descr=)
-            ticker1 = int_sub(ticker0, _)
-            setfield_raw(ticker_address, ticker1, descr=)
+            ticker0 = getfield_raw(#, descr=)
+            ticker1 = int_sub(ticker0, #)
+            setfield_raw(#, ticker1, descr=)
             ticker_cond0 = int_lt(ticker1, 0)
             guard_false(ticker_cond0, descr=...)
         """
@@ -333,7 +343,7 @@
         #
         # this is the ticker check generated in PyFrame.handle_operation_error
         exc_ticker_check = """
-            ticker2 = getfield_raw(ticker_address, descr=)
+            ticker2 = getfield_raw(#, descr=)
             ticker_cond1 = int_lt(ticker2, 0)
             guard_false(ticker_cond1, descr=...)
         """
@@ -351,18 +361,31 @@
 
     @staticmethod
     def as_numeric_const(v1):
+        # returns one of:  ('int', value)  ('float', value)  None
         try:
-            return int(v1)
-        except (ValueError, TypeError):
-            return None
+            return ('int', int(v1))
+        except ValueError:
+            pass
+        if '.' in v1:
+            try:
+                return ('float', float(v1))
+            except ValueError:
+                pass
+        return None
 
     def match_var(self, v1, exp_v2):
         assert v1 != '_'
-        if exp_v2 == '_':
+        if exp_v2 == '_':           # accept anything
             return True
+        if exp_v2 is None:
+            return v1 is None
+        assert exp_v2 != '...'      # bogus use of '...' in the expected code
         n1 = self.as_numeric_const(v1)
+        if exp_v2 == '#':           # accept any (integer or float) number
+            return n1 is not None
         n2 = self.as_numeric_const(exp_v2)
-        if n1 is not None and n2 is not None:
+        if n1 is not None or n2 is not None:
+            # at least one is a number; check that both are, and are equal
             return n1 == n2
         if self.is_const(v1) or self.is_const(exp_v2):
             return v1[:-1].startswith(exp_v2[:-1])
@@ -382,10 +405,13 @@
     def match_op(self, op, (exp_opname, exp_res, exp_args, exp_descr, _)):
         self._assert(op.name == exp_opname, "operation mismatch")
         self.match_var(op.res, exp_res)
-        if exp_args != ['...']:
+        if exp_args[-1:] == ['...']:      # exp_args ends with '...'
+            exp_args = exp_args[:-1]
+            self._assert(len(op.args) >= len(exp_args), "not enough arguments")
+        else:
             self._assert(len(op.args) == len(exp_args), "wrong number of arguments")
-            for arg, exp_arg in zip(op.args, exp_args):
-                self._assert(self.match_var(arg, exp_arg), "variable mismatch: %r instead of %r" % (arg, exp_arg))
+        for arg, exp_arg in zip(op.args, exp_args):
+            self._assert(self.match_var(arg, exp_arg), "variable mismatch: %r instead of %r" % (arg, exp_arg))
         self.match_descr(op.descr, exp_descr)
 
 
diff --git a/pypy/module/pypyjit/test_pypy_c/test_00_model.py b/pypy/module/pypyjit/test_pypy_c/test_00_model.py
--- a/pypy/module/pypyjit/test_pypy_c/test_00_model.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_00_model.py
@@ -158,6 +158,24 @@
         assert match_var('v0', 'V0')
         assert match_var('ConstPtr(ptr0)', '_')
         py.test.raises(AssertionError, "match_var('_', 'v0')")
+        #
+        # numerics
+        assert match_var('1234', '1234')
+        assert not match_var('1234', '1235')
+        assert not match_var('v0', '1234')
+        assert not match_var('1234', 'v0')
+        assert match_var('1234', '#')        # the '#' char matches any number
+        assert not match_var('v0', '#')
+        assert match_var('1234', '_')        # the '_' char matches anything
+        #
+        # float numerics
+        assert match_var('0.000000', '0.0')
+        assert not match_var('0.000000', '0')
+        assert not match_var('0', '0.0')
+        assert not match_var('v0', '0.0')
+        assert not match_var('0.0', 'v0')
+        assert match_var('0.0', '#')
+        assert match_var('0.0', '_')
 
     def test_parse_op(self):
         res = OpMatcher.parse_op("  a =   int_add(  b,  3 ) # foo")
@@ -210,6 +228,19 @@
         """
         assert not self.match(loop, expected)
 
+    def test_dotdotdot_in_operation(self):
+        loop = """
+            [i0, i1]
+            jit_debug(i0, 1, ConstClass(myclass), i1)
+        """
+        assert self.match(loop, "jit_debug(...)")
+        assert self.match(loop, "jit_debug(i0, ...)")
+        assert self.match(loop, "jit_debug(i0, 1, ...)")
+        assert self.match(loop, "jit_debug(i0, 1, _, ...)")
+        assert self.match(loop, "jit_debug(i0, 1, _, i1, ...)")
+        py.test.raises(AssertionError, self.match,
+                       loop, "jit_debug(i0, 1, ..., i1)")
+
     def test_match_descr(self):
         loop = """
             [p0]
@@ -232,7 +263,7 @@
             jump(i4)
         """
         expected = """
-            i1 = int_add(0, 1)
+            i1 = int_add(i0, 1)
             ...
             i4 = int_mul(i1, 1000)
             jump(i4, descr=...)
@@ -249,7 +280,7 @@
             jump(i4, descr=...)
         """
         expected = """
-            i1 = int_add(0, 1)
+            i1 = int_add(i0, 1)
             ...
             _ = int_mul(_, 1000)
             jump(i4, descr=...)
@@ -268,7 +299,7 @@
             jump(i4)
         """
         expected = """
-            i1 = int_add(0, 1)
+            i1 = int_add(i0, 1)
             ...
         """
         assert self.match(loop, expected)
diff --git a/pypy/module/pypyjit/test_pypy_c/test_buffers.py b/pypy/module/pypyjit/test_pypy_c/test_buffers.py
--- a/pypy/module/pypyjit/test_pypy_c/test_buffers.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_buffers.py
@@ -21,7 +21,7 @@
             i65 = getfield_gc(p18, descr=...)
             i67 = int_gt(0, i65)
             guard_false(i67, descr=...)
-            i69 = int_gt(., i65)
+            i69 = int_gt(#, i65)
             guard_true(i69, descr=...)
             --TICK--
         """)
@@ -56,7 +56,7 @@
             guard_false(i99, descr=...)
             i100 = int_lshift(i98, 24)
             i101 = int_or(i97, i100)
-            i102 = getfield_raw(\d+, descr=)
+            i102 = getfield_raw(#, descr=)
             i103 = int_lt(i102, 0)
             guard_false(i103, descr=...)
         """)
diff --git a/pypy/module/pypyjit/test_pypy_c/test_call.py b/pypy/module/pypyjit/test_pypy_c/test_call.py
--- a/pypy/module/pypyjit/test_pypy_c/test_call.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_call.py
@@ -395,7 +395,7 @@
             setarrayitem_gc(p24, 0, p26, descr=)
             setfield_gc(p22, p24, descr=)
             }}}
-            p32 = call_may_force(..., p18, p22, descr=)
+            p32 = call_may_force(_, p18, p22, descr=)
             ...
         """)
 
diff --git a/pypy/module/pypyjit/test_pypy_c/test_containers.py b/pypy/module/pypyjit/test_pypy_c/test_containers.py
--- a/pypy/module/pypyjit/test_pypy_c/test_containers.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_containers.py
@@ -82,7 +82,7 @@
             guard_no_exception(descr=...)
             i23 = call(ConstClass(ll_dict_lookup_trampoline), p13, p10, i12, descr=)
             guard_no_exception(descr=...)
-            i26 = int_and(i23, .*)
+            i26 = int_and(i23, #)
             i27 = int_is_true(i26)
             guard_false(i27, descr=...)
             p28 = getfield_gc(p13, descr=)
diff --git a/pypy/module/pypyjit/test_pypy_c/test_math.py b/pypy/module/pypyjit/test_pypy_c/test_math.py
--- a/pypy/module/pypyjit/test_pypy_c/test_math.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_math.py
@@ -21,7 +21,7 @@
             guard_true(i2, descr=...)
             guard_not_invalidated(descr=...)
             f1 = cast_int_to_float(i0)
-            i3 = float_le(f1, 0)
+            i3 = float_le(f1, 0.0)
             guard_false(i3, descr=...)
             f2 = call(ConstClass(log), f1, descr=)
             f3 = call(ConstClass(log10), f1, descr=)
@@ -56,7 +56,7 @@
             f3 = call(ConstClass(cos), f1, descr=)
             f4 = float_sub(f2, f3)
             f5 = float_add(f0, f4)
-            i7 = int_add(i0, f1)
+            i7 = int_add(i0, 1)
             --TICK--
             jump(..., descr=)
         """)
diff --git a/pypy/module/pypyjit/test_pypy_c/test_micronumpy.py b/pypy/module/pypyjit/test_pypy_c/test_micronumpy.py
--- a/pypy/module/pypyjit/test_pypy_c/test_micronumpy.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_micronumpy.py
@@ -104,7 +104,7 @@
             setarrayitem_gc(p150, 1, 0, descr=)
             setarrayitem_gc(p150, 0, 0, descr=)
             guard_not_invalidated(descr=...)
-            i154 = getfield_raw(ticker_address, descr=)
+            i154 = getfield_raw(#, descr=)
             i155 = int_lt(i154, 0)
             guard_false(i155, descr=...)
             p156 = new_with_vtable(...)
@@ -142,7 +142,7 @@
             raw_store(i103, i132, 42.000000, descr=)
             p152 = getfield_gc_pure(p126, descr=)
             i153 = int_add(i120, 1)
-            i154 = getfield_raw(ticker_address, descr=)
+            i154 = getfield_raw(#, descr=)
             setarrayitem_gc(p152, 1, 0, descr=)
             setarrayitem_gc(p152, 0, 0, descr=)
             i157 = int_lt(i154, 0)
diff --git a/pypy/module/pypyjit/test_pypy_c/test_string.py b/pypy/module/pypyjit/test_pypy_c/test_string.py
--- a/pypy/module/pypyjit/test_pypy_c/test_string.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_string.py
@@ -67,15 +67,15 @@
             i11 = int_lt(i6, i7)
             guard_true(i11, descr=...)
             guard_not_invalidated(descr=...)
-            i13 = int_eq(i6, %d)
+            i13 = int_eq(i6, %d)         # value provided below
             guard_false(i13, descr=...)
-            i15 = int_mod(i6, i8)
-            i17 = int_rshift(i15, %d)
-            i18 = int_and(i8, i17)
+            i15 = int_mod(i6, 10)
+            i17 = int_rshift(i15, %d)    # value provided below
+            i18 = int_and(10, i17)
             i19 = int_add(i15, i18)
             i21 = int_lt(i19, 0)
             guard_false(i21, descr=...)
-            i22 = int_ge(i19, i8)
+            i22 = int_ge(i19, 10)
             guard_false(i22, descr=...)
             i23 = strgetitem(p10, i19)
             p25 = newstr(1)
@@ -83,7 +83,7 @@
             p93 = call(ConstClass(fromstr), p25, 16, descr=)
             guard_no_exception(descr=...)
             i95 = getfield_gc_pure(p93, descr=)
-            i96 = int_gt(i95, .*)
+            i96 = int_gt(i95, #)
             guard_false(i96, descr=...)
             i94 = call(ConstClass(rbigint._toint_helper), p93, descr=)
             guard_no_exception(descr=...)
diff --git a/pypy/module/pypyjit/test_pypy_c/test_thread.py b/pypy/module/pypyjit/test_pypy_c/test_thread.py
--- a/pypy/module/pypyjit/test_pypy_c/test_thread.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_thread.py
@@ -64,7 +64,7 @@
         guard_true(i56, descr=...)
         p57 = force_token()
         setfield_gc(p0, p57, descr=)
-        i58 = call_release_gil(..., i37, 1, descr=)
+        i58 = call_release_gil(_, i37, 1, descr=)
         guard_not_forced(descr=...)
         guard_no_exception(descr=...)
         i59 = int_is_true(i58)
@@ -72,14 +72,14 @@
         i60 = int_sub(i44, 1)
         p62 = force_token()
         setfield_gc(p0, p62, descr=)
-        i63 = call_release_gil(..., i37, 0, descr=)
+        i63 = call_release_gil(_, i37, 0, descr=)
         guard_not_forced(descr=...)
         guard_no_exception(descr=...)
         i64 = int_is_true(i63)
         guard_false(i64, descr=...)
         p65 = force_token()
         setfield_gc(p0, p65, descr=)
-        call_release_gil(..., i37, descr=)
+        call_release_gil(_, i37, descr=)
         guard_not_forced(descr=...)
         guard_no_exception(descr=...)
         guard_not_invalidated(descr=...)

From noreply at buildbot.pypy.org  Mon Nov 24 12:27:09 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 24 Nov 2014 12:27:09 +0100 (CET)
Subject: [pypy-commit] pypy portable-threadlocal: Found out that the jit
 works, but produces too much cruft in the traces.
Message-ID: <20141124112709.78A811C022D@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: portable-threadlocal
Changeset: r74658:da917334290a
Date: 2014-11-24 12:26 +0100
http://bitbucket.org/pypy/pypy/changeset/da917334290a/

Log:	Found out that the jit works, but produces too much cruft in the
	traces. Change the llops a bit in the hope to improve the situation.
	(jit not fixed yet.)

diff --git a/rpython/rlib/rthread.py b/rpython/rlib/rthread.py
--- a/rpython/rlib/rthread.py
+++ b/rpython/rlib/rthread.py
@@ -284,12 +284,11 @@
 
         def getraw():
             _threadlocalref_seeme(self)
-            addr = llop.threadlocalref_addr(llmemory.Address)
-            return llop.raw_load(FIELDTYPE, addr, offset)
+            return llop.threadlocalref_get(FIELDTYPE, offset)
 
         def get_or_make_raw():
             _threadlocalref_seeme(self)
-            addr = llop.threadlocalref_make(llmemory.Address)
+            addr = llop.threadlocalref_addr(llmemory.Address)
             return llop.raw_load(FIELDTYPE, addr, offset)
 
         def setraw(value):
@@ -316,15 +315,15 @@
         unique_id = ThreadLocalReference._COUNT
         ThreadLocalReference._COUNT += 1
         ThreadLocalField.__init__(self, lltype.Signed, 'tlref%d' % unique_id)
-        getraw = self.getraw
         setraw = self.setraw
+        offset = self.offset
 
         def get():
             if we_are_translated():
                 from rpython.rtyper.annlowlevel import cast_gcref_to_instance
-                value = getraw()
-                value = lltype.cast_int_to_ptr(llmemory.GCREF, value)
-                return cast_gcref_to_instance(Cls, value)
+                _threadlocalref_seeme(self)
+                gcref = llop.threadlocalref_get(llmemory.GCREF, offset)
+                return cast_gcref_to_instance(Cls, gcref)
             else:
                 return getattr(self.local, 'value', None)
 
diff --git a/rpython/rtyper/lltypesystem/lloperation.py b/rpython/rtyper/lltypesystem/lloperation.py
--- a/rpython/rtyper/lltypesystem/lloperation.py
+++ b/rpython/rtyper/lltypesystem/lloperation.py
@@ -546,8 +546,8 @@
     'getslice':             LLOp(canraise=(Exception,)),
     'check_and_clear_exc':  LLOp(),
 
-    'threadlocalref_addr':  LLOp(sideeffects=False),
-    'threadlocalref_make':  LLOp(),
+    'threadlocalref_addr':  LLOp(sideeffects=False),  # get (or make) addr of tl
+    'threadlocalref_get':   LLOp(sideeffects=False),  # read field (no check)
 
     # __________ debugging __________
     'debug_view':           LLOp(),
diff --git a/rpython/translator/c/funcgen.py b/rpython/translator/c/funcgen.py
--- a/rpython/translator/c/funcgen.py
+++ b/rpython/translator/c/funcgen.py
@@ -13,6 +13,7 @@
 from rpython.translator.backendopt.ssa import SSI_to_SSA
 from rpython.translator.backendopt.innerloop import find_inner_loops
 from rpython.tool.identity_dict import identity_dict
+from rpython.rlib.objectmodel import CDefinedIntSymbolic
 
 
 LOCALVAR = 'l_%s'
@@ -900,4 +901,16 @@
         else:
             return None    # use the default
 
+    def OP_THREADLOCALREF_GET(self, op):
+        assert isinstance(op.args[0], Constant)
+        assert isinstance(op.args[0].value, CDefinedIntSymbolic)
+        fieldname = op.args[0].value.expr
+        assert fieldname.startswith('RPY_TLOFS_')
+        fieldname = fieldname[10:]
+        typename = self.db.gettype(op.result.concretetype)
+        return '%s = (%s)RPY_THREADLOCALREF_GET(%s);' % (
+            self.expr(op.result),
+            cdecl(typename, ''),
+            fieldname)
+
 assert not USESLOTS or '__dict__' not in dir(FunctionCodeGenerator)
diff --git a/rpython/translator/c/gc.py b/rpython/translator/c/gc.py
--- a/rpython/translator/c/gc.py
+++ b/rpython/translator/c/gc.py
@@ -75,7 +75,7 @@
         # in all cases except with framework+shadowstack.  In that
         # case the operation is removed because redundant with
         # rthread.get_or_make_ident().
-        return '{ char *r; OP_THREADLOCALREF_MAKE(r); (void)r; } '
+        return 'RPY_THREADLOCALREF_ENSURE();'
 
     def OP_GC_THREAD_START(self, funcgen, op):
         return ''
diff --git a/rpython/translator/c/src/threadlocal.c b/rpython/translator/c/src/threadlocal.c
--- a/rpython/translator/c/src/threadlocal.c
+++ b/rpython/translator/c/src/threadlocal.c
@@ -69,12 +69,6 @@
    explicitly, with malloc()/free(), and attached to (a single) thread-
    local key using the API of Windows or pthread. */
 
-#ifdef _WIN32
-#  define _RPy_ThreadLocals_Set(p)  TlsSetValue(pypy_threadlocal_key, p)
-#else
-#  define _RPy_ThreadLocals_Set(p)  pthread_setspecific(pypy_threadlocal_key, p)
-#endif
-
 void RPython_ThreadLocals_ProgramInit(void)
 {
 #ifdef _WIN32
@@ -106,11 +100,12 @@
 
 void RPython_ThreadLocals_ThreadDie(void)
 {
-    void *p;
-    OP_THREADLOCALREF_ADDR(p);
-    _RPy_ThreadLocals_Set(NULL);
-    memset(p, 0xDD, sizeof(struct pypy_threadlocal_s));  /* debug */
-    free(p);
+    void *p = _RPy_ThreadLocals_Get();
+    if (p != NULL) {
+        _RPy_ThreadLocals_Set(NULL);
+        memset(p, 0xDD, sizeof(struct pypy_threadlocal_s));  /* debug */
+        free(p);
+    }
 }
 
 
diff --git a/rpython/translator/c/src/threadlocal.h b/rpython/translator/c/src/threadlocal.h
--- a/rpython/translator/c/src/threadlocal.h
+++ b/rpython/translator/c/src/threadlocal.h
@@ -31,35 +31,18 @@
 
 RPY_EXTERN __thread struct pypy_threadlocal_s pypy_threadlocal;
 
-#define OP_THREADLOCALREF_ADDR(r)                       \
-    do {                                                \
-        RPyAssert(pypy_threadlocal.ready == 42,         \
-                  "uninitialized thread-local!");       \
-        r = (char *)&pypy_threadlocal;                  \
-    } while (0)
-
-#define OP_THREADLOCALREF_MAKE(r)               \
+#define OP_THREADLOCALREF_ADDR(r)               \
     do {                                        \
         r = (char *)&pypy_threadlocal;          \
         if (pypy_threadlocal.ready != 42)       \
             r = _RPython_ThreadLocals_Build();  \
     } while (0)
 
+#define RPY_THREADLOCALREF_ENSURE()             \
+    if (pypy_threadlocal.ready != 42)           \
+        (void)_RPython_ThreadLocals_Build();
 
-/* ------------------------------------------------------------ */
-#elif _WIN32
-/* ------------------------------------------------------------ */
-
-
-#include 
-#include 
-
-RPY_EXTERN DWORD pypy_threadlocal_key;
-#define OP_THREADLOCALREF_ADDR(r)    r = (char *)TlsGetValue(  \
-                                           pypy_threadlocal_key)
-#define OP_THREADLOCALREF_MAKE(r)                       \
-    (OP_THREADLOCALREF_ADDR(r),                         \
-     ((r) || (r = _RPython_ThreadLocals_Build())))
+#define RPY_THREADLOCALREF_GET(FIELD)   pypy_threadlocal.FIELD
 
 
 /* ------------------------------------------------------------ */
@@ -67,16 +50,35 @@
 /* ------------------------------------------------------------ */
 
 
-/* Other POSIX systems: use the pthread API */
+/* Don't use '__thread'. */
 
-#include 
+#ifdef _WIN32
+#  include 
+#  include 
+#  define _RPy_ThreadLocals_Get  TlsGetValue
+#  define _RPy_ThreadLocals_Set  TlsSetValue
+RPY_EXTERN DWORD pypy_threadlocal_key;
+#else
+#  include 
+#  define _RPy_ThreadLocals_Get  pthread_getspecific
+#  define _RPy_ThreadLocals_Set  pthread_setspecific
+RPY_EXTERN pthread_key_t pypy_threadlocal_key;
+#endif
 
-RPY_EXTERN pthread_key_t pypy_threadlocal_key;
-#define OP_THREADLOCALREF_ADDR(r)    r = (char *)pthread_getspecific(  \
-                                           pypy_threadlocal_key)
-#define OP_THREADLOCALREF_MAKE(r)                       \
-    (OP_THREADLOCALREF_ADDR(r),                         \
-     ((r) || (r = _RPython_ThreadLocals_Build())))
+
+#define OP_THREADLOCALREF_ADDR(r)               \
+    do {                                        \
+        r = (char *)_RPy_ThreadLocals_Get();    \
+        if (!r)                                 \
+            r = _RPython_ThreadLocals_Build();  \
+    } while (0)
+
+#define RPY_THREADLOCALREF_ENSURE()             \
+    if (!_RPy_ThreadLocals_Get())               \
+        (void)_RPython_ThreadLocals_Build();
+
+#define RPY_THREADLOCALREF_GET(FIELD)           \
+    ((struct pypy_threadlocal_s *)_RPy_ThreadLocals_Get())->FIELD
 
 
 /* ------------------------------------------------------------ */

From noreply at buildbot.pypy.org  Mon Nov 24 17:05:49 2014
From: noreply at buildbot.pypy.org (fijal)
Date: Mon, 24 Nov 2014 17:05:49 +0100 (CET)
Subject: [pypy-commit] pypy optresult: whack whack whack
Message-ID: <20141124160549.3D0AE1D2513@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: optresult
Changeset: r74659:813c9b629849
Date: 2014-11-24 18:05 +0200
http://bitbucket.org/pypy/pypy/changeset/813c9b629849/

Log:	whack whack whack

diff --git a/rpython/jit/backend/llgraph/runner.py b/rpython/jit/backend/llgraph/runner.py
--- a/rpython/jit/backend/llgraph/runner.py
+++ b/rpython/jit/backend/llgraph/runner.py
@@ -338,10 +338,10 @@
         values = []
         for box in frame.force_guard_op.getfailargs():
             if box is not None:
-                if box is not frame.current_op.result:
+                if box is not frame.current_op:
                     value = frame.env[box]
                 else:
-                    value = box.value    # 0 or 0.0 or NULL
+                    value = box.getvalue()    # 0 or 0.0 or NULL
             else:
                 value = None
             values.append(value)
@@ -679,7 +679,7 @@
 
 
 class LLFrame(object):
-    _TYPE = llmemory.GCREF
+    _TYPE = lltype.Signed
 
     forced_deadframe = None
     overflow_flag = False
@@ -912,7 +912,7 @@
         if not cond:
             return
         # cond_call can't have a return value
-        self.execute_call(calldescr, func, *args)
+        self.execute_call_n(calldescr, func, *args)
 
     def _execute_call(self, calldescr, func, *args):
         effectinfo = calldescr.get_extra_info()
@@ -935,15 +935,20 @@
     execute_call_f = _execute_call
     execute_call_n = _execute_call
 
-    def execute_call_may_force(self, calldescr, func, *args):
+    def _execute_call_may_force(self, calldescr, func, *args):
         call_op = self.lltrace.operations[self.current_index]
         guard_op = self.lltrace.operations[self.current_index + 1]
         assert guard_op.getopnum() == rop.GUARD_NOT_FORCED
         self.force_guard_op = guard_op
-        res = self.execute_call(calldescr, func, *args)
+        res = self._execute_call(calldescr, func, *args)
         del self.force_guard_op
         return res
 
+    execute_call_may_force_n = _execute_call_may_force
+    execute_call_may_force_r = _execute_call_may_force
+    execute_call_may_force_f = _execute_call_may_force
+    execute_call_may_force_i = _execute_call_may_force
+
     def execute_call_release_gil(self, descr, func, *args):
         if hasattr(descr, '_original_func_'):
             func = descr._original_func_     # see pyjitpl.py
diff --git a/rpython/jit/backend/test/runner_test.py b/rpython/jit/backend/test/runner_test.py
--- a/rpython/jit/backend/test/runner_test.py
+++ b/rpython/jit/backend/test/runner_test.py
@@ -37,13 +37,13 @@
         return InputArgInt(op.getint())
     elif op.type == 'r':
         return InputArgRef(op.getref_base())
-    xxx
+    return InputArgFloat(op.getfloatstorage())
 
 def boxlonglong(ll):
     if longlong.is_64_bit:
-        return BoxInt(ll)
+        return InputArgInt(ll)
     else:
-        return BoxFloat(ll)
+        return InputArgFloat(ll)
 
 STUFF = lltype.GcStruct('STUFF')
 random_gcref = lltype.cast_opaque_ptr(llmemory.GCREF,
@@ -1312,35 +1312,32 @@
     def test_compile_bridge_float(self):
         if not self.cpu.supports_floats:
             py.test.skip("requires floats")
-        fboxes = [BoxFloat() for i in range(12)]
-        i2 = InputArgInt()
         targettoken = TargetToken()
         faildescr1 = BasicFailDescr(1)
         faildescr2 = BasicFailDescr(2)
         faildescr3 = BasicFinalDescr(3)
-        operations = [
-            ResOperation(rop.LABEL, fboxes, None, descr=targettoken),
-            ResOperation(rop.FLOAT_LE, [fboxes[0], constfloat(9.2)], i2),
-            ResOperation(rop.GUARD_TRUE, [i2], None, descr=faildescr1),
-            ResOperation(rop.GUARD_FALSE, [i2], None, descr=faildescr2),
-            ResOperation(rop.FINISH, [], None, descr=faildescr3),
-            ]
-        operations[-3].setfailargs(fboxes)
-        operations[-2].setfailargs(fboxes)
+        loop = parse("""
+        [f0, f1, f2, f3, f4, f5, f6, f7, f8, f9, f10, f11]
+        label(f0, f1, f2, f3, f4, f5, f6, f7, f8, f9, f10, f11, descr=targettoken)
+        i2 = float_le(f0, 9.2)
+        guard_true(i2, descr=faildescr1) [f0, f1, f2, f3, f4, f5, f6, f7, f8, f9, f10, f11]
+        guard_false(i2, descr=faildescr2) [f0, f1, f2, f3, f4, f5, f6, f7, f8, f9, f10, f11]
+        finish(descr=faildescr3)
+        """, namespace=locals())
         looptoken = JitCellToken()
-        self.cpu.compile_loop(fboxes, operations, looptoken)
-
-        fboxes2 = [BoxFloat() for i in range(12)]
-        f3 = BoxFloat()
-        bridge = [
-            ResOperation(rop.FLOAT_SUB, [fboxes2[0], constfloat(1.0)], f3),
-            ResOperation(rop.JUMP, [f3]+fboxes2[1:], None, descr=targettoken),
-        ]
-
-        self.cpu.compile_bridge(faildescr1, fboxes2, bridge, looptoken)
+        self.cpu.compile_loop(loop.inputargs, loop.operations, looptoken)
+
+        bridge = parse("""
+        [f0, f1, f2, f3, f4, f5, f6, f7, f8, f9, f10, f11]
+        f15 = float_sub(f0, 1.0)
+        jump(f15, f1, f2, f3, f4, f5, f6, f7, f8, f9, f10, f11, descr=targettoken)
+        """, namespace=locals())
+
+        self.cpu.compile_bridge(faildescr1, bridge.inputargs,
+                                bridge.operations, looptoken)
 
         args = []
-        for i in range(len(fboxes)):
+        for i in range(len(loop.inputargs)):
             x = 13.5 + 6.73 * i
             args.append(longlong.getfloatstorage(x))
         deadframe = self.cpu.execute_token(looptoken, *args)
@@ -1348,7 +1345,7 @@
         assert fail.identifier == 2
         res = self.cpu.get_float_value(deadframe, 0)
         assert longlong.getrealfloat(res) == 8.5
-        for i in range(1, len(fboxes)):
+        for i in range(1, len(loop.inputargs)):
             got = longlong.getrealfloat(self.cpu.get_float_value(
                 deadframe, i))
             assert got == 13.5 + 6.73 * i
@@ -1356,7 +1353,6 @@
     def test_compile_bridge_spilled_float(self):
         if not self.cpu.supports_floats:
             py.test.skip("requires floats")
-        fboxes = [BoxFloat() for i in range(3)]
         faildescr1 = BasicFailDescr(100)
         faildescr2 = BasicFinalDescr(102)
         loopops = """
@@ -1383,15 +1379,15 @@
         assert longlong.getrealfloat(f2) == 0.75
         assert longlong.getrealfloat(f3) == 133.0
 
-        zero = InputArgInt()
-        bridgeops = [
-            ResOperation(rop.SAME_AS, [ConstInt(0)], zero),
-            ResOperation(rop.GUARD_TRUE, [zero], None, descr=faildescr1),
-            ResOperation(rop.FINISH, [], None, descr=faildescr2),
-            ]
-        bridgeops[-2].setfailargs(fboxes[:])
-        self.cpu.compile_bridge(loop.operations[-2].getdescr(), fboxes,
-                                                        bridgeops, looptoken)
+        bridge = parse("""
+        [f1, f2, f3]
+        i0 = same_as_i(0)
+        guard_true(i0, descr=faildescr1) [f1, f2, f3]
+        finish(descr=faildescr2)
+        """, namespace=locals())
+        self.cpu.compile_bridge(loop.operations[-2].getdescr(),
+                                bridge.inputargs, bridge.operations,
+                                looptoken)
         args = [1,
                 longlong.getfloatstorage(132.25),
                 longlong.getfloatstorage(0.75)]
@@ -1414,16 +1410,14 @@
                 (rop.GUARD_TRUE,  True),
                 ]:
                 box = InputArgInt()
-                res = InputArgInt()
                 faildescr1 = BasicFailDescr(1)
                 faildescr2 = BasicFinalDescr(2)
                 inputargs = [box]
-                operations = [
-                    ResOperation(opname, [box], res),
-                    ResOperation(opguard, [res], None, descr=faildescr1),
-                    ResOperation(rop.FINISH, [], None, descr=faildescr2),
-                    ]
-                operations[1].setfailargs([])
+                op0 = ResOperation(opname, [box])
+                op1 = ResOperation(opguard, [op0], descr=faildescr1)
+                op2 = ResOperation(rop.FINISH, [], descr=faildescr2)
+                operations = [op0, op1, op2]
+                op1.setfailargs([])
                 looptoken = JitCellToken()
                 self.cpu.compile_loop(inputargs, operations, looptoken)
                 #
@@ -1458,16 +1452,14 @@
                         ibox2 = InputArgInt()
                     else:
                         ibox2 = ConstInt(-42)
-                    b1 = InputArgInt()
                     faildescr1 = BasicFailDescr(1)
                     faildescr2 = BasicFinalDescr(2)
                     inputargs = [ib for ib in [ibox1, ibox2]
                                     if isinstance(ib, InputArgInt)]
-                    operations = [
-                        ResOperation(opname, [ibox1, ibox2], b1),
-                        ResOperation(opguard, [b1], None, descr=faildescr1),
-                        ResOperation(rop.FINISH, [], None, descr=faildescr2),
-                        ]
+                    op0 = ResOperation(opname, [ibox1, ibox2])
+                    op1 = ResOperation(opguard, [op0], descr=faildescr1)
+                    op2 = ResOperation(rop.FINISH, [], descr=faildescr2)
+                    operations = [op0, op1, op2]
                     operations[-2].setfailargs([])
                     looptoken = JitCellToken()
                     self.cpu.compile_loop(inputargs, operations, looptoken)
@@ -1510,16 +1502,14 @@
                         ibox2 = InputArgInt()
                     else:
                         ibox2 = ConstInt(42)
-                    b1 = InputArgInt()
                     faildescr1 = BasicFailDescr(1)
                     faildescr2 = BasicFinalDescr(2)
                     inputargs = [ib for ib in [ibox1, ibox2]
                                     if isinstance(ib, InputArgInt)]
-                    operations = [
-                        ResOperation(opname, [ibox1, ibox2], b1),
-                        ResOperation(opguard, [b1], None, descr=faildescr1),
-                        ResOperation(rop.FINISH, [], None, descr=faildescr2),
-                        ]
+                    op0 = ResOperation(opname, [ibox1, ibox2])
+                    op1 = ResOperation(opguard, [op0], descr=faildescr1)
+                    op2 = ResOperation(rop.FINISH, [], descr=faildescr2)
+                    operations = [op0, op1, op2]
                     operations[-2].setfailargs([])
                     looptoken = JitCellToken()
                     self.cpu.compile_loop(inputargs, operations, looptoken)
@@ -1559,23 +1549,21 @@
                 for combinaison in ["bb", "bc", "cb"]:
                     #
                     if combinaison[0] == 'b':
-                        fbox1 = BoxFloat()
+                        fbox1 = InputArgFloat()
                     else:
                         fbox1 = constfloat(-4.5)
                     if combinaison[1] == 'b':
-                        fbox2 = BoxFloat()
+                        fbox2 = InputArgFloat()
                     else:
                         fbox2 = constfloat(-4.5)
-                    b1 = InputArgInt()
                     faildescr1 = BasicFailDescr(1)
                     faildescr2 = BasicFinalDescr(2)
                     inputargs = [fb for fb in [fbox1, fbox2]
-                                    if isinstance(fb, BoxFloat)]
-                    operations = [
-                        ResOperation(opname, [fbox1, fbox2], b1),
-                        ResOperation(opguard, [b1], None, descr=faildescr1),
-                        ResOperation(rop.FINISH, [], None, descr=faildescr2),
-                        ]
+                                    if not isinstance(fb, Const)]
+                    op0 = ResOperation(opname, [fbox1, fbox2])
+                    op1 = ResOperation(opguard, [op0], descr=faildescr1)
+                    op2 = ResOperation(rop.FINISH, [], descr=faildescr2)
+                    operations = [op0, op1, op2]
                     operations[-2].setfailargs([])
                     looptoken = JitCellToken()
                     self.cpu.compile_loop(inputargs, operations, looptoken)
@@ -1622,19 +1610,12 @@
         inputargs = []
         operations = []
         for opnum, boxargs, rettype, retvalue in tests:
-            inputargs += [box for box in boxargs if isinstance(box, Box)]
-            if rettype == 'int':
-                boxres = InputArgInt()
-            elif rettype == 'float':
-                boxres = BoxFloat()
-            else:
-                assert 0
-            operations.append(ResOperation(opnum, boxargs, boxres))
+            inputargs += [box for box in boxargs if not isinstance(box, Const)]
+            operations.append(ResOperation(opnum, boxargs))
         # Unique-ify inputargs
         inputargs = list(set(inputargs))
         faildescr = BasicFinalDescr(1)
-        operations.append(ResOperation(rop.FINISH, [], None,
-                                       descr=faildescr))
+        operations.append(ResOperation(rop.FINISH, [], descr=faildescr))
         looptoken = JitCellToken()
         #
         self.cpu.compile_loop(inputargs, operations, looptoken)
@@ -1643,7 +1624,7 @@
         for box in inputargs:
             if isinstance(box, InputArgInt):
                 args.append(box.getint())
-            elif isinstance(box, BoxFloat):
+            elif isinstance(box, InputArgFloat):
                 args.append(box.getfloatstorage())
             else:
                 assert 0
@@ -1674,46 +1655,43 @@
 
         def nan_and_infinity(opnum, realoperation, testcases):
             for testcase in testcases:
-                realvalues = [b.getfloat() for b in testcase]
-                expected = realoperation(*realvalues)
+                expected = realoperation(*testcase)
+                inputargs = [boxfloat(x) for x in testcase]
                 if isinstance(expected, float):
                     expectedtype = 'float'
                 else:
                     expectedtype = 'int'
-                got = self.execute_operation(opnum, list(testcase),
+                got = self.execute_operation(opnum, inputargs,
                                              expectedtype)
                 if isnan(expected):
-                    ok = isnan(got.getfloat())
+                    ok = isnan(got)
                 elif isinf(expected):
-                    ok = isinf(got.getfloat())
-                elif isinstance(got, BoxFloat):
-                    ok = (got.getfloat() == expected)
+                    ok = isinf(got)
                 else:
-                    ok = got.value == expected
+                    ok = got == expected
                 if not ok:
                     raise AssertionError("%s(%s): got %r, expected %r" % (
-                        opname[opnum], ', '.join(map(repr, realvalues)),
-                        got.getfloat(), expected))
+                        opname[opnum], ', '.join(map(repr, testcase)),
+                        got, expected))
                 # if we expect a boolean, also check the combination with
                 # a GUARD_TRUE or GUARD_FALSE
                 if isinstance(expected, bool):
                     for guard_opnum, expected_id in [(rop.GUARD_TRUE, 1),
                                                      (rop.GUARD_FALSE, 0)]:
-                        box = InputArgInt()
-                        operations = [
-                            ResOperation(opnum, list(testcase), box),
-                            ResOperation(guard_opnum, [box], None,
-                                         descr=BasicFailDescr(4)),
-                            ResOperation(rop.FINISH, [], None,
-                                         descr=BasicFinalDescr(5))]
+                        op0 = ResOperation(opnum, inputargs)
+                        op1 = ResOperation(guard_opnum, [op0],
+                                           descr=BasicFailDescr(4))
+                        op2 = ResOperation(rop.FINISH, [],
+                                           descr=BasicFinalDescr(5))
+                        operations = [op0, op1, op2]
                         operations[1].setfailargs([])
                         looptoken = JitCellToken()
                         # Use "set" to unique-ify inputargs
-                        unique_testcase_list = list(set(testcase))
+                        unique_testcase_list = list(set(inputargs))
                         self.cpu.compile_loop(unique_testcase_list, operations,
                                               looptoken)
-                        args = [box.getfloatstorage()
-                                for box in unique_testcase_list]
+                        args = [box.getfloatstorage() for box in
+                                unique_testcase_list]
                         deadframe = self.cpu.execute_token(looptoken, *args)
                         fail = self.cpu.get_latest_descr(deadframe)
                         if fail.identifier != 5 - (expected_id^expected):
@@ -1725,7 +1703,7 @@
                                 "%s(%s)/%s took the wrong path: "
                                 "the failure path of the guard %s" % (
                                     opname[opnum],
-                                    ', '.join(map(repr, realvalues)),
+                                    ', '.join(map(repr, testcase)),
                                     opname[guard_opnum], msg))
 
         yield nan_and_infinity, rop.FLOAT_ADD, operator.add, all_cases_binary
@@ -1743,7 +1721,7 @@
         yield nan_and_infinity, rop.FLOAT_GE,  operator.ge,  all_cases_binary
 
     def test_noops(self):
-        c_box = self.alloc_string("hi there").constbox()
+        c_box = wrap_constant(self.alloc_string("hi there").getref_base())
         c_nest = ConstInt(0)
         c_id = ConstInt(0)
         self.execute_operation(rop.DEBUG_MERGE_POINT, [c_box, c_nest, c_id], 'void')
@@ -1782,17 +1760,13 @@
 
         calldescr = self.cpu.calldescrof(FUNC, FUNC.ARGS, FUNC.RESULT, effectinfo)
         if longlong.is_64_bit:
-            got1 = self.execute_operation(rop.CALL, [funcbox], 'int', calldescr)
+            res1 = self.execute_operation(rop.CALL_I, [funcbox], 'int', calldescr)
             wait_a_bit()
-            got2 = self.execute_operation(rop.CALL, [funcbox], 'int', calldescr)
-            res1 = got1.getint()
-            res2 = got2.getint()
+            res2 = self.execute_operation(rop.CALL_I, [funcbox], 'int', calldescr)
         else:
-            got1 = self.execute_operation(rop.CALL, [funcbox],'float',calldescr)
+            res1 = self.execute_operation(rop.CALL_I, [funcbox],'float',calldescr)
             wait_a_bit()
-            got2 = self.execute_operation(rop.CALL, [funcbox],'float',calldescr)
-            res1 = got1.getlonglong()
-            res2 = got2.getlonglong()
+            res2 = self.execute_operation(rop.CALL_I, [funcbox],'float',calldescr)
         assert res1 < res2 < res1 + 2**32
 
 
@@ -1876,13 +1850,13 @@
 
     def test_cast_int_to_ptr(self):
         res = self.execute_operation(rop.CAST_INT_TO_PTR,
-                                     [InputArgInt(-17)],  'ref').value
+                                     [InputArgInt(-17)],  'ref')
         assert lltype.cast_ptr_to_int(res) == -17
 
     def test_cast_ptr_to_int(self):
         x = lltype.cast_int_to_ptr(llmemory.GCREF, -19)
         res = self.execute_operation(rop.CAST_PTR_TO_INT,
-                                     [BoxPtr(x)], 'int').value
+                                     [InputArgRef(x)], 'int')
         assert res == -19
 
     def test_cast_int_to_float(self):
@@ -1890,7 +1864,7 @@
             py.test.skip("requires floats")
         for x in [-10, -1, 0, 3, 42, sys.maxint-1]:
             res = self.execute_operation(rop.CAST_INT_TO_FLOAT,
-                                         [InputArgInt(x)],  'float').value
+                                         [InputArgInt(x)],  'float')
             assert longlong.getrealfloat(res) == float(x)
             # --- the front-end never generates CAST_INT_TO_FLOAT(Const)
             #res = self.execute_operation(rop.CAST_INT_TO_FLOAT,
@@ -1903,7 +1877,7 @@
         for x in [-24.23, -5.3, 0.0, 3.1234, 11.1, 0.1]:
             v = longlong.getfloatstorage(x)
             res = self.execute_operation(rop.CAST_FLOAT_TO_INT,
-                                         [BoxFloat(v)],  'int').value
+                                         [InputArgFloat(v)],  'int')
             assert res == int(x)
             # --- the front-end never generates CAST_FLOAT_TO_INT(Const)
             #res = self.execute_operation(rop.CAST_FLOAT_TO_INT,
@@ -1917,21 +1891,21 @@
             py.test.skip("longlong test")
         t = 'int' if longlong.is_64_bit else 'float'
         res = self.execute_operation(rop.CONVERT_FLOAT_BYTES_TO_LONGLONG,
-                                     [boxfloat(2.5)], t).value
+                                     [boxfloat(2.5)], t)
         assert res == longlong2float.float2longlong(2.5)
 
         bytes = longlong2float.float2longlong(2.5)
         res = self.execute_operation(rop.CONVERT_LONGLONG_BYTES_TO_FLOAT,
-                                     [boxlonglong(res)], 'float').value
+                                     [boxlonglong(bytes)], 'float')
         assert longlong.getrealfloat(res) == 2.5
 
     def test_ooops_non_gc(self):
         x = lltype.malloc(lltype.Struct('x'), flavor='raw')
         v = heaptracker.adr2int(llmemory.cast_ptr_to_adr(x))
         r = self.execute_operation(rop.PTR_EQ, [InputArgInt(v), InputArgInt(v)], 'int')
-        assert r.value == 1
+        assert r == 1
         r = self.execute_operation(rop.PTR_NE, [InputArgInt(v), InputArgInt(v)], 'int')
-        assert r.value == 0
+        assert r == 0
         lltype.free(x, flavor='raw')
 
     def test_new_plain_struct(self):
@@ -1940,14 +1914,16 @@
         sizedescr = cpu.sizeof(S)
         r1 = self.execute_operation(rop.NEW, [], 'ref', descr=sizedescr)
         r2 = self.execute_operation(rop.NEW, [], 'ref', descr=sizedescr)
-        assert r1.value != r2.value
+        assert r1 != r2
         xdescr = cpu.fielddescrof(S, 'x')
         ydescr = cpu.fielddescrof(S, 'y')
-        self.execute_operation(rop.SETFIELD_GC, [r1, InputArgInt(150)],
+        self.execute_operation(rop.SETFIELD_GC, [InputArgRef(r1),
+                                                 InputArgInt(150)],
                                'void', descr=ydescr)
-        self.execute_operation(rop.SETFIELD_GC, [r1, InputArgInt(190)],
+        self.execute_operation(rop.SETFIELD_GC, [InputArgRef(r1),
+                                                 InputArgInt(190)],
                                'void', descr=xdescr)
-        s = lltype.cast_opaque_ptr(lltype.Ptr(S), r1.value)
+        s = lltype.cast_opaque_ptr(lltype.Ptr(S), r1)
         assert s.x == chr(190)
         assert s.y == chr(150)
 
@@ -1955,32 +1931,35 @@
         cpu = self.cpu
         t_box, T_box = self.alloc_instance(self.T)
         vtable = llmemory.cast_adr_to_ptr(
-            llmemory.cast_int_to_adr(T_box.value), heaptracker.VTABLETYPE)
+            llmemory.cast_int_to_adr(T_box.getint()), heaptracker.VTABLETYPE)
         heaptracker.register_known_gctype(cpu, vtable, self.T)
         r1 = self.execute_operation(rop.NEW_WITH_VTABLE, [T_box], 'ref')
         r2 = self.execute_operation(rop.NEW_WITH_VTABLE, [T_box], 'ref')
-        assert r1.value != r2.value
+        assert r1 != r2
         descr1 = cpu.fielddescrof(self.S, 'chr1')
         descr2 = cpu.fielddescrof(self.S, 'chr2')
         descrshort = cpu.fielddescrof(self.S, 'short')
-        self.execute_operation(rop.SETFIELD_GC, [r1, InputArgInt(150)],
+        self.execute_operation(rop.SETFIELD_GC, [InputArgRef(r1),
+                                                 InputArgInt(150)],
                                'void', descr=descr2)
-        self.execute_operation(rop.SETFIELD_GC, [r1, InputArgInt(190)],
+        self.execute_operation(rop.SETFIELD_GC, [InputArgRef(r1),
+                                                 InputArgInt(190)],
                                'void', descr=descr1)
-        self.execute_operation(rop.SETFIELD_GC, [r1, InputArgInt(1313)],
+        self.execute_operation(rop.SETFIELD_GC, [InputArgRef(r1),
+                                                 InputArgInt(1313)],
                                'void', descr=descrshort)
-        s = lltype.cast_opaque_ptr(lltype.Ptr(self.T), r1.value)
+        s = lltype.cast_opaque_ptr(lltype.Ptr(self.T), r1)
         assert s.parent.chr1 == chr(190)
         assert s.parent.chr2 == chr(150)
-        r = self.cpu.bh_getfield_gc_i(r1.value, descrshort)
+        r = self.cpu.bh_getfield_gc_i(r1, descrshort)
         assert r == 1313
-        self.cpu.bh_setfield_gc_i(r1.value, 1333, descrshort)
-        r = self.cpu.bh_getfield_gc_i(r1.value, descrshort)
+        self.cpu.bh_setfield_gc_i(r1, 1333, descrshort)
+        r = self.cpu.bh_getfield_gc_i(r1, descrshort)
         assert r == 1333
-        r = self.execute_operation(rop.GETFIELD_GC, [r1], 'int',
+        r = self.execute_operation(rop.GETFIELD_GC_I, [InputArgRef(r1)], 'int',
                                    descr=descrshort)
-        assert r.value == 1333
-        t = lltype.cast_opaque_ptr(lltype.Ptr(self.T), t_box.value)
+        assert r == 1333
+        t = lltype.cast_opaque_ptr(lltype.Ptr(self.T), t_box.getref_base())
         assert s.parent.parent.typeptr == t.parent.parent.typeptr
 
     def test_new_array(self):
@@ -1990,8 +1969,8 @@
                                     'ref', descr=arraydescr)
         r2 = self.execute_operation(rop.NEW_ARRAY, [InputArgInt(342)],
                                     'ref', descr=arraydescr)
-        assert r1.value != r2.value
-        a = lltype.cast_opaque_ptr(lltype.Ptr(A), r1.value)
+        assert r1 != r2
+        a = lltype.cast_opaque_ptr(lltype.Ptr(A), r1)
         assert len(a) == 342
 
     def test_new_array_clear(self):
@@ -1999,22 +1978,22 @@
         arraydescr = self.cpu.arraydescrof(A)
         r1 = self.execute_operation(rop.NEW_ARRAY_CLEAR, [InputArgInt(342)],
                                     'ref', descr=arraydescr)
-        a = lltype.cast_opaque_ptr(lltype.Ptr(A), r1.value)
+        a = lltype.cast_opaque_ptr(lltype.Ptr(A), r1)
         assert a[0] == 0
         assert len(a) == 342
 
     def test_new_string(self):
         r1 = self.execute_operation(rop.NEWSTR, [InputArgInt(342)], 'ref')
         r2 = self.execute_operation(rop.NEWSTR, [InputArgInt(342)], 'ref')
-        assert r1.value != r2.value
-        a = lltype.cast_opaque_ptr(lltype.Ptr(rstr.STR), r1.value)
+        assert r1 != r2
+        a = lltype.cast_opaque_ptr(lltype.Ptr(rstr.STR), r1)
         assert len(a.chars) == 342
 
     def test_new_unicode(self):
         r1 = self.execute_operation(rop.NEWUNICODE, [InputArgInt(342)], 'ref')
         r2 = self.execute_operation(rop.NEWUNICODE, [InputArgInt(342)], 'ref')
-        assert r1.value != r2.value
-        a = lltype.cast_opaque_ptr(lltype.Ptr(rstr.UNICODE), r1.value)
+        assert r1 != r2
+        a = lltype.cast_opaque_ptr(lltype.Ptr(rstr.UNICODE), r1)
         assert len(a.chars) == 342
 
     def test_exceptions(self):
@@ -2026,8 +2005,8 @@
 
         ops = '''
         [i0]
-        i1 = same_as(1)
-        call(ConstClass(fptr), i0, descr=calldescr)
+        i1 = same_as_i(1)
+        call_n(ConstClass(fptr), i0, descr=calldescr)
         p0 = guard_exception(ConstClass(xtp)) [i1]
         finish(p0)
         '''
@@ -2081,8 +2060,8 @@
         exc_ptr = xptr
         ops = '''
         [i0]
-        i1 = same_as(1)
-        call(ConstClass(fptr), i0, descr=calldescr)
+        i1 = same_as_i(1)
+        call_n(ConstClass(fptr), i0, descr=calldescr)
         guard_no_exception() [i1]
         finish(0)
         '''
@@ -2125,7 +2104,7 @@
             sgcref = lltype.cast_opaque_ptr(llmemory.GCREF, s)
             del record[:]
             self.execute_operation(rop.COND_CALL_GC_WB,
-                                   [BoxPtr(sgcref)],
+                                   [InputArgRef(sgcref)],
                                    'void', descr=WriteBarrierDescr())
             if cond:
                 assert record == [rffi.cast(lltype.Signed, sgcref)]
@@ -2160,7 +2139,7 @@
             sgcref = lltype.cast_opaque_ptr(llmemory.GCREF, s)
             del record[:]
             self.execute_operation(rop.COND_CALL_GC_WB_ARRAY,
-                       [BoxPtr(sgcref), ConstInt(123)],
+                       [InputArgRef(sgcref), ConstInt(123)],
                        'void', descr=WriteBarrierDescr())
             if cond:
                 assert record == [rffi.cast(lltype.Signed, sgcref)]
@@ -2225,7 +2204,7 @@
                 del record[:]
                 box_index = BoxIndexCls((9<<7) + 17)
                 self.execute_operation(rop.COND_CALL_GC_WB_ARRAY,
-                           [BoxPtr(sgcref), box_index],
+                           [InputArgRef(sgcref), box_index],
                            'void', descr=WriteBarrierDescr())
                 if cond in [0, 1]:
                     assert record == [rffi.cast(lltype.Signed, s.data)]
@@ -2293,26 +2272,22 @@
                 values.append(self.cpu.get_int_value(deadframe, 1))
                 self.cpu.set_savedata_ref(deadframe, random_gcref)
 
-        FUNC = self.FuncType([llmemory.GCREF, lltype.Signed], lltype.Void)
+        FUNC = self.FuncType([lltype.Signed, lltype.Signed], lltype.Void)
         func_ptr = llhelper(lltype.Ptr(FUNC), maybe_force)
-        funcbox = self.get_funcbox(self.cpu, func_ptr).constbox()
         calldescr = self.cpu.calldescrof(FUNC, FUNC.ARGS, FUNC.RESULT,
                                          EffectInfo.MOST_GENERAL)
         cpu = self.cpu
-        i0 = InputArgInt()
-        i1 = InputArgInt()
-        tok = BoxPtr()
         faildescr = BasicFailDescr(1)
-        ops = [
-        ResOperation(rop.FORCE_TOKEN, [], tok),
-        ResOperation(rop.CALL_MAY_FORCE, [funcbox, tok, i1], None,
-                     descr=calldescr),
-        ResOperation(rop.GUARD_NOT_FORCED, [], None, descr=faildescr),
-        ResOperation(rop.FINISH, [i0], None, descr=BasicFinalDescr(0))
-        ]
-        ops[2].setfailargs([i1, i0])
+        finaldescr = BasicFinalDescr(0)
+        loop = parse("""
+        [i0, i1]
+        i2 = force_token()
+        call_may_force_n(ConstClass(func_ptr), i2, i1, descr=calldescr)
+        guard_not_forced(descr=faildescr) [i1, i0]
+        finish(i0, descr=finaldescr)
+        """, namespace=locals())
         looptoken = JitCellToken()
-        self.cpu.compile_loop([i0, i1], ops, looptoken)
+        self.cpu.compile_loop(loop.inputargs, loop.operations, looptoken)
         deadframe = self.cpu.execute_token(looptoken, 20, 0)
         fail = self.cpu.get_latest_descr(deadframe)
         assert fail.identifier == 0
@@ -2338,27 +2313,22 @@
                 self.cpu.set_savedata_ref(deadframe, random_gcref)
             return 42
 
-        FUNC = self.FuncType([llmemory.GCREF, lltype.Signed], lltype.Signed)
+        FUNC = self.FuncType([lltype.Signed, lltype.Signed], lltype.Signed)
         func_ptr = llhelper(lltype.Ptr(FUNC), maybe_force)
-        funcbox = self.get_funcbox(self.cpu, func_ptr).constbox()
         calldescr = self.cpu.calldescrof(FUNC, FUNC.ARGS, FUNC.RESULT,
                                          EffectInfo.MOST_GENERAL)
         cpu = self.cpu
-        i0 = InputArgInt()
-        i1 = InputArgInt()
-        i2 = InputArgInt()
-        tok = BoxPtr()
         faildescr = BasicFailDescr(1)
-        ops = [
-        ResOperation(rop.FORCE_TOKEN, [], tok),
-        ResOperation(rop.CALL_MAY_FORCE, [funcbox, tok, i1], i2,
-                     descr=calldescr),
-        ResOperation(rop.GUARD_NOT_FORCED, [], None, descr=faildescr),
-        ResOperation(rop.FINISH, [i2], None, descr=BasicFinalDescr(0))
-        ]
-        ops[2].setfailargs([i1, i2, i0])
+        finaldescr = BasicFinalDescr(0)
+        loop = parse("""
+        [i0, i1]
+        i3 = force_token()
+        i2 = call_may_force_i(ConstClass(func_ptr), i3, i1, descr=calldescr)
+        guard_not_forced(descr=faildescr) [i1, i2, i0]
+        finish(i2, descr=finaldescr)
+        """, namespace=locals())
         looptoken = JitCellToken()
-        self.cpu.compile_loop([i0, i1], ops, looptoken)
+        self.cpu.compile_loop(loop.inputargs, loop.operations, looptoken)
         deadframe = self.cpu.execute_token(looptoken, 20, 0)
         fail = self.cpu.get_latest_descr(deadframe)
         assert fail.identifier == 0
@@ -2386,27 +2356,23 @@
                 self.cpu.set_savedata_ref(deadframe, random_gcref)
             return 42.5
 
-        FUNC = self.FuncType([llmemory.GCREF, lltype.Signed], lltype.Float)
+        FUNC = self.FuncType([lltype.Signed, lltype.Signed], lltype.Float)
         func_ptr = llhelper(lltype.Ptr(FUNC), maybe_force)
         funcbox = self.get_funcbox(self.cpu, func_ptr).constbox()
         calldescr = self.cpu.calldescrof(FUNC, FUNC.ARGS, FUNC.RESULT,
                                          EffectInfo.MOST_GENERAL)
         cpu = self.cpu
-        i0 = InputArgInt()
-        i1 = InputArgInt()
-        f2 = BoxFloat()
-        tok = BoxPtr()
         faildescr = BasicFailDescr(1)
-        ops = [
-        ResOperation(rop.FORCE_TOKEN, [], tok),
-        ResOperation(rop.CALL_MAY_FORCE, [funcbox, tok, i1], f2,
-                     descr=calldescr),
-        ResOperation(rop.GUARD_NOT_FORCED, [], None, descr=faildescr),
-        ResOperation(rop.FINISH, [f2], None, descr=BasicFinalDescr(0))
-        ]
-        ops[2].setfailargs([i1, f2, i0])
+        finaldescr = BasicFinalDescr(0)
+        loop = parse("""
+        [i0, i1]
+        i3 = force_token()
+        f2 = call_may_force_f(ConstClass(func_ptr), i3, i1, descr=calldescr)
+        guard_not_forced(descr=faildescr) [i1, f2, i0]
+        finish(f2, descr=finaldescr)
+        """, namespace=locals())
         looptoken = JitCellToken()
-        self.cpu.compile_loop([i0, i1], ops, looptoken)
+        self.cpu.compile_loop(loop.inputargs, loop.operations, looptoken)
         deadframe = self.cpu.execute_token(looptoken, 20, 0)
         fail = self.cpu.get_latest_descr(deadframe)
         assert fail.identifier == 0
@@ -2426,23 +2392,21 @@
 
     def test_guard_not_forced_2(self):
         cpu = self.cpu
-        i0 = InputArgInt()
-        i1 = InputArgInt()
-        tok = BoxPtr()
         faildescr = BasicFailDescr(1)
-        ops = [
-        ResOperation(rop.INT_ADD, [i0, ConstInt(10)], i1),
-        ResOperation(rop.FORCE_TOKEN, [], tok),
-        ResOperation(rop.GUARD_NOT_FORCED_2, [], None, descr=faildescr),
-        ResOperation(rop.FINISH, [tok], None, descr=BasicFinalDescr(0))
-        ]
-        ops[-2].setfailargs([i1])
+        finaldescr = BasicFinalDescr(0)
+        loop = parse("""
+        [i0]
+        i1 = int_add(i0, 10)
+        i2 = force_token()
+        guard_not_forced_2(descr=faildescr) [i1]
+        finish(i2, descr=finaldescr)
+        """, namespace=locals())
         looptoken = JitCellToken()
-        self.cpu.compile_loop([i0], ops, looptoken)
+        self.cpu.compile_loop(loop.inputargs, loop.operations, looptoken)
         deadframe = self.cpu.execute_token(looptoken, 20)
         fail = self.cpu.get_latest_descr(deadframe)
         assert fail.identifier == 0
-        frame = self.cpu.get_ref_value(deadframe, 0)
+        frame = self.cpu.get_int_value(deadframe, 0)
         # actually, we should get the same pointer in 'frame' and 'deadframe'
         # but it is not the case on LLGraph
         if not getattr(self.cpu, 'is_llgraph', False):
diff --git a/rpython/jit/tool/oparser.py b/rpython/jit/tool/oparser.py
--- a/rpython/jit/tool/oparser.py
+++ b/rpython/jit/tool/oparser.py
@@ -44,9 +44,9 @@
         op.initarglist(self.getarglist()[:])
         return op
 
-    def copy_and_change(self, opnum, args=None, result=None, descr=None):
+    def _copy_and_change(self, opnum, args=None, descr=None):
         assert opnum == self.OPNUM
-        newop = FORCE_SPILL(result or self.result)
+        newop = FORCE_SPILL()
         newop.initarglist(args or self.getarglist())
         return newop
 

From noreply at buildbot.pypy.org  Mon Nov 24 17:29:26 2014
From: noreply at buildbot.pypy.org (rlamy)
Date: Mon, 24 Nov 2014 17:29:26 +0100 (CET)
Subject: [pypy-commit] pypy framestate: cache FrameState.mergeable
Message-ID: <20141124162926.BD2C41C3271@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy 
Branch: framestate
Changeset: r74660:67c6cf446c89
Date: 2014-11-21 15:30 +0000
http://bitbucket.org/pypy/pypy/changeset/67c6cf446c89/

Log:	cache FrameState.mergeable

diff --git a/rpython/flowspace/framestate.py b/rpython/flowspace/framestate.py
--- a/rpython/flowspace/framestate.py
+++ b/rpython/flowspace/framestate.py
@@ -22,10 +22,13 @@
         self.last_exception = last_exception
         self.blocklist = blocklist
         self.next_offset = next_offset
+        self._mergeable = None
 
     @property
     def mergeable(self):
-        data = self.locals_w + self.stack
+        if self._mergeable is not None:
+            return self._mergeable
+        self._mergeable = data = self.locals_w + self.stack
         if self.last_exception is None:
             data.append(Constant(None))
             data.append(Constant(None))

From noreply at buildbot.pypy.org  Mon Nov 24 17:29:27 2014
From: noreply at buildbot.pypy.org (rlamy)
Date: Mon, 24 Nov 2014 17:29:27 +0100 (CET)
Subject: [pypy-commit] pypy framestate: Create BCInstruction class
Message-ID: <20141124162927.E7BD51C3271@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy 
Branch: framestate
Changeset: r74661:e6c01f65048e
Date: 2014-11-19 19:15 +0000
http://bitbucket.org/pypy/pypy/changeset/e6c01f65048e/

Log:	Create BCInstruction class

diff --git a/rpython/flowspace/bytecode.py b/rpython/flowspace/bytecode.py
--- a/rpython/flowspace/bytecode.py
+++ b/rpython/flowspace/bytecode.py
@@ -35,8 +35,6 @@
     """
     A wrapper around a native code object of the host interpreter
     """
-    opnames = host_bytecode_spec.method_names
-
     def __init__(self, argcount, nlocals, stacksize, flags,
                  code, consts, names, varnames, filename,
                  name, firstlineno, lnotab, freevars):
@@ -85,7 +83,7 @@
         """
         Decode the instruction starting at position ``offset``.
 
-        Returns (next_offset, opname, oparg).
+        Returns (next_offset, instruction).
         """
         co_code = self.co_code
         opnum = ord(co_code[offset])
@@ -110,9 +108,20 @@
 
         if opnum in opcode.hasjrel:
             oparg += next_offset
-        opname = self.opnames[opnum]
-        return next_offset, opname, oparg
+        return next_offset, BCInstruction(opnum, oparg, pos)
 
     @property
     def is_generator(self):
         return bool(self.co_flags & CO_GENERATOR)
+
+OPNAMES = host_bytecode_spec.method_names
+
+class BCInstruction(object):
+    """A bytecode instruction, comprising an opcode and an optional argument."""
+    def __init__(self, opcode, arg, offset=-1):
+        self.name = OPNAMES[opcode]
+        self.arg = arg
+        self.offset = offset
+
+    def eval(self, ctx):
+        return getattr(ctx, self.name)(self.arg)
diff --git a/rpython/flowspace/flowcontext.py b/rpython/flowspace/flowcontext.py
--- a/rpython/flowspace/flowcontext.py
+++ b/rpython/flowspace/flowcontext.py
@@ -503,9 +503,9 @@
 
     def handle_bytecode(self, next_offset):
         self.last_offset = next_offset
-        next_offset, methodname, oparg = self.pycode.read(next_offset)
+        next_offset, instr = self.pycode.read(next_offset)
         try:
-            offset = getattr(self, methodname)(oparg)
+            res = instr.eval(self)
             return offset if offset is not None else next_offset
         except FlowSignal as signal:
             return self.unroll(signal)

From noreply at buildbot.pypy.org  Mon Nov 24 17:29:29 2014
From: noreply at buildbot.pypy.org (rlamy)
Date: Mon, 24 Nov 2014 17:29:29 +0100 (CET)
Subject: [pypy-commit] pypy framestate: Use consistent terminology for
	offset vs instruction:
Message-ID: <20141124162929.3F0EE1C3271@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy 
Branch: framestate
Changeset: r74662:456a7f470921
Date: 2014-05-04 21:00 +0100
http://bitbucket.org/pypy/pypy/changeset/456a7f470921/

Log:	Use consistent terminology for offset vs instruction:

	offset = index into the code bytestring instruction = an opcode
	together with its argument

diff --git a/rpython/flowspace/bytecode.py b/rpython/flowspace/bytecode.py
--- a/rpython/flowspace/bytecode.py
+++ b/rpython/flowspace/bytecode.py
@@ -108,7 +108,7 @@
 
         if opnum in opcode.hasjrel:
             oparg += next_offset
-        return next_offset, BCInstruction(opnum, oparg, pos)
+        return next_offset, BCInstruction(opnum, oparg, offset)
 
     @property
     def is_generator(self):

From noreply at buildbot.pypy.org  Mon Nov 24 17:29:30 2014
From: noreply at buildbot.pypy.org (rlamy)
Date: Mon, 24 Nov 2014 17:29:30 +0100 (CET)
Subject: [pypy-commit] pypy framestate: Implement registration of opcodes
Message-ID: <20141124162930.641BB1C3271@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy 
Branch: framestate
Changeset: r74663:52ce913fb3b6
Date: 2014-05-04 21:13 +0100
http://bitbucket.org/pypy/pypy/changeset/52ce913fb3b6/

Log:	Implement registration of opcodes

diff --git a/rpython/flowspace/bytecode.py b/rpython/flowspace/bytecode.py
--- a/rpython/flowspace/bytecode.py
+++ b/rpython/flowspace/bytecode.py
@@ -108,7 +108,11 @@
 
         if opnum in opcode.hasjrel:
             oparg += next_offset
-        return next_offset, BCInstruction(opnum, oparg, offset)
+        try:
+            op = BCInstruction.num2op[opnum](oparg, offset)
+        except KeyError:
+            op = BCInstruction(opnum, oparg, offset)
+        return next_offset, op
 
     @property
     def is_generator(self):
@@ -117,11 +121,35 @@
 OPNAMES = host_bytecode_spec.method_names
 
 class BCInstruction(object):
-    """A bytecode instruction, comprising an opcode and an optional argument."""
+    """
+    A bytecode instruction, comprising an opcode and an optional argument.
+
+    """
+    num2op = {}
+
     def __init__(self, opcode, arg, offset=-1):
         self.name = OPNAMES[opcode]
+        self.num = opcode
         self.arg = arg
         self.offset = offset
 
     def eval(self, ctx):
         return getattr(ctx, self.name)(self.arg)
+
+    @classmethod
+    def register_name(cls, name, op_class):
+        try:
+            num = OPNAMES.index(name)
+            cls.num2op[num] = op_class
+            return num
+        except ValueError:
+            return -1
+
+    def __repr__(self):
+        return "%s(%s)" % (self.name, self.arg)
+
+def register_opcode(cls):
+    """Class decorator: register opcode class as real Python opcode"""
+    name = cls.__name__
+    cls.num = Opcode.register_name(name, cls)
+    return cls

From noreply at buildbot.pypy.org  Mon Nov 24 17:29:31 2014
From: noreply at buildbot.pypy.org (rlamy)
Date: Mon, 24 Nov 2014 17:29:31 +0100 (CET)
Subject: [pypy-commit] pypy framestate: Add hook to customize opcode decoding
Message-ID: <20141124162931.9FCFA1C3271@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy 
Branch: framestate
Changeset: r74664:34072670478e
Date: 2013-05-02 15:29 +0100
http://bitbucket.org/pypy/pypy/changeset/34072670478e/

Log:	Add hook to customize opcode decoding

diff --git a/rpython/flowspace/bytecode.py b/rpython/flowspace/bytecode.py
--- a/rpython/flowspace/bytecode.py
+++ b/rpython/flowspace/bytecode.py
@@ -109,7 +109,7 @@
         if opnum in opcode.hasjrel:
             oparg += next_offset
         try:
-            op = BCInstruction.num2op[opnum](oparg, offset)
+            op = BCInstruction.num2op[opnum].decode(oparg, offset, self)
         except KeyError:
             op = BCInstruction(opnum, oparg, offset)
         return next_offset, op
@@ -133,6 +133,10 @@
         self.arg = arg
         self.offset = offset
 
+    @classmethod
+    def decode(cls, arg, offset, code):
+        return cls(arg, offset)
+
     def eval(self, ctx):
         return getattr(ctx, self.name)(self.arg)
 

From noreply at buildbot.pypy.org  Mon Nov 24 17:29:32 2014
From: noreply at buildbot.pypy.org (rlamy)
Date: Mon, 24 Nov 2014 17:29:32 +0100 (CET)
Subject: [pypy-commit] pypy framestate: Resolve LOAD_CONST constants at
	decoding time
Message-ID: <20141124162932.D83F21C3271@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy 
Branch: framestate
Changeset: r74665:0d89e1419435
Date: 2013-08-11 17:42 +0100
http://bitbucket.org/pypy/pypy/changeset/0d89e1419435/

Log:	Resolve LOAD_CONST constants at decoding time

diff --git a/rpython/flowspace/bytecode.py b/rpython/flowspace/bytecode.py
--- a/rpython/flowspace/bytecode.py
+++ b/rpython/flowspace/bytecode.py
@@ -155,5 +155,16 @@
 def register_opcode(cls):
     """Class decorator: register opcode class as real Python opcode"""
     name = cls.__name__
-    cls.num = Opcode.register_name(name, cls)
+    cls.name = name
+    cls.num = BCInstruction.register_name(name, cls)
     return cls
+
+ at register_opcode
+class LOAD_CONST(BCInstruction):
+    def __init__(self, arg, offset=-1):
+        self.arg = arg
+        self.offset = offset
+
+    @staticmethod
+    def decode(arg, offset, code):
+        return LOAD_CONST(code.consts[arg], offset)
diff --git a/rpython/flowspace/flowcontext.py b/rpython/flowspace/flowcontext.py
--- a/rpython/flowspace/flowcontext.py
+++ b/rpython/flowspace/flowcontext.py
@@ -521,9 +521,6 @@
     def getlocalvarname(self, index):
         return self.pycode.co_varnames[index]
 
-    def getconstant_w(self, index):
-        return const(self.pycode.consts[index])
-
     def getname_u(self, index):
         return self.pycode.names[index]
 
@@ -850,8 +847,8 @@
             raise FlowingError("Local variable referenced before assignment")
         self.pushvalue(w_value)
 
-    def LOAD_CONST(self, constindex):
-        w_const = self.getconstant_w(constindex)
+    def LOAD_CONST(self, constant):
+        w_const = const(constant)
         self.pushvalue(w_const)
 
     def find_global(self, w_globals, varname):

From noreply at buildbot.pypy.org  Mon Nov 24 17:29:34 2014
From: noreply at buildbot.pypy.org (rlamy)
Date: Mon, 24 Nov 2014 17:29:34 +0100 (CET)
Subject: [pypy-commit] pypy framestate: Resolve names at decoding time
Message-ID: <20141124162934.07C7D1C3271@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy 
Branch: framestate
Changeset: r74666:306ae54ebe6c
Date: 2013-05-02 17:04 +0100
http://bitbucket.org/pypy/pypy/changeset/306ae54ebe6c/

Log:	Resolve names at decoding time

diff --git a/rpython/flowspace/bytecode.py b/rpython/flowspace/bytecode.py
--- a/rpython/flowspace/bytecode.py
+++ b/rpython/flowspace/bytecode.py
@@ -108,6 +108,8 @@
 
         if opnum in opcode.hasjrel:
             oparg += next_offset
+        elif opnum in opcode.hasname:
+            oparg = self.names[oparg]
         try:
             op = BCInstruction.num2op[opnum].decode(oparg, offset, self)
         except KeyError:
diff --git a/rpython/flowspace/flowcontext.py b/rpython/flowspace/flowcontext.py
--- a/rpython/flowspace/flowcontext.py
+++ b/rpython/flowspace/flowcontext.py
@@ -521,12 +521,6 @@
     def getlocalvarname(self, index):
         return self.pycode.co_varnames[index]
 
-    def getname_u(self, index):
-        return self.pycode.names[index]
-
-    def getname_w(self, index):
-        return Constant(self.pycode.names[index])
-
     def appcall(self, func, *args_w):
         """Call an app-level RPython function directly"""
         w_func = const(func)
@@ -666,8 +660,7 @@
             raise Raise(const(e))
         return const(mod)
 
-    def IMPORT_NAME(self, nameindex):
-        modulename = self.getname_u(nameindex)
+    def IMPORT_NAME(self, modulename):
         glob = self.w_globals.value
         fromlist = self.popvalue().value
         level = self.popvalue().value
@@ -683,8 +676,8 @@
             exc = ImportError("cannot import name '%s'" % w_name.value)
             raise Raise(const(exc))
 
-    def IMPORT_FROM(self, nameindex):
-        w_name = self.getname_w(nameindex)
+    def IMPORT_FROM(self, name):
+        w_name = Constant(name)
         w_module = self.peekvalue()
         self.pushvalue(self.import_from(w_module, w_name))
 
@@ -862,15 +855,15 @@
                 raise FlowingError("global name '%s' is not defined" % varname)
         return const(value)
 
-    def LOAD_GLOBAL(self, nameindex):
-        w_result = self.find_global(self.w_globals, self.getname_u(nameindex))
+    def LOAD_GLOBAL(self, name):
+        w_result = self.find_global(self.w_globals, name)
         self.pushvalue(w_result)
     LOAD_NAME = LOAD_GLOBAL
 
-    def LOAD_ATTR(self, nameindex):
+    def LOAD_ATTR(self, name):
         "obj.attributename"
         w_obj = self.popvalue()
-        w_attributename = self.getname_w(nameindex)
+        w_attributename = Constant(name)
         w_value = op.getattr(w_obj, w_attributename).eval(self)
         self.pushvalue(w_value)
     LOOKUP_METHOD = LOAD_ATTR
@@ -891,8 +884,7 @@
         if isinstance(w_newvalue, Variable):
             w_newvalue.rename(self.getlocalvarname(varindex))
 
-    def STORE_GLOBAL(self, nameindex):
-        varname = self.getname_u(nameindex)
+    def STORE_GLOBAL(self, varname):
         raise FlowingError(
             "Attempting to modify global variable  %r." % (varname))
 
@@ -1006,9 +998,9 @@
         fn = self.newfunction(w_codeobj, defaults)
         self.pushvalue(fn)
 
-    def STORE_ATTR(self, nameindex):
+    def STORE_ATTR(self, name):
         "obj.attributename = newvalue"
-        w_attributename = self.getname_w(nameindex)
+        w_attributename = Constant(name)
         w_obj = self.popvalue()
         w_newvalue = self.popvalue()
         op.setattr(w_obj, w_attributename, w_newvalue).eval(self)

From noreply at buildbot.pypy.org  Mon Nov 24 17:29:35 2014
From: noreply at buildbot.pypy.org (rlamy)
Date: Mon, 24 Nov 2014 17:29:35 +0100 (CET)
Subject: [pypy-commit] pypy framestate: create class GenericOpcode
Message-ID: <20141124162935.377781C3271@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy 
Branch: framestate
Changeset: r74667:c6e0fb13a2d8
Date: 2013-08-11 22:03 +0100
http://bitbucket.org/pypy/pypy/changeset/c6e0fb13a2d8/

Log:	create class GenericOpcode

diff --git a/rpython/flowspace/bytecode.py b/rpython/flowspace/bytecode.py
--- a/rpython/flowspace/bytecode.py
+++ b/rpython/flowspace/bytecode.py
@@ -5,6 +5,7 @@
 from opcode import EXTENDED_ARG, HAVE_ARGUMENT
 import opcode
 from rpython.flowspace.argument import Signature
+from rpython.flowspace.model import const
 
 CO_GENERATOR = 0x0020
 CO_VARARGS = 0x0004
@@ -59,19 +60,11 @@
     def _from_code(cls, code):
         """Initialize the code object from a real (CPython) one.
         """
-        return cls(code.co_argcount,
-                   code.co_nlocals,
-                   code.co_stacksize,
-                   code.co_flags,
-                   code.co_code,
-                   list(code.co_consts),
-                   list(code.co_names),
-                   list(code.co_varnames),
-                   code.co_filename,
-                   code.co_name,
-                   code.co_firstlineno,
-                   code.co_lnotab,
-                   list(code.co_freevars))
+        return cls(code.co_argcount, code.co_nlocals, code.co_stacksize,
+                code.co_flags, code.co_code, list(code.co_consts),
+                list(code.co_names), list(code.co_varnames), code.co_filename,
+                code.co_name, code.co_firstlineno, code.co_lnotab,
+                list(code.co_freevars))
 
     @property
     def formalargcount(self):
@@ -113,7 +106,7 @@
         try:
             op = BCInstruction.num2op[opnum].decode(oparg, offset, self)
         except KeyError:
-            op = BCInstruction(opnum, oparg, offset)
+            op = GenericOpcode(opnum, oparg, offset)
         return next_offset, op
 
     @property
@@ -129,9 +122,7 @@
     """
     num2op = {}
 
-    def __init__(self, opcode, arg, offset=-1):
-        self.name = OPNAMES[opcode]
-        self.num = opcode
+    def __init__(self, arg, offset=-1):
         self.arg = arg
         self.offset = offset
 
@@ -140,7 +131,7 @@
         return cls(arg, offset)
 
     def eval(self, ctx):
-        return getattr(ctx, self.name)(self.arg)
+        pass
 
     @classmethod
     def register_name(cls, name, op_class):
@@ -154,6 +145,17 @@
     def __repr__(self):
         return "%s(%s)" % (self.name, self.arg)
 
+class GenericOpcode(BCInstruction):
+    def __init__(self, opcode, arg, offset=-1):
+        self.name = OPNAMES[opcode]
+        self.num = opcode
+        self.arg = arg
+        self.offset = offset
+
+    def eval(self, ctx):
+        return getattr(ctx, self.name)(self.arg)
+
+
 def register_opcode(cls):
     """Class decorator: register opcode class as real Python opcode"""
     name = cls.__name__
@@ -163,10 +165,9 @@
 
 @register_opcode
 class LOAD_CONST(BCInstruction):
-    def __init__(self, arg, offset=-1):
-        self.arg = arg
-        self.offset = offset
-
     @staticmethod
     def decode(arg, offset, code):
         return LOAD_CONST(code.consts[arg], offset)
+
+    def eval(self, ctx):
+        ctx.pushvalue(const(self.arg))

From noreply at buildbot.pypy.org  Mon Nov 24 17:29:36 2014
From: noreply at buildbot.pypy.org (rlamy)
Date: Mon, 24 Nov 2014 17:29:36 +0100 (CET)
Subject: [pypy-commit] pypy framestate: fix test_all_opcodes_defined()
Message-ID: <20141124162936.66AC81C3271@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy 
Branch: framestate
Changeset: r74668:ae2d25eea35d
Date: 2013-08-11 21:30 +0100
http://bitbucket.org/pypy/pypy/changeset/ae2d25eea35d/

Log:	fix test_all_opcodes_defined()

diff --git a/rpython/flowspace/test/test_objspace.py b/rpython/flowspace/test/test_objspace.py
--- a/rpython/flowspace/test/test_objspace.py
+++ b/rpython/flowspace/test/test_objspace.py
@@ -8,6 +8,7 @@
 from rpython.translator.simplify import simplify_graph
 from rpython.flowspace.objspace import build_flow
 from rpython.flowspace.flowcontext import FlowingError, FlowContext
+from rpython.flowspace.bytecode import BCInstruction
 from rpython.conftest import option
 from rpython.tool.stdlib_opcode import host_bytecode_spec
 
@@ -56,7 +57,8 @@
     opnames = set(host_bytecode_spec.method_names)
     methods = set([name for name in dir(FlowContext) if name.upper() == name])
     handled_elsewhere = set(['EXTENDED_ARG'])
-    missing = opnames - methods - handled_elsewhere
+    opcode_classes = set([cls.name for cls in BCInstruction.num2op.values()])
+    missing = opnames - methods - handled_elsewhere - opcode_classes
     assert not missing
 
 class TestFlowObjSpace(Base):

From noreply at buildbot.pypy.org  Mon Nov 24 17:29:37 2014
From: noreply at buildbot.pypy.org (rlamy)
Date: Mon, 24 Nov 2014 17:29:37 +0100 (CET)
Subject: [pypy-commit] pypy framestate: create BytecodeReader
Message-ID: <20141124162937.89C301C3271@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy 
Branch: framestate
Changeset: r74669:21b8a1414a68
Date: 2014-05-05 19:44 +0100
http://bitbucket.org/pypy/pypy/changeset/21b8a1414a68/

Log:	create BytecodeReader

diff --git a/rpython/flowspace/bytecode.py b/rpython/flowspace/bytecode.py
--- a/rpython/flowspace/bytecode.py
+++ b/rpython/flowspace/bytecode.py
@@ -72,13 +72,21 @@
         and **varkwarg, if they exist."""
         return self.signature.scope_length()
 
+    @property
+    def is_generator(self):
+        return bool(self.co_flags & CO_GENERATOR)
+
     def read(self, offset):
+        return bc_reader.read(self, offset)
+
+class BytecodeReader(object):
+    def read(self, code, offset):
         """
         Decode the instruction starting at position ``offset``.
 
         Returns (next_offset, instruction).
         """
-        co_code = self.co_code
+        co_code = code.co_code
         opnum = ord(co_code[offset])
         next_offset = offset + 1
 
@@ -102,16 +110,15 @@
         if opnum in opcode.hasjrel:
             oparg += next_offset
         elif opnum in opcode.hasname:
-            oparg = self.names[oparg]
+            oparg = code.names[oparg]
         try:
-            op = BCInstruction.num2op[opnum].decode(oparg, offset, self)
+            op = BCInstruction.num2op[opnum].decode(oparg, offset, code)
         except KeyError:
             op = GenericOpcode(opnum, oparg, offset)
         return next_offset, op
 
-    @property
-    def is_generator(self):
-        return bool(self.co_flags & CO_GENERATOR)
+bc_reader = BytecodeReader()
+
 
 OPNAMES = host_bytecode_spec.method_names
 

From noreply at buildbot.pypy.org  Mon Nov 24 17:29:38 2014
From: noreply at buildbot.pypy.org (rlamy)
Date: Mon, 24 Nov 2014 17:29:38 +0100 (CET)
Subject: [pypy-commit] pypy framestate: store opcode definitions in the
	bc_reader object
Message-ID: <20141124162938.BE5A61C3271@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy 
Branch: framestate
Changeset: r74670:7818088e6df6
Date: 2014-05-10 17:17 +0100
http://bitbucket.org/pypy/pypy/changeset/7818088e6df6/

Log:	store opcode definitions in the bc_reader object

diff --git a/rpython/flowspace/bytecode.py b/rpython/flowspace/bytecode.py
--- a/rpython/flowspace/bytecode.py
+++ b/rpython/flowspace/bytecode.py
@@ -80,6 +80,22 @@
         return bc_reader.read(self, offset)
 
 class BytecodeReader(object):
+    def __init__(self, opnames):
+        self.opnames = opnames
+        self.num2cls = {}
+
+    def register_name(self, name, InstrClass):
+        num = self.opnames.index(name)
+        self.num2cls[num] = InstrClass
+        return num
+
+    def register_opcode(self, cls):
+        """Class decorator: register opcode class as real Python opcode"""
+        name = cls.__name__
+        cls.name = name
+        cls.num = self.register_name(name, cls)
+        return cls
+
     def read(self, code, offset):
         """
         Decode the instruction starting at position ``offset``.
@@ -112,23 +128,19 @@
         elif opnum in opcode.hasname:
             oparg = code.names[oparg]
         try:
-            op = BCInstruction.num2op[opnum].decode(oparg, offset, code)
+            op = self.num2cls[opnum].decode(oparg, offset, code)
         except KeyError:
-            op = GenericOpcode(opnum, oparg, offset)
+            op = GenericOpcode(self.opnames[opnum], opnum, oparg, offset)
         return next_offset, op
 
-bc_reader = BytecodeReader()
+bc_reader = BytecodeReader(host_bytecode_spec.method_names)
 
 
-OPNAMES = host_bytecode_spec.method_names
-
 class BCInstruction(object):
     """
     A bytecode instruction, comprising an opcode and an optional argument.
 
     """
-    num2op = {}
-
     def __init__(self, arg, offset=-1):
         self.arg = arg
         self.offset = offset
@@ -140,21 +152,12 @@
     def eval(self, ctx):
         pass
 
-    @classmethod
-    def register_name(cls, name, op_class):
-        try:
-            num = OPNAMES.index(name)
-            cls.num2op[num] = op_class
-            return num
-        except ValueError:
-            return -1
-
     def __repr__(self):
         return "%s(%s)" % (self.name, self.arg)
 
 class GenericOpcode(BCInstruction):
-    def __init__(self, opcode, arg, offset=-1):
-        self.name = OPNAMES[opcode]
+    def __init__(self, name, opcode, arg, offset=-1):
+        self.name = name
         self.num = opcode
         self.arg = arg
         self.offset = offset
@@ -163,14 +166,7 @@
         return getattr(ctx, self.name)(self.arg)
 
 
-def register_opcode(cls):
-    """Class decorator: register opcode class as real Python opcode"""
-    name = cls.__name__
-    cls.name = name
-    cls.num = BCInstruction.register_name(name, cls)
-    return cls
-
- at register_opcode
+ at bc_reader.register_opcode
 class LOAD_CONST(BCInstruction):
     @staticmethod
     def decode(arg, offset, code):
diff --git a/rpython/flowspace/test/test_objspace.py b/rpython/flowspace/test/test_objspace.py
--- a/rpython/flowspace/test/test_objspace.py
+++ b/rpython/flowspace/test/test_objspace.py
@@ -8,7 +8,7 @@
 from rpython.translator.simplify import simplify_graph
 from rpython.flowspace.objspace import build_flow
 from rpython.flowspace.flowcontext import FlowingError, FlowContext
-from rpython.flowspace.bytecode import BCInstruction
+from rpython.flowspace.bytecode import bc_reader
 from rpython.conftest import option
 from rpython.tool.stdlib_opcode import host_bytecode_spec
 
@@ -57,7 +57,7 @@
     opnames = set(host_bytecode_spec.method_names)
     methods = set([name for name in dir(FlowContext) if name.upper() == name])
     handled_elsewhere = set(['EXTENDED_ARG'])
-    opcode_classes = set([cls.name for cls in BCInstruction.num2op.values()])
+    opcode_classes = set([cls.name for cls in bc_reader.num2cls.values()])
     missing = opnames - methods - handled_elsewhere - opcode_classes
     assert not missing
 

From noreply at buildbot.pypy.org  Mon Nov 24 17:29:39 2014
From: noreply at buildbot.pypy.org (rlamy)
Date: Mon, 24 Nov 2014 17:29:39 +0100 (CET)
Subject: [pypy-commit] pypy framestate: implement unary operations as
	BCInstruction classes
Message-ID: <20141124162939.F10221C3271@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy 
Branch: framestate
Changeset: r74671:d742c0d16449
Date: 2014-05-10 05:34 +0100
http://bitbucket.org/pypy/pypy/changeset/d742c0d16449/

Log:	implement unary operations as BCInstruction classes

diff --git a/rpython/flowspace/bytecode.py b/rpython/flowspace/bytecode.py
--- a/rpython/flowspace/bytecode.py
+++ b/rpython/flowspace/bytecode.py
@@ -6,6 +6,7 @@
 import opcode
 from rpython.flowspace.argument import Signature
 from rpython.flowspace.model import const
+from rpython.flowspace.operation import op
 
 CO_GENERATOR = 0x0020
 CO_VARARGS = 0x0004
@@ -174,3 +175,23 @@
 
     def eval(self, ctx):
         ctx.pushvalue(const(self.arg))
+
+_unary_ops = [
+    ('UNARY_POSITIVE', op.pos),
+    ('UNARY_NEGATIVE', op.neg),
+    ('UNARY_CONVERT', op.repr),
+    ('UNARY_INVERT', op.invert),
+]
+
+def unaryoperation(OPCODE, oper):
+    class UNARY_OP(BCInstruction):
+        def eval(self, ctx):
+            w_1 = ctx.popvalue()
+            w_result = oper(w_1).eval(ctx)
+            ctx.pushvalue(w_result)
+    UNARY_OP.__name__ = OPCODE
+    bc_reader.register_opcode(UNARY_OP)
+    return UNARY_OP
+
+for OPCODE, oper in _unary_ops:
+    globals()[OPCODE] = unaryoperation(OPCODE, oper)
diff --git a/rpython/flowspace/flowcontext.py b/rpython/flowspace/flowcontext.py
--- a/rpython/flowspace/flowcontext.py
+++ b/rpython/flowspace/flowcontext.py
@@ -189,21 +189,6 @@
 
 # ____________________________________________________________
 
-_unary_ops = [
-    ('UNARY_POSITIVE', op.pos),
-    ('UNARY_NEGATIVE', op.neg),
-    ('UNARY_CONVERT', op.repr),
-    ('UNARY_INVERT', op.invert),
-]
-
-def unaryoperation(OPCODE, operation):
-    def UNARY_OP(self, *ignored):
-        w_1 = self.popvalue()
-        w_result = operation(w_1).eval(self)
-        self.pushvalue(w_result)
-    UNARY_OP.func_name = OPCODE
-    return UNARY_OP
-
 _binary_ops = [
     ('BINARY_MULTIPLY', op.mul),
     ('BINARY_TRUE_DIVIDE', op.truediv),
@@ -928,9 +913,6 @@
             w_value = self.peekvalue(delta)
             self.pushvalue(w_value)
 
-    for OPCODE, op in _unary_ops:
-        locals()[OPCODE] = unaryoperation(OPCODE, op)
-
     for OPCODE, op in _binary_ops:
         locals()[OPCODE] = binaryoperation(OPCODE, op)
 

From noreply at buildbot.pypy.org  Mon Nov 24 17:29:41 2014
From: noreply at buildbot.pypy.org (rlamy)
Date: Mon, 24 Nov 2014 17:29:41 +0100 (CET)
Subject: [pypy-commit] pypy framestate: implement binary operations as
	BCInstruction classes
Message-ID: <20141124162941.248EE1C3271@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy 
Branch: framestate
Changeset: r74672:27dd847844a9
Date: 2014-05-10 05:35 +0100
http://bitbucket.org/pypy/pypy/changeset/27dd847844a9/

Log:	implement binary operations as BCInstruction classes

diff --git a/rpython/flowspace/bytecode.py b/rpython/flowspace/bytecode.py
--- a/rpython/flowspace/bytecode.py
+++ b/rpython/flowspace/bytecode.py
@@ -195,3 +195,46 @@
 
 for OPCODE, oper in _unary_ops:
     globals()[OPCODE] = unaryoperation(OPCODE, oper)
+
+
+_binary_ops = [
+    ('BINARY_MULTIPLY', op.mul),
+    ('BINARY_TRUE_DIVIDE', op.truediv),
+    ('BINARY_FLOOR_DIVIDE', op.floordiv),
+    ('BINARY_DIVIDE', op.div),
+    ('BINARY_MODULO', op.mod),
+    ('BINARY_ADD', op.add),
+    ('BINARY_SUBTRACT', op.sub),
+    ('BINARY_SUBSCR', op.getitem),
+    ('BINARY_LSHIFT', op.lshift),
+    ('BINARY_RSHIFT', op.rshift),
+    ('BINARY_AND', op.and_),
+    ('BINARY_XOR', op.xor),
+    ('BINARY_OR', op.or_),
+    ('INPLACE_MULTIPLY', op.inplace_mul),
+    ('INPLACE_TRUE_DIVIDE', op.inplace_truediv),
+    ('INPLACE_FLOOR_DIVIDE', op.inplace_floordiv),
+    ('INPLACE_DIVIDE', op.inplace_div),
+    ('INPLACE_MODULO', op.inplace_mod),
+    ('INPLACE_ADD', op.inplace_add),
+    ('INPLACE_SUBTRACT', op.inplace_sub),
+    ('INPLACE_LSHIFT', op.inplace_lshift),
+    ('INPLACE_RSHIFT', op.inplace_rshift),
+    ('INPLACE_AND', op.inplace_and),
+    ('INPLACE_XOR', op.inplace_xor),
+    ('INPLACE_OR', op.inplace_or),
+]
+
+def binaryoperation(OPCODE, oper):
+    class BINARY_OP(BCInstruction):
+        def eval(self, ctx):
+            w_2 = ctx.popvalue()
+            w_1 = ctx.popvalue()
+            w_result = oper(w_1, w_2).eval(ctx)
+            ctx.pushvalue(w_result)
+    BINARY_OP.__name__ = OPCODE
+    bc_reader.register_opcode(BINARY_OP)
+    return BINARY_OP
+
+for OPCODE, oper in _binary_ops:
+    globals()[OPCODE] = binaryoperation(OPCODE, oper)
diff --git a/rpython/flowspace/flowcontext.py b/rpython/flowspace/flowcontext.py
--- a/rpython/flowspace/flowcontext.py
+++ b/rpython/flowspace/flowcontext.py
@@ -189,44 +189,6 @@
 
 # ____________________________________________________________
 
-_binary_ops = [
-    ('BINARY_MULTIPLY', op.mul),
-    ('BINARY_TRUE_DIVIDE', op.truediv),
-    ('BINARY_FLOOR_DIVIDE', op.floordiv),
-    ('BINARY_DIVIDE', op.div),
-    ('BINARY_MODULO', op.mod),
-    ('BINARY_ADD', op.add),
-    ('BINARY_SUBTRACT', op.sub),
-    ('BINARY_SUBSCR', op.getitem),
-    ('BINARY_LSHIFT', op.lshift),
-    ('BINARY_RSHIFT', op.rshift),
-    ('BINARY_AND', op.and_),
-    ('BINARY_XOR', op.xor),
-    ('BINARY_OR', op.or_),
-    ('INPLACE_MULTIPLY', op.inplace_mul),
-    ('INPLACE_TRUE_DIVIDE', op.inplace_truediv),
-    ('INPLACE_FLOOR_DIVIDE', op.inplace_floordiv),
-    ('INPLACE_DIVIDE', op.inplace_div),
-    ('INPLACE_MODULO', op.inplace_mod),
-    ('INPLACE_ADD', op.inplace_add),
-    ('INPLACE_SUBTRACT', op.inplace_sub),
-    ('INPLACE_LSHIFT', op.inplace_lshift),
-    ('INPLACE_RSHIFT', op.inplace_rshift),
-    ('INPLACE_AND', op.inplace_and),
-    ('INPLACE_XOR', op.inplace_xor),
-    ('INPLACE_OR', op.inplace_or),
-]
-
-def binaryoperation(OPCODE, operation):
-    """NOT_RPYTHON"""
-    def BINARY_OP(self, _):
-        w_2 = self.popvalue()
-        w_1 = self.popvalue()
-        w_result = operation(w_1, w_2).eval(self)
-        self.pushvalue(w_result)
-    BINARY_OP.func_name = OPCODE
-    return BINARY_OP
-
 _unsupported_ops = [
     ('BINARY_POWER', "a ** b"),
     ('BUILD_CLASS', 'defining classes inside functions'),
@@ -913,9 +875,6 @@
             w_value = self.peekvalue(delta)
             self.pushvalue(w_value)
 
-    for OPCODE, op in _binary_ops:
-        locals()[OPCODE] = binaryoperation(OPCODE, op)
-
     for OPCODE, op in _unsupported_ops:
         locals()[OPCODE] = unsupportedoperation(OPCODE, op)
 

From noreply at buildbot.pypy.org  Mon Nov 24 17:29:42 2014
From: noreply at buildbot.pypy.org (rlamy)
Date: Mon, 24 Nov 2014 17:29:42 +0100 (CET)
Subject: [pypy-commit] pypy framestate: Create HostCode.disassemble()
Message-ID: <20141124162942.47D8F1C3271@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy 
Branch: framestate
Changeset: r74673:f3685601c1d7
Date: 2013-05-02 23:24 +0100
http://bitbucket.org/pypy/pypy/changeset/f3685601c1d7/

Log:	Create HostCode.disassemble()

diff --git a/rpython/flowspace/bytecode.py b/rpython/flowspace/bytecode.py
--- a/rpython/flowspace/bytecode.py
+++ b/rpython/flowspace/bytecode.py
@@ -57,6 +57,23 @@
         self.co_lnotab = lnotab
         self.signature = cpython_code_signature(self)
 
+    def disassemble(self):
+        contents = []
+        offsets = []
+        jumps = {}
+        pos = 0
+        i = 0
+        while pos < len(self.co_code):
+            offsets.append(pos)
+            next_pos, op = self.decode(pos)
+            contents.append(op)
+            if op.has_jump():
+                jumps[pos] = op.arg
+            pos = next_pos
+            i += 1
+        return contents, offsets, jumps
+
+
     @classmethod
     def _from_code(cls, code):
         """Initialize the code object from a real (CPython) one.
@@ -153,6 +170,9 @@
     def eval(self, ctx):
         pass
 
+    def has_jump(self):
+        return self.num in opcode.hasjrel or self.num in opcode.hasjabs
+
     def __repr__(self):
         return "%s(%s)" % (self.name, self.arg)
 

From noreply at buildbot.pypy.org  Mon Nov 24 17:29:43 2014
From: noreply at buildbot.pypy.org (rlamy)
Date: Mon, 24 Nov 2014 17:29:43 +0100 (CET)
Subject: [pypy-commit] pypy framestate: Decode the bytecode up-front
Message-ID: <20141124162943.65B201C3271@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy 
Branch: framestate
Changeset: r74674:272d69dae528
Date: 2014-05-05 01:33 +0100
http://bitbucket.org/pypy/pypy/changeset/272d69dae528/

Log:	Decode the bytecode up-front

diff --git a/rpython/flowspace/bytecode.py b/rpython/flowspace/bytecode.py
--- a/rpython/flowspace/bytecode.py
+++ b/rpython/flowspace/bytecode.py
@@ -56,6 +56,7 @@
         self.co_firstlineno = firstlineno
         self.co_lnotab = lnotab
         self.signature = cpython_code_signature(self)
+        self.build_flow()
 
     def disassemble(self):
         contents = []
@@ -73,6 +74,14 @@
             i += 1
         return contents, offsets, jumps
 
+    def build_flow(self):
+        next_pos = pos = 0
+        contents, offsets, jumps = self.disassemble()
+        self.contents = zip(offsets, contents)
+        self.pos_index = dict((offset, i) for i, offset in enumerate(offsets))
+        # add end marker
+        self.contents.append((len(self.co_code), None))
+
 
     @classmethod
     def _from_code(cls, code):
@@ -95,6 +104,12 @@
         return bool(self.co_flags & CO_GENERATOR)
 
     def read(self, offset):
+        i = self.pos_index[offset]
+        op = self.contents[i][1]
+        next_offset = self.contents[i+1][0]
+        return next_offset, op
+
+    def decode(self, offset):
         return bc_reader.read(self, offset)
 
 class BytecodeReader(object):

From noreply at buildbot.pypy.org  Mon Nov 24 17:29:44 2014
From: noreply at buildbot.pypy.org (rlamy)
Date: Mon, 24 Nov 2014 17:29:44 +0100 (CET)
Subject: [pypy-commit] pypy framestate: move disassemble() to BytecodeReader
Message-ID: <20141124162944.889401C3271@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy 
Branch: framestate
Changeset: r74675:2309d510d8bb
Date: 2014-05-10 20:30 +0100
http://bitbucket.org/pypy/pypy/changeset/2309d510d8bb/

Log:	move disassemble() to BytecodeReader

diff --git a/rpython/flowspace/bytecode.py b/rpython/flowspace/bytecode.py
--- a/rpython/flowspace/bytecode.py
+++ b/rpython/flowspace/bytecode.py
@@ -58,25 +58,9 @@
         self.signature = cpython_code_signature(self)
         self.build_flow()
 
-    def disassemble(self):
-        contents = []
-        offsets = []
-        jumps = {}
-        pos = 0
-        i = 0
-        while pos < len(self.co_code):
-            offsets.append(pos)
-            next_pos, op = self.decode(pos)
-            contents.append(op)
-            if op.has_jump():
-                jumps[pos] = op.arg
-            pos = next_pos
-            i += 1
-        return contents, offsets, jumps
-
     def build_flow(self):
         next_pos = pos = 0
-        contents, offsets, jumps = self.disassemble()
+        contents, offsets, jumps = bc_reader.disassemble(self)
         self.contents = zip(offsets, contents)
         self.pos_index = dict((offset, i) for i, offset in enumerate(offsets))
         # add end marker
@@ -109,8 +93,6 @@
         next_offset = self.contents[i+1][0]
         return next_offset, op
 
-    def decode(self, offset):
-        return bc_reader.read(self, offset)
 
 class BytecodeReader(object):
     def __init__(self, opnames):
@@ -129,6 +111,22 @@
         cls.num = self.register_name(name, cls)
         return cls
 
+    def disassemble(self, code):
+        contents = []
+        offsets = []
+        jumps = {}
+        pos = 0
+        i = 0
+        while pos < len(code.co_code):
+            offsets.append(pos)
+            next_pos, op = self.read(code, pos)
+            contents.append(op)
+            if op.has_jump():
+                jumps[pos] = op.arg
+            pos = next_pos
+            i += 1
+        return contents, offsets, jumps
+
     def read(self, code, offset):
         """
         Decode the instruction starting at position ``offset``.

From noreply at buildbot.pypy.org  Mon Nov 24 17:29:45 2014
From: noreply at buildbot.pypy.org (rlamy)
Date: Mon, 24 Nov 2014 17:29:45 +0100 (CET)
Subject: [pypy-commit] pypy framestate: Set up infrastructure for
	bytecode-level flow graphs
Message-ID: <20141124162945.AC4631C3271@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy 
Branch: framestate
Changeset: r74676:a699440d109d
Date: 2014-05-10 05:28 +0100
http://bitbucket.org/pypy/pypy/changeset/a699440d109d/

Log:	Set up infrastructure for bytecode-level flow graphs

	Create classes BytecodeGraph and BytecodeBlock

diff --git a/rpython/flowspace/bytecode.py b/rpython/flowspace/bytecode.py
--- a/rpython/flowspace/bytecode.py
+++ b/rpython/flowspace/bytecode.py
@@ -59,13 +59,25 @@
         self.build_flow()
 
     def build_flow(self):
-        next_pos = pos = 0
         contents, offsets, jumps = bc_reader.disassemble(self)
-        self.contents = zip(offsets, contents)
-        self.pos_index = dict((offset, i) for i, offset in enumerate(offsets))
-        # add end marker
-        self.contents.append((len(self.co_code), None))
+        pos_map = dict([(pos, i) for i, pos in enumerate(offsets)])
+        cuts = sorted([pos_map[n] + 1 for n in jumps.keys()] +
+                [pos_map[n] for n in jumps.values()])
+        pendingblocks = [SimpleBlock(contents[i:j])
+                for i, j in zip([0] + cuts, cuts + [len(self.co_code)])]
 
+        graph = self.graph = BytecodeGraph(pendingblocks[0])
+        graph.pendingblocks = pendingblocks
+        for block in pendingblocks:
+            for i, op in enumerate(block.operations):
+                graph.pos_index[op.offset] = block, i
+        graph.next_pos = dict([(offsets[i], offsets[i+1])
+            for i in range(len(offsets) - 1)])
+        graph.next_pos[offsets[-1]] = len(self.co_code)
+        while graph.pendingblocks:
+            block = graph.next_block()
+            for i, op in enumerate(block.operations):
+                op.bc_flow(block, graph)
 
     @classmethod
     def _from_code(cls, code):
@@ -88,9 +100,9 @@
         return bool(self.co_flags & CO_GENERATOR)
 
     def read(self, offset):
-        i = self.pos_index[offset]
-        op = self.contents[i][1]
-        next_offset = self.contents[i+1][0]
+        block, i = self.graph.pos_index[offset]
+        op = block[i]
+        next_offset = self.graph.next_pos[offset]
         return next_offset, op
 
 
@@ -166,6 +178,61 @@
 
 bc_reader = BytecodeReader(host_bytecode_spec.method_names)
 
+class BytecodeGraph(object):
+    def __init__(self, startblock):
+        self.entry = EntryBlock()
+        self.entry.set_exits([startblock])
+        self.pos_index = {}
+        self.pendingblocks = [startblock]
+
+    def next_block(self):
+        return self.pendingblocks.pop()
+
+
+class BytecodeBlock(object):
+    """Base class for opcode blocks"""
+    def __init__(self):
+        self.parents = set()
+        self._exits = []
+
+    def __getitem__(self, i):
+        return self.operations[i]
+
+    def add_exit(self, exit):
+        self._exits.append(exit)
+        exit.parents.add(self)
+
+    def set_exits(self, exits):
+        for old_exit in self._exits:
+            old_exit.parents.remove(self)
+        self._exits = exits
+        for new_exit in exits:
+            new_exit.parents.add(self)
+
+    def change_exit(self, old_exit, new_exit):
+        self._exits = [new_exit if exit is old_exit else exit
+                for exit in self._exits]
+        old_exit.parents.remove(self)
+        new_exit.parents.add(self)
+
+    @property
+    def startpos(self):
+        return self.operations[0].offset
+
+
+class EntryBlock(BytecodeBlock):
+    """A fake block to represent the beginning of a code object"""
+
+class SimpleBlock(BytecodeBlock):
+    """A block with a single exit."""
+    def __init__(self, operations, exit=None):
+        BytecodeBlock.__init__(self)
+        self.operations = operations
+        if exit:
+            self.set_exits([exit])
+
+
+OPNAMES = host_bytecode_spec.method_names
 
 class BCInstruction(object):
     """
@@ -183,6 +250,9 @@
     def eval(self, ctx):
         pass
 
+    def bc_flow(self, block, graph):
+        pass
+
     def has_jump(self):
         return self.num in opcode.hasjrel or self.num in opcode.hasjabs
 

From noreply at buildbot.pypy.org  Mon Nov 24 17:29:46 2014
From: noreply at buildbot.pypy.org (rlamy)
Date: Mon, 24 Nov 2014 17:29:46 +0100 (CET)
Subject: [pypy-commit] pypy framestate: kill BytecodeGraph.pendingblocks
Message-ID: <20141124162946.D5BE61C3271@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy 
Branch: framestate
Changeset: r74677:14bbfa87a0eb
Date: 2014-05-11 15:18 +0100
http://bitbucket.org/pypy/pypy/changeset/14bbfa87a0eb/

Log:	kill BytecodeGraph.pendingblocks

diff --git a/rpython/flowspace/bytecode.py b/rpython/flowspace/bytecode.py
--- a/rpython/flowspace/bytecode.py
+++ b/rpython/flowspace/bytecode.py
@@ -67,15 +67,14 @@
                 for i, j in zip([0] + cuts, cuts + [len(self.co_code)])]
 
         graph = self.graph = BytecodeGraph(pendingblocks[0])
-        graph.pendingblocks = pendingblocks
         for block in pendingblocks:
             for i, op in enumerate(block.operations):
                 graph.pos_index[op.offset] = block, i
         graph.next_pos = dict([(offsets[i], offsets[i+1])
             for i in range(len(offsets) - 1)])
         graph.next_pos[offsets[-1]] = len(self.co_code)
-        while graph.pendingblocks:
-            block = graph.next_block()
+        while pendingblocks:
+            block = pendingblocks.pop()
             for i, op in enumerate(block.operations):
                 op.bc_flow(block, graph)
 
@@ -183,10 +182,6 @@
         self.entry = EntryBlock()
         self.entry.set_exits([startblock])
         self.pos_index = {}
-        self.pendingblocks = [startblock]
-
-    def next_block(self):
-        return self.pendingblocks.pop()
 
 
 class BytecodeBlock(object):

From noreply at buildbot.pypy.org  Mon Nov 24 17:29:47 2014
From: noreply at buildbot.pypy.org (rlamy)
Date: Mon, 24 Nov 2014 17:29:47 +0100 (CET)
Subject: [pypy-commit] pypy framestate: move .read() to BytecodeGraph
Message-ID: <20141124162947.F3A771C3271@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy 
Branch: framestate
Changeset: r74678:98ad1b9e8759
Date: 2014-05-11 15:51 +0100
http://bitbucket.org/pypy/pypy/changeset/98ad1b9e8759/

Log:	move .read() to BytecodeGraph

diff --git a/rpython/flowspace/bytecode.py b/rpython/flowspace/bytecode.py
--- a/rpython/flowspace/bytecode.py
+++ b/rpython/flowspace/bytecode.py
@@ -98,13 +98,6 @@
     def is_generator(self):
         return bool(self.co_flags & CO_GENERATOR)
 
-    def read(self, offset):
-        block, i = self.graph.pos_index[offset]
-        op = block[i]
-        next_offset = self.graph.next_pos[offset]
-        return next_offset, op
-
-
 class BytecodeReader(object):
     def __init__(self, opnames):
         self.opnames = opnames
@@ -183,6 +176,12 @@
         self.entry.set_exits([startblock])
         self.pos_index = {}
 
+    def read(self, offset):
+        block, i = self.pos_index[offset]
+        op = block[i]
+        next_offset = self.next_pos[offset]
+        return next_offset, op
+
 
 class BytecodeBlock(object):
     """Base class for opcode blocks"""
diff --git a/rpython/flowspace/flowcontext.py b/rpython/flowspace/flowcontext.py
--- a/rpython/flowspace/flowcontext.py
+++ b/rpython/flowspace/flowcontext.py
@@ -452,7 +452,7 @@
         self.last_offset = next_offset
         next_offset, instr = self.pycode.read(next_offset)
         try:
-            res = instr.eval(self)
+            offset = instr.eval(self)
             return offset if offset is not None else next_offset
         except FlowSignal as signal:
             return self.unroll(signal)

From noreply at buildbot.pypy.org  Mon Nov 24 17:29:49 2014
From: noreply at buildbot.pypy.org (rlamy)
Date: Mon, 24 Nov 2014 17:29:49 +0100 (CET)
Subject: [pypy-commit] pypy framestate: Create BytecodeGraph.next_pos() and
	use it.
Message-ID: <20141124162949.2784A1C3271@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy 
Branch: framestate
Changeset: r74679:aa96c8f015da
Date: 2014-05-05 02:05 +0100
http://bitbucket.org/pypy/pypy/changeset/aa96c8f015da/

Log:	Create BytecodeGraph.next_pos() and use it.

diff --git a/rpython/flowspace/bytecode.py b/rpython/flowspace/bytecode.py
--- a/rpython/flowspace/bytecode.py
+++ b/rpython/flowspace/bytecode.py
@@ -70,9 +70,9 @@
         for block in pendingblocks:
             for i, op in enumerate(block.operations):
                 graph.pos_index[op.offset] = block, i
-        graph.next_pos = dict([(offsets[i], offsets[i+1])
+        graph._next_pos = dict([(offsets[i], offsets[i+1])
             for i in range(len(offsets) - 1)])
-        graph.next_pos[offsets[-1]] = len(self.co_code)
+        graph._next_pos[offsets[-1]] = len(self.co_code)
         while pendingblocks:
             block = pendingblocks.pop()
             for i, op in enumerate(block.operations):
@@ -98,6 +98,7 @@
     def is_generator(self):
         return bool(self.co_flags & CO_GENERATOR)
 
+
 class BytecodeReader(object):
     def __init__(self, opnames):
         self.opnames = opnames
@@ -176,11 +177,17 @@
         self.entry.set_exits([startblock])
         self.pos_index = {}
 
-    def read(self, offset):
-        block, i = self.pos_index[offset]
-        op = block[i]
-        next_offset = self.next_pos[offset]
-        return next_offset, op
+    def read(self, pos):
+        bc_block, i = self.pos_index[pos]
+        return bc_block[i]
+
+    def next_pos(self, opcode):
+        return self._next_pos[opcode.offset]
+
+    def add_jump(self, block, target_block):
+        last_op = block.operations[-1]
+        self._next_pos[last_op.offset] = target_block.startpos
+        block.set_exits([target_block])
 
 
 class BytecodeBlock(object):
diff --git a/rpython/flowspace/flowcontext.py b/rpython/flowspace/flowcontext.py
--- a/rpython/flowspace/flowcontext.py
+++ b/rpython/flowspace/flowcontext.py
@@ -15,6 +15,7 @@
 from rpython.flowspace.framestate import FrameState
 from rpython.flowspace.specialcase import (rpython_print_item,
     rpython_print_newline)
+from rpython.flowspace.bytecode import BytecodeBlock
 from rpython.flowspace.operation import op
 from rpython.flowspace.bytecode import BytecodeCorruption
 
@@ -449,13 +450,18 @@
                     break
 
     def handle_bytecode(self, next_offset):
-        self.last_offset = next_offset
+        bc_graph = self.pycode.graph
         next_offset, instr = self.pycode.read(next_offset)
+        self.last_offset = instr.offset
         try:
             offset = instr.eval(self)
-            return offset if offset is not None else next_offset
         except FlowSignal as signal:
             return self.unroll(signal)
+        if next_offset is None:
+            next_offset = bc_graph.next_pos(instr)
+        elif isinstance(next_offset, BytecodeBlock):
+            next_offset = next_offset.startpos
+        return next_offset
 
     def unroll(self, signal):
         while self.blockstack:

From noreply at buildbot.pypy.org  Mon Nov 24 17:29:50 2014
From: noreply at buildbot.pypy.org (rlamy)
Date: Mon, 24 Nov 2014 17:29:50 +0100 (CET)
Subject: [pypy-commit] pypy framestate: Add Opcodes JUMP_FORWARD and
	JUMP_ABSOLUTE
Message-ID: <20141124162950.4B54F1C3271@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy 
Branch: framestate
Changeset: r74680:0b2185c0097b
Date: 2013-09-15 06:22 +0100
http://bitbucket.org/pypy/pypy/changeset/0b2185c0097b/

Log:	Add Opcodes JUMP_FORWARD and JUMP_ABSOLUTE

diff --git a/rpython/flowspace/bytecode.py b/rpython/flowspace/bytecode.py
--- a/rpython/flowspace/bytecode.py
+++ b/rpython/flowspace/bytecode.py
@@ -271,6 +271,20 @@
         return getattr(ctx, self.name)(self.arg)
 
 
+def flow_opcode(func):
+    name = func.__name__
+    class Op(BCInstruction):
+        def __init__(self, arg=0, offset=-1):
+            self.arg = arg
+            self.offset = offset
+
+        def eval(self, ctx):
+            pass
+    Op.__name__ = Op.name = name
+    Op.bc_flow = func
+    bc_reader.register_opcode(Op)
+    return Op
+
 @bc_reader.register_opcode
 class LOAD_CONST(BCInstruction):
     @staticmethod
@@ -280,6 +294,16 @@
     def eval(self, ctx):
         ctx.pushvalue(const(self.arg))
 
+ at flow_opcode
+def JUMP_ABSOLUTE(self, block, graph):
+    target_block, _ = graph.pos_index[self.arg]
+    graph.add_jump(block, target_block)
+
+ at flow_opcode
+def JUMP_FORWARD(self, block, graph):
+    target_block, _ = graph.pos_index[self.arg]
+    graph.add_jump(block, target_block)
+
 _unary_ops = [
     ('UNARY_POSITIVE', op.pos),
     ('UNARY_NEGATIVE', op.neg),
diff --git a/rpython/flowspace/flowcontext.py b/rpython/flowspace/flowcontext.py
--- a/rpython/flowspace/flowcontext.py
+++ b/rpython/flowspace/flowcontext.py
@@ -666,7 +666,7 @@
         block.cleanupstack(self)  # the block knows how to clean up the value stack
 
     def JUMP_ABSOLUTE(self, jumpto):
-        return jumpto
+        pass
 
     def YIELD_VALUE(self, _):
         assert self.pycode.is_generator
@@ -689,7 +689,7 @@
         self.appcall(rpython_print_newline)
 
     def JUMP_FORWARD(self, target):
-        return target
+        pass
 
     def JUMP_IF_FALSE(self, target):
         # Python <= 2.6 only

From noreply at buildbot.pypy.org  Mon Nov 24 17:29:51 2014
From: noreply at buildbot.pypy.org (rlamy)
Date: Mon, 24 Nov 2014 17:29:51 +0100 (CET)
Subject: [pypy-commit] pypy framestate: remove superseded FSFrame opcode
	methods
Message-ID: <20141124162951.76C611C3271@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy 
Branch: framestate
Changeset: r74681:986166f4326e
Date: 2013-08-11 21:37 +0100
http://bitbucket.org/pypy/pypy/changeset/986166f4326e/

Log:	remove superseded FSFrame opcode methods

diff --git a/rpython/flowspace/flowcontext.py b/rpython/flowspace/flowcontext.py
--- a/rpython/flowspace/flowcontext.py
+++ b/rpython/flowspace/flowcontext.py
@@ -665,9 +665,6 @@
         block = self.blockstack.pop()
         block.cleanupstack(self)  # the block knows how to clean up the value stack
 
-    def JUMP_ABSOLUTE(self, jumpto):
-        pass
-
     def YIELD_VALUE(self, _):
         assert self.pycode.is_generator
         w_result = self.popvalue()
@@ -688,9 +685,6 @@
     def PRINT_NEWLINE(self, oparg):
         self.appcall(rpython_print_newline)
 
-    def JUMP_FORWARD(self, target):
-        pass
-
     def JUMP_IF_FALSE(self, target):
         # Python <= 2.6 only
         w_cond = self.peekvalue()
@@ -793,10 +787,6 @@
             raise FlowingError("Local variable referenced before assignment")
         self.pushvalue(w_value)
 
-    def LOAD_CONST(self, constant):
-        w_const = const(constant)
-        self.pushvalue(w_const)
-
     def find_global(self, w_globals, varname):
         try:
             value = w_globals.value[varname]

From noreply at buildbot.pypy.org  Mon Nov 24 17:29:52 2014
From: noreply at buildbot.pypy.org (rlamy)
Date: Mon, 24 Nov 2014 17:29:52 +0100 (CET)
Subject: [pypy-commit] pypy framestate: handle POP_JUMP_IF_FALSE in the
	bc_flow() phase
Message-ID: <20141124162952.A248A1C3271@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy 
Branch: framestate
Changeset: r74682:1bd3dde04463
Date: 2014-05-10 05:38 +0100
http://bitbucket.org/pypy/pypy/changeset/1bd3dde04463/

Log:	handle POP_JUMP_IF_FALSE in the bc_flow() phase

diff --git a/rpython/flowspace/bytecode.py b/rpython/flowspace/bytecode.py
--- a/rpython/flowspace/bytecode.py
+++ b/rpython/flowspace/bytecode.py
@@ -295,6 +295,26 @@
         ctx.pushvalue(const(self.arg))
 
 @flow_opcode
+def POP_JUMP_IF_FALSE(self, block, graph):
+    on_False, _ = graph.pos_index[self.arg]
+    on_True, _ = graph.pos_index[graph.next_pos(self)]
+    block.operations[-1] = SWITCH_BOOL(on_False, on_True, offset=self.offset)
+    block.set_exits([on_False, on_True])
+
+class SWITCH_BOOL(BCInstruction):
+    def __init__(self, on_False, on_True, offset=-1):
+        self.on_False = on_False
+        self.on_True = on_True
+        self.offset = offset
+
+    def eval(self, ctx):
+        w_value = ctx.popvalue()
+        if ctx.guessbool(op.bool(w_value).eval(ctx)):
+            return self.on_True
+        else:
+            return self.on_False
+
+ at flow_opcode
 def JUMP_ABSOLUTE(self, block, graph):
     target_block, _ = graph.pos_index[self.arg]
     graph.add_jump(block, target_block)
diff --git a/rpython/flowspace/flowcontext.py b/rpython/flowspace/flowcontext.py
--- a/rpython/flowspace/flowcontext.py
+++ b/rpython/flowspace/flowcontext.py
@@ -697,11 +697,6 @@
         if self.guessbool(op.bool(w_cond).eval(self)):
             return target
 
-    def POP_JUMP_IF_FALSE(self, target):
-        w_value = self.popvalue()
-        if not self.guessbool(op.bool(w_value).eval(self)):
-            return target
-
     def POP_JUMP_IF_TRUE(self, target):
         w_value = self.popvalue()
         if self.guessbool(op.bool(w_value).eval(self)):

From noreply at buildbot.pypy.org  Mon Nov 24 17:29:53 2014
From: noreply at buildbot.pypy.org (rlamy)
Date: Mon, 24 Nov 2014 17:29:53 +0100 (CET)
Subject: [pypy-commit] pypy framestate: handle POP_JUMP_IF_TRUE in the
	bc_flow() phase
Message-ID: <20141124162953.C6D971C3271@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy 
Branch: framestate
Changeset: r74683:4d0aeb03a210
Date: 2013-05-06 15:51 +0100
http://bitbucket.org/pypy/pypy/changeset/4d0aeb03a210/

Log:	handle POP_JUMP_IF_TRUE in the bc_flow() phase

diff --git a/rpython/flowspace/bytecode.py b/rpython/flowspace/bytecode.py
--- a/rpython/flowspace/bytecode.py
+++ b/rpython/flowspace/bytecode.py
@@ -301,6 +301,13 @@
     block.operations[-1] = SWITCH_BOOL(on_False, on_True, offset=self.offset)
     block.set_exits([on_False, on_True])
 
+ at flow_opcode
+def POP_JUMP_IF_TRUE(self, block, graph):
+    on_True, _ = graph.pos_index[self.arg]
+    on_False, _ = graph.pos_index[graph.next_pos(self)]
+    block.operations[-1] = SWITCH_BOOL(on_False, on_True, offset=self.offset)
+    block.set_exits([on_False, on_True])
+
 class SWITCH_BOOL(BCInstruction):
     def __init__(self, on_False, on_True, offset=-1):
         self.on_False = on_False
diff --git a/rpython/flowspace/flowcontext.py b/rpython/flowspace/flowcontext.py
--- a/rpython/flowspace/flowcontext.py
+++ b/rpython/flowspace/flowcontext.py
@@ -697,11 +697,6 @@
         if self.guessbool(op.bool(w_cond).eval(self)):
             return target
 
-    def POP_JUMP_IF_TRUE(self, target):
-        w_value = self.popvalue()
-        if self.guessbool(op.bool(w_value).eval(self)):
-            return target
-
     def JUMP_IF_FALSE_OR_POP(self, target):
         w_value = self.peekvalue()
         if not self.guessbool(op.bool(w_value).eval(self)):

From noreply at buildbot.pypy.org  Mon Nov 24 17:29:54 2014
From: noreply at buildbot.pypy.org (rlamy)
Date: Mon, 24 Nov 2014 17:29:54 +0100 (CET)
Subject: [pypy-commit] pypy framestate: Add SETUP_EXCEPT instruction
Message-ID: <20141124162954.ECFD11C3271@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy 
Branch: framestate
Changeset: r74684:1ee1acf65ff1
Date: 2014-05-05 02:28 +0100
http://bitbucket.org/pypy/pypy/changeset/1ee1acf65ff1/

Log:	Add SETUP_EXCEPT instruction

diff --git a/rpython/flowspace/bytecode.py b/rpython/flowspace/bytecode.py
--- a/rpython/flowspace/bytecode.py
+++ b/rpython/flowspace/bytecode.py
@@ -331,6 +331,13 @@
     target_block, _ = graph.pos_index[self.arg]
     graph.add_jump(block, target_block)
 
+ at bc_reader.register_opcode
+class SETUP_EXCEPT(BCInstruction):
+    def eval(self, ctx):
+        from rpython.flowspace.flowcontext import ExceptBlock
+        block = ExceptBlock(ctx, self.arg)
+        ctx.blockstack.append(block)
+
 _unary_ops = [
     ('UNARY_POSITIVE', op.pos),
     ('UNARY_NEGATIVE', op.neg),
diff --git a/rpython/flowspace/flowcontext.py b/rpython/flowspace/flowcontext.py
--- a/rpython/flowspace/flowcontext.py
+++ b/rpython/flowspace/flowcontext.py
@@ -734,10 +734,6 @@
         block = LoopBlock(self, target)
         self.blockstack.append(block)
 
-    def SETUP_EXCEPT(self, target):
-        block = ExceptBlock(self, target)
-        self.blockstack.append(block)
-
     def SETUP_FINALLY(self, target):
         block = FinallyBlock(self, target)
         self.blockstack.append(block)

From noreply at buildbot.pypy.org  Mon Nov 24 17:29:56 2014
From: noreply at buildbot.pypy.org (rlamy)
Date: Mon, 24 Nov 2014 17:29:56 +0100 (CET)
Subject: [pypy-commit] pypy framestate: pass instructions into
 ctx.handle_bytecode() instead of offsets
Message-ID: <20141124162956.206061C3271@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy 
Branch: framestate
Changeset: r74685:33f277317ece
Date: 2014-05-11 17:11 +0100
http://bitbucket.org/pypy/pypy/changeset/33f277317ece/

Log:	pass instructions into ctx.handle_bytecode() instead of offsets

diff --git a/rpython/flowspace/flowcontext.py b/rpython/flowspace/flowcontext.py
--- a/rpython/flowspace/flowcontext.py
+++ b/rpython/flowspace/flowcontext.py
@@ -349,9 +349,12 @@
         self.setstate(block.framestate)
         next_offset = block.framestate.next_offset
         self.recorder = block.make_recorder()
+        bc_graph = self.pycode.graph
         try:
             while True:
-                next_offset = self.handle_bytecode(next_offset)
+                instr = bc_graph.read(next_offset)
+                self.last_offset = instr.offset
+                next_offset = self.handle_bytecode(instr)
                 self.recorder.final_state = self.getstate(next_offset)
 
         except RaiseImplicit as e:
@@ -449,12 +452,10 @@
                     stack_items_w[i] = w_new
                     break
 
-    def handle_bytecode(self, next_offset):
+    def handle_bytecode(self, instr):
         bc_graph = self.pycode.graph
-        next_offset, instr = self.pycode.read(next_offset)
-        self.last_offset = instr.offset
         try:
-            offset = instr.eval(self)
+            next_offset = instr.eval(self)
         except FlowSignal as signal:
             return self.unroll(signal)
         if next_offset is None:

From noreply at buildbot.pypy.org  Mon Nov 24 17:29:57 2014
From: noreply at buildbot.pypy.org (rlamy)
Date: Mon, 24 Nov 2014 17:29:57 +0100 (CET)
Subject: [pypy-commit] pypy framestate: create bc_reader.build_code()
Message-ID: <20141124162957.4DF101C3271@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy 
Branch: framestate
Changeset: r74686:1eb9dc57448a
Date: 2014-11-18 15:19 +0000
http://bitbucket.org/pypy/pypy/changeset/1eb9dc57448a/

Log:	create bc_reader.build_code()

diff --git a/rpython/flowspace/bytecode.py b/rpython/flowspace/bytecode.py
--- a/rpython/flowspace/bytecode.py
+++ b/rpython/flowspace/bytecode.py
@@ -169,6 +169,9 @@
             op = GenericOpcode(self.opnames[opnum], opnum, oparg, offset)
         return next_offset, op
 
+    def build_code(self, code):
+        return HostCode._from_code(code)
+
 bc_reader = BytecodeReader(host_bytecode_spec.method_names)
 
 class BytecodeGraph(object):
diff --git a/rpython/flowspace/generator.py b/rpython/flowspace/generator.py
--- a/rpython/flowspace/generator.py
+++ b/rpython/flowspace/generator.py
@@ -1,7 +1,7 @@
 """Flow graph building for generators"""
 
 from rpython.flowspace.argument import Signature
-from rpython.flowspace.bytecode import HostCode
+from rpython.flowspace.bytecode import bc_reader
 from rpython.flowspace.pygraph import PyGraph
 from rpython.flowspace.model import (Block, Link, Variable,
     Constant, checkgraph, const)
@@ -18,7 +18,7 @@
 def make_generator_entry_graph(func):
     # This is the first copy of the graph.  We replace it with
     # a small bootstrap graph.
-    code = HostCode._from_code(func.func_code)
+    code = bc_reader.build_code(func.func_code)
     graph = PyGraph(func, code)
     block = graph.startblock
     for name, w_value in zip(code.co_varnames, block.framestate.mergeable):
diff --git a/rpython/flowspace/objspace.py b/rpython/flowspace/objspace.py
--- a/rpython/flowspace/objspace.py
+++ b/rpython/flowspace/objspace.py
@@ -4,7 +4,7 @@
 from inspect import CO_NEWLOCALS, isgeneratorfunction
 
 from rpython.flowspace.model import checkgraph
-from rpython.flowspace.bytecode import HostCode
+from rpython.flowspace.bytecode import bc_reader
 from rpython.flowspace.flowcontext import (FlowContext, fixeggblocks)
 from rpython.flowspace.generator import (tweak_generator_graph,
         make_generator_entry_graph)
@@ -36,7 +36,7 @@
     if (isgeneratorfunction(func) and
             not hasattr(func, '_generator_next_method_of_')):
         return make_generator_entry_graph(func)
-    code = HostCode._from_code(func.func_code)
+    code = bc_reader.build_code(func.func_code)
     graph = PyGraph(func, code)
     ctx = FlowContext(graph, code)
     ctx.build_flow()
diff --git a/rpython/flowspace/test/test_framestate.py b/rpython/flowspace/test/test_framestate.py
--- a/rpython/flowspace/test/test_framestate.py
+++ b/rpython/flowspace/test/test_framestate.py
@@ -1,7 +1,7 @@
-from rpython.flowspace.model import *
+from rpython.flowspace.model import Constant, Variable
 from rpython.rlib.unroll import SpecTag
 from rpython.flowspace.flowcontext import FlowContext
-from rpython.flowspace.bytecode import HostCode
+from rpython.flowspace.bytecode import bc_reader
 from rpython.flowspace.pygraph import PyGraph
 
 class TestFrameState:
@@ -10,7 +10,7 @@
             func = func.im_func
         except AttributeError:
             pass
-        code = HostCode._from_code(func.func_code)
+        code = bc_reader.build_code(func.func_code)
         graph = PyGraph(func, code)
         ctx = FlowContext(graph, code)
         # hack the frame

From noreply at buildbot.pypy.org  Mon Nov 24 17:29:58 2014
From: noreply at buildbot.pypy.org (rlamy)
Date: Mon, 24 Nov 2014 17:29:58 +0100 (CET)
Subject: [pypy-commit] pypy framestate: initialise HostCode.graph more
	explicitly
Message-ID: <20141124162958.6EA631C3271@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy 
Branch: framestate
Changeset: r74687:d2bc0f5d6391
Date: 2014-05-11 17:41 +0100
http://bitbucket.org/pypy/pypy/changeset/d2bc0f5d6391/

Log:	initialise HostCode.graph more explicitly

diff --git a/rpython/flowspace/bytecode.py b/rpython/flowspace/bytecode.py
--- a/rpython/flowspace/bytecode.py
+++ b/rpython/flowspace/bytecode.py
@@ -56,7 +56,7 @@
         self.co_firstlineno = firstlineno
         self.co_lnotab = lnotab
         self.signature = cpython_code_signature(self)
-        self.build_flow()
+        self.graph = self.build_flow()
 
     def build_flow(self):
         contents, offsets, jumps = bc_reader.disassemble(self)
@@ -66,7 +66,7 @@
         pendingblocks = [SimpleBlock(contents[i:j])
                 for i, j in zip([0] + cuts, cuts + [len(self.co_code)])]
 
-        graph = self.graph = BytecodeGraph(pendingblocks[0])
+        graph = BytecodeGraph(pendingblocks[0])
         for block in pendingblocks:
             for i, op in enumerate(block.operations):
                 graph.pos_index[op.offset] = block, i
@@ -77,6 +77,7 @@
             block = pendingblocks.pop()
             for i, op in enumerate(block.operations):
                 op.bc_flow(block, graph)
+        return graph
 
     @classmethod
     def _from_code(cls, code):

From noreply at buildbot.pypy.org  Mon Nov 24 17:29:59 2014
From: noreply at buildbot.pypy.org (rlamy)
Date: Mon, 24 Nov 2014 17:29:59 +0100 (CET)
Subject: [pypy-commit] pypy framestate: move .build_flow() to BytecodeReader
Message-ID: <20141124162959.A12661C3271@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy 
Branch: framestate
Changeset: r74688:7338c63fc059
Date: 2014-05-11 18:00 +0100
http://bitbucket.org/pypy/pypy/changeset/7338c63fc059/

Log:	move .build_flow() to BytecodeReader

diff --git a/rpython/flowspace/bytecode.py b/rpython/flowspace/bytecode.py
--- a/rpython/flowspace/bytecode.py
+++ b/rpython/flowspace/bytecode.py
@@ -56,28 +56,7 @@
         self.co_firstlineno = firstlineno
         self.co_lnotab = lnotab
         self.signature = cpython_code_signature(self)
-        self.graph = self.build_flow()
-
-    def build_flow(self):
-        contents, offsets, jumps = bc_reader.disassemble(self)
-        pos_map = dict([(pos, i) for i, pos in enumerate(offsets)])
-        cuts = sorted([pos_map[n] + 1 for n in jumps.keys()] +
-                [pos_map[n] for n in jumps.values()])
-        pendingblocks = [SimpleBlock(contents[i:j])
-                for i, j in zip([0] + cuts, cuts + [len(self.co_code)])]
-
-        graph = BytecodeGraph(pendingblocks[0])
-        for block in pendingblocks:
-            for i, op in enumerate(block.operations):
-                graph.pos_index[op.offset] = block, i
-        graph._next_pos = dict([(offsets[i], offsets[i+1])
-            for i in range(len(offsets) - 1)])
-        graph._next_pos[offsets[-1]] = len(self.co_code)
-        while pendingblocks:
-            block = pendingblocks.pop()
-            for i, op in enumerate(block.operations):
-                op.bc_flow(block, graph)
-        return graph
+        self.graph = bc_reader.build_flow(self)
 
     @classmethod
     def _from_code(cls, code):
@@ -170,6 +149,27 @@
             op = GenericOpcode(self.opnames[opnum], opnum, oparg, offset)
         return next_offset, op
 
+    def build_flow(self, code):
+        contents, offsets, jumps = self.disassemble(code)
+        pos_map = dict([(pos, i) for i, pos in enumerate(offsets)])
+        cuts = sorted([pos_map[n] + 1 for n in jumps.keys()] +
+                [pos_map[n] for n in jumps.values()])
+        pendingblocks = [SimpleBlock(contents[i:j])
+                for i, j in zip([0] + cuts, cuts + [len(code.co_code)])]
+
+        graph = BytecodeGraph(pendingblocks[0])
+        for block in pendingblocks:
+            for i, op in enumerate(block.operations):
+                graph.pos_index[op.offset] = block, i
+        graph._next_pos = dict([(offsets[i], offsets[i+1])
+            for i in range(len(offsets) - 1)])
+        graph._next_pos[offsets[-1]] = len(code.co_code)
+        while pendingblocks:
+            block = pendingblocks.pop()
+            for i, op in enumerate(block.operations):
+                op.bc_flow(block, graph)
+        return graph
+
     def build_code(self, code):
         return HostCode._from_code(code)
 

From noreply at buildbot.pypy.org  Mon Nov 24 17:30:00 2014
From: noreply at buildbot.pypy.org (rlamy)
Date: Mon, 24 Nov 2014 17:30:00 +0100 (CET)
Subject: [pypy-commit] pypy framestate: simplify FrameBlock constructor
Message-ID: <20141124163000.CE51F1C3271@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy 
Branch: framestate
Changeset: r74689:fd00fc579edc
Date: 2014-11-22 01:27 +0000
http://bitbucket.org/pypy/pypy/changeset/fd00fc579edc/

Log:	simplify FrameBlock constructor

diff --git a/rpython/flowspace/bytecode.py b/rpython/flowspace/bytecode.py
--- a/rpython/flowspace/bytecode.py
+++ b/rpython/flowspace/bytecode.py
@@ -339,7 +339,7 @@
 class SETUP_EXCEPT(BCInstruction):
     def eval(self, ctx):
         from rpython.flowspace.flowcontext import ExceptBlock
-        block = ExceptBlock(ctx, self.arg)
+        block = ExceptBlock(ctx.stackdepth, self.arg)
         ctx.blockstack.append(block)
 
 _unary_ops = [
diff --git a/rpython/flowspace/flowcontext.py b/rpython/flowspace/flowcontext.py
--- a/rpython/flowspace/flowcontext.py
+++ b/rpython/flowspace/flowcontext.py
@@ -732,11 +732,11 @@
                 raise
 
     def SETUP_LOOP(self, target):
-        block = LoopBlock(self, target)
+        block = LoopBlock(self.stackdepth, target)
         self.blockstack.append(block)
 
     def SETUP_FINALLY(self, target):
-        block = FinallyBlock(self, target)
+        block = FinallyBlock(self.stackdepth, target)
         self.blockstack.append(block)
 
     def SETUP_WITH(self, target):
@@ -748,7 +748,7 @@
         self.settopvalue(w_exit)
         w_enter = op.getattr(w_manager, const('__enter__')).eval(self)
         w_result = op.simple_call(w_enter).eval(self)
-        block = WithBlock(self, target)
+        block = WithBlock(self.stackdepth, target)
         self.blockstack.append(block)
         self.pushvalue(w_result)
 
@@ -1188,9 +1188,9 @@
     """Abstract base class for frame blocks from the blockstack,
     used by the SETUP_XXX and POP_BLOCK opcodes."""
 
-    def __init__(self, ctx, handlerposition):
+    def __init__(self, stackdepth, handlerposition):
         self.handlerposition = handlerposition
-        self.stackdepth = ctx.stackdepth
+        self.stackdepth = stackdepth
 
     def __eq__(self, other):
         return (self.__class__ is other.__class__ and

From noreply at buildbot.pypy.org  Mon Nov 24 17:30:02 2014
From: noreply at buildbot.pypy.org (rlamy)
Date: Mon, 24 Nov 2014 17:30:02 +0100 (CET)
Subject: [pypy-commit] pypy framestate: start refactoring
	bc_reader.build_flow()
Message-ID: <20141124163002.008401C3271@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy 
Branch: framestate
Changeset: r74690:2e6b3ffcdaab
Date: 2014-11-22 20:25 +0000
http://bitbucket.org/pypy/pypy/changeset/2e6b3ffcdaab/

Log:	start refactoring bc_reader.build_flow()

diff --git a/rpython/flowspace/bytecode.py b/rpython/flowspace/bytecode.py
--- a/rpython/flowspace/bytecode.py
+++ b/rpython/flowspace/bytecode.py
@@ -96,22 +96,6 @@
         cls.num = self.register_name(name, cls)
         return cls
 
-    def disassemble(self, code):
-        contents = []
-        offsets = []
-        jumps = {}
-        pos = 0
-        i = 0
-        while pos < len(code.co_code):
-            offsets.append(pos)
-            next_pos, op = self.read(code, pos)
-            contents.append(op)
-            if op.has_jump():
-                jumps[pos] = op.arg
-            pos = next_pos
-            i += 1
-        return contents, offsets, jumps
-
     def read(self, code, offset):
         """
         Decode the instruction starting at position ``offset``.
@@ -150,10 +134,25 @@
         return next_offset, op
 
     def build_flow(self, code):
-        contents, offsets, jumps = self.disassemble(code)
-        pos_map = dict([(pos, i) for i, pos in enumerate(offsets)])
-        cuts = sorted([pos_map[n] + 1 for n in jumps.keys()] +
-                [pos_map[n] for n in jumps.values()])
+        contents = []
+        offsets = []
+        jumps = {}
+        pos_map = {}
+        cuts = []
+        pos = 0
+        i = 0
+        while pos < len(code.co_code):
+            offsets.append(pos)
+            pos_map[pos] = i
+            next_pos, op = self.read(code, pos)
+            contents.append(op)
+            if op.has_jump():
+                jumps[pos] = op.arg
+                cuts.append(i + 1)
+            pos = next_pos
+            i += 1
+        cuts.extend([pos_map[n] for n in jumps.values()])
+        cuts.sort()
         pendingblocks = [SimpleBlock(contents[i:j])
                 for i, j in zip([0] + cuts, cuts + [len(code.co_code)])]
 

From noreply at buildbot.pypy.org  Mon Nov 24 17:30:03 2014
From: noreply at buildbot.pypy.org (rlamy)
Date: Mon, 24 Nov 2014 17:30:03 +0100 (CET)
Subject: [pypy-commit] pypy framestate: wip
Message-ID: <20141124163003.2A6371C3271@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy 
Branch: framestate
Changeset: r74691:426f97e0e042
Date: 2014-11-22 22:23 +0000
http://bitbucket.org/pypy/pypy/changeset/426f97e0e042/

Log:	wip

diff --git a/rpython/flowspace/bytecode.py b/rpython/flowspace/bytecode.py
--- a/rpython/flowspace/bytecode.py
+++ b/rpython/flowspace/bytecode.py
@@ -133,23 +133,27 @@
             op = GenericOpcode(self.opnames[opnum], opnum, oparg, offset)
         return next_offset, op
 
+    def _iter_instr(self, code):
+        while self.offset < len(code.co_code):
+            next_offset, instr = self.read(code, self.offset)
+            yield instr
+            self.offset = next_offset
+
     def build_flow(self, code):
         contents = []
         offsets = []
         jumps = {}
         pos_map = {}
         cuts = []
-        pos = 0
+        self.offset = 0
         i = 0
-        while pos < len(code.co_code):
-            offsets.append(pos)
-            pos_map[pos] = i
-            next_pos, op = self.read(code, pos)
-            contents.append(op)
-            if op.has_jump():
-                jumps[pos] = op.arg
+        for instr in self._iter_instr(code):
+            offsets.append(self.offset)
+            pos_map[self.offset] = i
+            contents.append(instr)
+            if instr.has_jump():
+                jumps[self.offset] = instr.arg
                 cuts.append(i + 1)
-            pos = next_pos
             i += 1
         cuts.extend([pos_map[n] for n in jumps.values()])
         cuts.sort()

From noreply at buildbot.pypy.org  Mon Nov 24 17:30:04 2014
From: noreply at buildbot.pypy.org (rlamy)
Date: Mon, 24 Nov 2014 17:30:04 +0100 (CET)
Subject: [pypy-commit] pypy framestate: wip
Message-ID: <20141124163004.78FB51C3271@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy 
Branch: framestate
Changeset: r74692:f6216e53dcae
Date: 2014-11-22 22:44 +0000
http://bitbucket.org/pypy/pypy/changeset/f6216e53dcae/

Log:	wip

diff --git a/rpython/flowspace/bytecode.py b/rpython/flowspace/bytecode.py
--- a/rpython/flowspace/bytecode.py
+++ b/rpython/flowspace/bytecode.py
@@ -134,31 +134,34 @@
         return next_offset, op
 
     def _iter_instr(self, code):
+        self.offset = 0
         while self.offset < len(code.co_code):
             next_offset, instr = self.read(code, self.offset)
             yield instr
             self.offset = next_offset
 
+    def _add_jump(self, i, target):
+        self.cuts.append(i + 1)
+        self.pending_cuts.append(target)
+
     def build_flow(self, code):
         contents = []
         offsets = []
-        jumps = {}
+        self.pending_cuts = []
         pos_map = {}
-        cuts = []
-        self.offset = 0
+        self.cuts = []
         i = 0
         for instr in self._iter_instr(code):
             offsets.append(self.offset)
             pos_map[self.offset] = i
             contents.append(instr)
             if instr.has_jump():
-                jumps[self.offset] = instr.arg
-                cuts.append(i + 1)
+                self._add_jump(i, instr.arg)
             i += 1
-        cuts.extend([pos_map[n] for n in jumps.values()])
-        cuts.sort()
+        self.cuts.extend([pos_map[n] for n in self.pending_cuts])
+        self.cuts.sort()
         pendingblocks = [SimpleBlock(contents[i:j])
-                for i, j in zip([0] + cuts, cuts + [len(code.co_code)])]
+                for i, j in zip([0] + self.cuts, self.cuts + [len(code.co_code)])]
 
         graph = BytecodeGraph(pendingblocks[0])
         for block in pendingblocks:

From noreply at buildbot.pypy.org  Mon Nov 24 17:30:05 2014
From: noreply at buildbot.pypy.org (rlamy)
Date: Mon, 24 Nov 2014 17:30:05 +0100 (CET)
Subject: [pypy-commit] pypy framestate: wip
Message-ID: <20141124163005.9A7B71C3271@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy 
Branch: framestate
Changeset: r74693:dc249e852d88
Date: 2014-11-22 22:54 +0000
http://bitbucket.org/pypy/pypy/changeset/dc249e852d88/

Log:	wip

diff --git a/rpython/flowspace/bytecode.py b/rpython/flowspace/bytecode.py
--- a/rpython/flowspace/bytecode.py
+++ b/rpython/flowspace/bytecode.py
@@ -142,23 +142,27 @@
 
     def _add_jump(self, i, target):
         self.cuts.append(i + 1)
-        self.pending_cuts.append(target)
+        if target <= self.offset:
+            j = self.pos_map[target]
+            self.cuts.append(j)
+        else:
+            self.pending_cuts.append(target)
 
     def build_flow(self, code):
         contents = []
         offsets = []
         self.pending_cuts = []
-        pos_map = {}
+        self.pos_map = {}
         self.cuts = []
         i = 0
         for instr in self._iter_instr(code):
             offsets.append(self.offset)
-            pos_map[self.offset] = i
+            self.pos_map[self.offset] = i
             contents.append(instr)
             if instr.has_jump():
                 self._add_jump(i, instr.arg)
             i += 1
-        self.cuts.extend([pos_map[n] for n in self.pending_cuts])
+        self.cuts.extend([self.pos_map[n] for n in self.pending_cuts])
         self.cuts.sort()
         pendingblocks = [SimpleBlock(contents[i:j])
                 for i, j in zip([0] + self.cuts, self.cuts + [len(code.co_code)])]

From noreply at buildbot.pypy.org  Mon Nov 24 17:30:06 2014
From: noreply at buildbot.pypy.org (rlamy)
Date: Mon, 24 Nov 2014 17:30:06 +0100 (CET)
Subject: [pypy-commit] pypy framestate: wip
Message-ID: <20141124163006.C0B971C3271@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy 
Branch: framestate
Changeset: r74694:e380b790156e
Date: 2014-11-22 23:32 +0000
http://bitbucket.org/pypy/pypy/changeset/e380b790156e/

Log:	wip

diff --git a/rpython/flowspace/bytecode.py b/rpython/flowspace/bytecode.py
--- a/rpython/flowspace/bytecode.py
+++ b/rpython/flowspace/bytecode.py
@@ -135,10 +135,15 @@
 
     def _iter_instr(self, code):
         self.offset = 0
+        i = 0
         while self.offset < len(code.co_code):
+            self.pos_map[self.offset] = i
+            if self.offset in self.pending_cuts:
+                self.cuts.append(self.pos_map[self.offset])
             next_offset, instr = self.read(code, self.offset)
             yield instr
             self.offset = next_offset
+            i += 1
 
     def _add_jump(self, i, target):
         self.cuts.append(i + 1)
@@ -146,23 +151,19 @@
             j = self.pos_map[target]
             self.cuts.append(j)
         else:
-            self.pending_cuts.append(target)
+            self.pending_cuts.add(target)
 
     def build_flow(self, code):
         contents = []
         offsets = []
-        self.pending_cuts = []
+        self.pending_cuts = set()
         self.pos_map = {}
         self.cuts = []
-        i = 0
         for instr in self._iter_instr(code):
             offsets.append(self.offset)
-            self.pos_map[self.offset] = i
             contents.append(instr)
             if instr.has_jump():
-                self._add_jump(i, instr.arg)
-            i += 1
-        self.cuts.extend([self.pos_map[n] for n in self.pending_cuts])
+                self._add_jump(self.pos_map[self.offset], instr.arg)
         self.cuts.sort()
         pendingblocks = [SimpleBlock(contents[i:j])
                 for i, j in zip([0] + self.cuts, self.cuts + [len(code.co_code)])]

From noreply at buildbot.pypy.org  Mon Nov 24 17:30:07 2014
From: noreply at buildbot.pypy.org (rlamy)
Date: Mon, 24 Nov 2014 17:30:07 +0100 (CET)
Subject: [pypy-commit] pypy framestate: wip
Message-ID: <20141124163007.E2FEA1C3271@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy 
Branch: framestate
Changeset: r74695:71547c416c37
Date: 2014-11-23 01:57 +0000
http://bitbucket.org/pypy/pypy/changeset/71547c416c37/

Log:	wip

diff --git a/rpython/flowspace/bytecode.py b/rpython/flowspace/bytecode.py
--- a/rpython/flowspace/bytecode.py
+++ b/rpython/flowspace/bytecode.py
@@ -139,44 +139,54 @@
         while self.offset < len(code.co_code):
             self.pos_map[self.offset] = i
             if self.offset in self.pending_cuts:
-                self.cuts.append(self.pos_map[self.offset])
+                self.new_block()
             next_offset, instr = self.read(code, self.offset)
             yield instr
             self.offset = next_offset
             i += 1
 
-    def _add_jump(self, i, target):
-        self.cuts.append(i + 1)
+    def find_position(self, offset):
+        for i, block in enumerate(self.blocks):
+            if block.startpos <= offset:
+                n = i
+            else:
+                break
+        for i, instr in enumerate(self.blocks[n]):
+            if instr.offset == offset:
+                return n, i
+
+    def new_block(self):
+        if self.blocks[-1].operations:
+            new_block = SimpleBlock([])
+            self.blocks.append(new_block)
+
+    def _add_jump(self, target):
         if target <= self.offset:
-            j = self.pos_map[target]
-            self.cuts.append(j)
+            i_block, i_instr = self.find_position(target)
+            self.blocks[i_block:i_block + 1] = self.blocks[i_block].split_at(i_instr)
         else:
             self.pending_cuts.add(target)
 
     def build_flow(self, code):
-        contents = []
         offsets = []
         self.pending_cuts = set()
         self.pos_map = {}
-        self.cuts = []
+        self.blocks = [SimpleBlock([])]
         for instr in self._iter_instr(code):
             offsets.append(self.offset)
-            contents.append(instr)
+            self.blocks[-1].operations.append(instr)
             if instr.has_jump():
-                self._add_jump(self.pos_map[self.offset], instr.arg)
-        self.cuts.sort()
-        pendingblocks = [SimpleBlock(contents[i:j])
-                for i, j in zip([0] + self.cuts, self.cuts + [len(code.co_code)])]
+                self.new_block()
+                self._add_jump(instr.arg)
 
-        graph = BytecodeGraph(pendingblocks[0])
-        for block in pendingblocks:
+        graph = BytecodeGraph(self.blocks[0])
+        for block in self.blocks:
             for i, op in enumerate(block.operations):
                 graph.pos_index[op.offset] = block, i
-        graph._next_pos = dict([(offsets[i], offsets[i+1])
+        graph._next_pos = dict([(offsets[i], offsets[i + 1])
             for i in range(len(offsets) - 1)])
         graph._next_pos[offsets[-1]] = len(code.co_code)
-        while pendingblocks:
-            block = pendingblocks.pop()
+        for block in self.blocks:
             for i, op in enumerate(block.operations):
                 op.bc_flow(block, graph)
         return graph
@@ -235,6 +245,16 @@
     def startpos(self):
         return self.operations[0].offset
 
+    def split_at(self, i):
+        if i == 0 or i == len(self.operations):
+            return [self]
+        assert 0 < i < len(self.operations)
+        tail = self.operations[i:]
+        assert tail
+        del self.operations[i:]
+        new_block = SimpleBlock(tail)
+        return [self, new_block]
+
 
 class EntryBlock(BytecodeBlock):
     """A fake block to represent the beginning of a code object"""

From noreply at buildbot.pypy.org  Mon Nov 24 17:30:09 2014
From: noreply at buildbot.pypy.org (rlamy)
Date: Mon, 24 Nov 2014 17:30:09 +0100 (CET)
Subject: [pypy-commit] pypy framestate: move graph.pos_index init into the
 main loop of build_flow()
Message-ID: <20141124163009.193EA1C3271@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy 
Branch: framestate
Changeset: r74696:0cd515a70a90
Date: 2014-11-23 02:58 +0000
http://bitbucket.org/pypy/pypy/changeset/0cd515a70a90/

Log:	move graph.pos_index init into the main loop of build_flow()

diff --git a/rpython/flowspace/bytecode.py b/rpython/flowspace/bytecode.py
--- a/rpython/flowspace/bytecode.py
+++ b/rpython/flowspace/bytecode.py
@@ -137,7 +137,6 @@
         self.offset = 0
         i = 0
         while self.offset < len(code.co_code):
-            self.pos_map[self.offset] = i
             if self.offset in self.pending_cuts:
                 self.new_block()
             next_offset, instr = self.read(code, self.offset)
@@ -163,26 +162,29 @@
     def _add_jump(self, target):
         if target <= self.offset:
             i_block, i_instr = self.find_position(target)
-            self.blocks[i_block:i_block + 1] = self.blocks[i_block].split_at(i_instr)
+            split = self.blocks[i_block].split_at(i_instr)
+            if len(split) == 2:
+                new_block = split[1]
+                for i, instr in enumerate(new_block.operations):
+                    self.graph.pos_index[instr.offset] = new_block, i
+            self.blocks[i_block:i_block + 1] = split
         else:
             self.pending_cuts.add(target)
 
     def build_flow(self, code):
         offsets = []
         self.pending_cuts = set()
-        self.pos_map = {}
         self.blocks = [SimpleBlock([])]
+        self.graph = graph = BytecodeGraph(self.blocks[0])
         for instr in self._iter_instr(code):
             offsets.append(self.offset)
-            self.blocks[-1].operations.append(instr)
+            block = self.blocks[-1]
+            graph.pos_index[self.offset] = block, len(block.operations)
+            block.operations.append(instr)
             if instr.has_jump():
                 self.new_block()
                 self._add_jump(instr.arg)
 
-        graph = BytecodeGraph(self.blocks[0])
-        for block in self.blocks:
-            for i, op in enumerate(block.operations):
-                graph.pos_index[op.offset] = block, i
         graph._next_pos = dict([(offsets[i], offsets[i + 1])
             for i in range(len(offsets) - 1)])
         graph._next_pos[offsets[-1]] = len(code.co_code)

From noreply at buildbot.pypy.org  Mon Nov 24 17:30:10 2014
From: noreply at buildbot.pypy.org (rlamy)
Date: Mon, 24 Nov 2014 17:30:10 +0100 (CET)
Subject: [pypy-commit] pypy framestate: move graph._next_pos init into the
 main loop of build_flow()
Message-ID: <20141124163010.407C61C3271@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy 
Branch: framestate
Changeset: r74697:d60d6e647179
Date: 2014-11-23 03:06 +0000
http://bitbucket.org/pypy/pypy/changeset/d60d6e647179/

Log:	move graph._next_pos init into the main loop of build_flow()

diff --git a/rpython/flowspace/bytecode.py b/rpython/flowspace/bytecode.py
--- a/rpython/flowspace/bytecode.py
+++ b/rpython/flowspace/bytecode.py
@@ -176,17 +176,18 @@
         self.pending_cuts = set()
         self.blocks = [SimpleBlock([])]
         self.graph = graph = BytecodeGraph(self.blocks[0])
+        last_offset = -1
         for instr in self._iter_instr(code):
-            offsets.append(self.offset)
+            offsets.append(instr.offset)
             block = self.blocks[-1]
-            graph.pos_index[self.offset] = block, len(block.operations)
+            graph.pos_index[instr.offset] = block, len(block.operations)
+            graph._next_pos[last_offset] = instr.offset
             block.operations.append(instr)
             if instr.has_jump():
                 self.new_block()
                 self._add_jump(instr.arg)
+            last_offset = instr.offset
 
-        graph._next_pos = dict([(offsets[i], offsets[i + 1])
-            for i in range(len(offsets) - 1)])
         graph._next_pos[offsets[-1]] = len(code.co_code)
         for block in self.blocks:
             for i, op in enumerate(block.operations):
@@ -203,6 +204,7 @@
         self.entry = EntryBlock()
         self.entry.set_exits([startblock])
         self.pos_index = {}
+        self._next_pos = {}
 
     def read(self, pos):
         bc_block, i = self.pos_index[pos]

From noreply at buildbot.pypy.org  Mon Nov 24 17:30:11 2014
From: noreply at buildbot.pypy.org (rlamy)
Date: Mon, 24 Nov 2014 17:30:11 +0100 (CET)
Subject: [pypy-commit] pypy framestate: Add .pending_blocks: mapping of
 target offsets to peredeclared blocks
Message-ID: <20141124163011.74D7E1C3271@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy 
Branch: framestate
Changeset: r74698:30190c212671
Date: 2014-11-23 19:04 +0000
http://bitbucket.org/pypy/pypy/changeset/30190c212671/

Log:	Add .pending_blocks: mapping of target offsets to peredeclared
	blocks

diff --git a/rpython/flowspace/bytecode.py b/rpython/flowspace/bytecode.py
--- a/rpython/flowspace/bytecode.py
+++ b/rpython/flowspace/bytecode.py
@@ -137,8 +137,11 @@
         self.offset = 0
         i = 0
         while self.offset < len(code.co_code):
-            if self.offset in self.pending_cuts:
-                self.new_block()
+            if self.offset in self.pending_blocks:
+                new_block = self.pending_blocks[self.offset]
+                if not self.blocks[-1].operations:
+                    self.blocks.pop()
+                self.blocks.append(new_block)
             next_offset, instr = self.read(code, self.offset)
             yield instr
             self.offset = next_offset
@@ -155,25 +158,27 @@
                 return n, i
 
     def new_block(self):
-        if self.blocks[-1].operations:
-            new_block = SimpleBlock([])
-            self.blocks.append(new_block)
+        return SimpleBlock([])
 
-    def _add_jump(self, target):
-        if target <= self.offset:
-            i_block, i_instr = self.find_position(target)
+    def get_block_at(self, offset):
+        """Get or create the block starting at ``offset``"""
+        if offset <= self.offset:
+            i_block, i_instr = self.find_position(offset)
             split = self.blocks[i_block].split_at(i_instr)
             if len(split) == 2:
                 new_block = split[1]
                 for i, instr in enumerate(new_block.operations):
                     self.graph.pos_index[instr.offset] = new_block, i
             self.blocks[i_block:i_block + 1] = split
+            return split[-1]
         else:
-            self.pending_cuts.add(target)
+            new_block = self.new_block()
+            self.pending_blocks[offset] = new_block
+            return self.new_block()
 
     def build_flow(self, code):
         offsets = []
-        self.pending_cuts = set()
+        self.pending_blocks = {}
         self.blocks = [SimpleBlock([])]
         self.graph = graph = BytecodeGraph(self.blocks[0])
         last_offset = -1
@@ -184,8 +189,9 @@
             graph._next_pos[last_offset] = instr.offset
             block.operations.append(instr)
             if instr.has_jump():
-                self.new_block()
-                self._add_jump(instr.arg)
+                new_block = self.new_block()
+                self.blocks.append(new_block)
+                self.get_block_at(instr.arg)
             last_offset = instr.offset
 
         graph._next_pos[offsets[-1]] = len(code.co_code)

From noreply at buildbot.pypy.org  Mon Nov 24 17:30:12 2014
From: noreply at buildbot.pypy.org (rlamy)
Date: Mon, 24 Nov 2014 17:30:12 +0100 (CET)
Subject: [pypy-commit] pypy framestate: wip
Message-ID: <20141124163012.9EE041C3271@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy 
Branch: framestate
Changeset: r74699:217d9dc3f557
Date: 2014-11-23 20:00 +0000
http://bitbucket.org/pypy/pypy/changeset/217d9dc3f557/

Log:	wip

diff --git a/rpython/flowspace/bytecode.py b/rpython/flowspace/bytecode.py
--- a/rpython/flowspace/bytecode.py
+++ b/rpython/flowspace/bytecode.py
@@ -139,9 +139,9 @@
         while self.offset < len(code.co_code):
             if self.offset in self.pending_blocks:
                 new_block = self.pending_blocks[self.offset]
-                if not self.blocks[-1].operations:
+                if not self.curr_block.operations:
                     self.blocks.pop()
-                self.blocks.append(new_block)
+                self.enter_next_block(new_block)
             next_offset, instr = self.read(code, self.offset)
             yield instr
             self.offset = next_offset
@@ -176,26 +176,32 @@
             self.pending_blocks[offset] = new_block
             return self.new_block()
 
+    def enter_next_block(self, block):
+        self.curr_block = block
+        self.blocks.append(block)
+
     def build_flow(self, code):
         offsets = []
         self.pending_blocks = {}
         self.blocks = [SimpleBlock([])]
+        self.curr_block = self.blocks[0]
         self.graph = graph = BytecodeGraph(self.blocks[0])
         last_offset = -1
         for instr in self._iter_instr(code):
             offsets.append(instr.offset)
-            block = self.blocks[-1]
+            block = self.curr_block
             graph.pos_index[instr.offset] = block, len(block.operations)
             graph._next_pos[last_offset] = instr.offset
             block.operations.append(instr)
             if instr.has_jump():
                 new_block = self.new_block()
-                self.blocks.append(new_block)
+                self.enter_next_block(new_block)
                 self.get_block_at(instr.arg)
             last_offset = instr.offset
 
         graph._next_pos[offsets[-1]] = len(code.co_code)
         for block in self.blocks:
+            self.curr_block = block
             for i, op in enumerate(block.operations):
                 op.bc_flow(block, graph)
         return graph

From noreply at buildbot.pypy.org  Mon Nov 24 17:30:13 2014
From: noreply at buildbot.pypy.org (rlamy)
Date: Mon, 24 Nov 2014 17:30:13 +0100 (CET)
Subject: [pypy-commit] pypy framestate: Extract opcode-specific parts of
 build_flow() into a method
Message-ID: <20141124163013.CD4BD1C3271@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy 
Branch: framestate
Changeset: r74700:f31d045f9bbd
Date: 2014-11-24 01:51 +0000
http://bitbucket.org/pypy/pypy/changeset/f31d045f9bbd/

Log:	Extract opcode-specific parts of build_flow() into a method

diff --git a/rpython/flowspace/bytecode.py b/rpython/flowspace/bytecode.py
--- a/rpython/flowspace/bytecode.py
+++ b/rpython/flowspace/bytecode.py
@@ -192,18 +192,14 @@
             block = self.curr_block
             graph.pos_index[instr.offset] = block, len(block.operations)
             graph._next_pos[last_offset] = instr.offset
-            block.operations.append(instr)
-            if instr.has_jump():
-                new_block = self.new_block()
-                self.enter_next_block(new_block)
-                self.get_block_at(instr.arg)
+            instr.prepare_flow(self)
             last_offset = instr.offset
 
         graph._next_pos[offsets[-1]] = len(code.co_code)
         for block in self.blocks:
             self.curr_block = block
             for i, op in enumerate(block.operations):
-                op.bc_flow(block, graph)
+                op.bc_flow(self)
         return graph
 
     def build_code(self, code):
@@ -302,7 +298,15 @@
     def eval(self, ctx):
         pass
 
-    def bc_flow(self, block, graph):
+    def prepare_flow(self, reader):
+        block = reader.curr_block
+        block.operations.append(self)
+        if self.has_jump():
+            new_block = reader.new_block()
+            reader.enter_next_block(new_block)
+            reader.get_block_at(self.arg)
+
+    def bc_flow(self, reader):
         pass
 
     def has_jump(self):
@@ -346,14 +350,18 @@
         ctx.pushvalue(const(self.arg))
 
 @flow_opcode
-def POP_JUMP_IF_FALSE(self, block, graph):
+def POP_JUMP_IF_FALSE(self, reader):
+    block = reader.curr_block
+    graph = reader.graph
     on_False, _ = graph.pos_index[self.arg]
     on_True, _ = graph.pos_index[graph.next_pos(self)]
     block.operations[-1] = SWITCH_BOOL(on_False, on_True, offset=self.offset)
     block.set_exits([on_False, on_True])
 
 @flow_opcode
-def POP_JUMP_IF_TRUE(self, block, graph):
+def POP_JUMP_IF_TRUE(self, reader):
+    block = reader.curr_block
+    graph = reader.graph
     on_True, _ = graph.pos_index[self.arg]
     on_False, _ = graph.pos_index[graph.next_pos(self)]
     block.operations[-1] = SWITCH_BOOL(on_False, on_True, offset=self.offset)
@@ -373,12 +381,16 @@
             return self.on_False
 
 @flow_opcode
-def JUMP_ABSOLUTE(self, block, graph):
+def JUMP_ABSOLUTE(self, reader):
+    block = reader.curr_block
+    graph = reader.graph
     target_block, _ = graph.pos_index[self.arg]
     graph.add_jump(block, target_block)
 
 @flow_opcode
-def JUMP_FORWARD(self, block, graph):
+def JUMP_FORWARD(self, reader):
+    block = reader.curr_block
+    graph = reader.graph
     target_block, _ = graph.pos_index[self.arg]
     graph.add_jump(block, target_block)
 

From noreply at buildbot.pypy.org  Mon Nov 24 17:30:15 2014
From: noreply at buildbot.pypy.org (rlamy)
Date: Mon, 24 Nov 2014 17:30:15 +0100 (CET)
Subject: [pypy-commit] pypy framestate: wip
Message-ID: <20141124163015.13EE61C3271@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy 
Branch: framestate
Changeset: r74701:50ce8f28910e
Date: 2014-11-24 02:03 +0000
http://bitbucket.org/pypy/pypy/changeset/50ce8f28910e/

Log:	wip

diff --git a/rpython/flowspace/bytecode.py b/rpython/flowspace/bytecode.py
--- a/rpython/flowspace/bytecode.py
+++ b/rpython/flowspace/bytecode.py
@@ -353,8 +353,8 @@
 def POP_JUMP_IF_FALSE(self, reader):
     block = reader.curr_block
     graph = reader.graph
-    on_False, _ = graph.pos_index[self.arg]
-    on_True, _ = graph.pos_index[graph.next_pos(self)]
+    on_False = reader.get_block_at(self.arg)
+    on_True = reader.get_block_at(graph.next_pos(self))
     block.operations[-1] = SWITCH_BOOL(on_False, on_True, offset=self.offset)
     block.set_exits([on_False, on_True])
 
@@ -362,8 +362,8 @@
 def POP_JUMP_IF_TRUE(self, reader):
     block = reader.curr_block
     graph = reader.graph
-    on_True, _ = graph.pos_index[self.arg]
-    on_False, _ = graph.pos_index[graph.next_pos(self)]
+    on_True = reader.get_block_at(self.arg)
+    on_False = reader.get_block_at(graph.next_pos(self))
     block.operations[-1] = SWITCH_BOOL(on_False, on_True, offset=self.offset)
     block.set_exits([on_False, on_True])
 
@@ -384,14 +384,14 @@
 def JUMP_ABSOLUTE(self, reader):
     block = reader.curr_block
     graph = reader.graph
-    target_block, _ = graph.pos_index[self.arg]
+    target_block = reader.get_block_at(self.arg)
     graph.add_jump(block, target_block)
 
 @flow_opcode
 def JUMP_FORWARD(self, reader):
     block = reader.curr_block
     graph = reader.graph
-    target_block, _ = graph.pos_index[self.arg]
+    target_block = reader.get_block_at(self.arg)
     graph.add_jump(block, target_block)
 
 @bc_reader.register_opcode

From noreply at buildbot.pypy.org  Mon Nov 24 17:30:16 2014
From: noreply at buildbot.pypy.org (rlamy)
Date: Mon, 24 Nov 2014 17:30:16 +0100 (CET)
Subject: [pypy-commit] pypy framestate: make prepare_flow() methods explicit
	for flow opcodes
Message-ID: <20141124163016.3B7201C3271@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy 
Branch: framestate
Changeset: r74702:ac3ec0b9b518
Date: 2014-11-24 02:12 +0000
http://bitbucket.org/pypy/pypy/changeset/ac3ec0b9b518/

Log:	make prepare_flow() methods explicit for flow opcodes

diff --git a/rpython/flowspace/bytecode.py b/rpython/flowspace/bytecode.py
--- a/rpython/flowspace/bytecode.py
+++ b/rpython/flowspace/bytecode.py
@@ -358,6 +358,15 @@
     block.operations[-1] = SWITCH_BOOL(on_False, on_True, offset=self.offset)
     block.set_exits([on_False, on_True])
 
+def prepare(self, reader):
+    block = reader.curr_block
+    block.operations.append(self)
+    if self.has_jump():
+        new_block = reader.new_block()
+        reader.enter_next_block(new_block)
+        reader.get_block_at(self.arg)
+POP_JUMP_IF_FALSE.prepare_flow = prepare
+
 @flow_opcode
 def POP_JUMP_IF_TRUE(self, reader):
     block = reader.curr_block
@@ -367,6 +376,15 @@
     block.operations[-1] = SWITCH_BOOL(on_False, on_True, offset=self.offset)
     block.set_exits([on_False, on_True])
 
+def prepare(self, reader):
+    block = reader.curr_block
+    block.operations.append(self)
+    if self.has_jump():
+        new_block = reader.new_block()
+        reader.enter_next_block(new_block)
+        reader.get_block_at(self.arg)
+POP_JUMP_IF_TRUE.prepare_flow = prepare
+
 class SWITCH_BOOL(BCInstruction):
     def __init__(self, on_False, on_True, offset=-1):
         self.on_False = on_False
@@ -387,6 +405,15 @@
     target_block = reader.get_block_at(self.arg)
     graph.add_jump(block, target_block)
 
+def prepare(self, reader):
+    block = reader.curr_block
+    block.operations.append(self)
+    if self.has_jump():
+        new_block = reader.new_block()
+        reader.enter_next_block(new_block)
+        reader.get_block_at(self.arg)
+JUMP_ABSOLUTE.prepare_flow = prepare
+
 @flow_opcode
 def JUMP_FORWARD(self, reader):
     block = reader.curr_block
@@ -394,6 +421,15 @@
     target_block = reader.get_block_at(self.arg)
     graph.add_jump(block, target_block)
 
+def prepare(self, reader):
+    block = reader.curr_block
+    block.operations.append(self)
+    if self.has_jump():
+        new_block = reader.new_block()
+        reader.enter_next_block(new_block)
+        reader.get_block_at(self.arg)
+JUMP_FORWARD.prepare_flow = prepare
+
 @bc_reader.register_opcode
 class SETUP_EXCEPT(BCInstruction):
     def eval(self, ctx):

From noreply at buildbot.pypy.org  Mon Nov 24 17:30:17 2014
From: noreply at buildbot.pypy.org (rlamy)
Date: Mon, 24 Nov 2014 17:30:17 +0100 (CET)
Subject: [pypy-commit] pypy framestate: wip
Message-ID: <20141124163017.5B7A21C3271@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy 
Branch: framestate
Changeset: r74703:b4f0ba5e0250
Date: 2014-11-24 02:18 +0000
http://bitbucket.org/pypy/pypy/changeset/b4f0ba5e0250/

Log:	wip

diff --git a/rpython/flowspace/bytecode.py b/rpython/flowspace/bytecode.py
--- a/rpython/flowspace/bytecode.py
+++ b/rpython/flowspace/bytecode.py
@@ -361,10 +361,9 @@
 def prepare(self, reader):
     block = reader.curr_block
     block.operations.append(self)
-    if self.has_jump():
-        new_block = reader.new_block()
-        reader.enter_next_block(new_block)
-        reader.get_block_at(self.arg)
+    new_block = reader.new_block()
+    reader.enter_next_block(new_block)
+    reader.get_block_at(self.arg)
 POP_JUMP_IF_FALSE.prepare_flow = prepare
 
 @flow_opcode
@@ -379,10 +378,9 @@
 def prepare(self, reader):
     block = reader.curr_block
     block.operations.append(self)
-    if self.has_jump():
-        new_block = reader.new_block()
-        reader.enter_next_block(new_block)
-        reader.get_block_at(self.arg)
+    new_block = reader.new_block()
+    reader.enter_next_block(new_block)
+    reader.get_block_at(self.arg)
 POP_JUMP_IF_TRUE.prepare_flow = prepare
 
 class SWITCH_BOOL(BCInstruction):
@@ -408,10 +406,9 @@
 def prepare(self, reader):
     block = reader.curr_block
     block.operations.append(self)
-    if self.has_jump():
-        new_block = reader.new_block()
-        reader.enter_next_block(new_block)
-        reader.get_block_at(self.arg)
+    new_block = reader.new_block()
+    reader.enter_next_block(new_block)
+    reader.get_block_at(self.arg)
 JUMP_ABSOLUTE.prepare_flow = prepare
 
 @flow_opcode
@@ -424,10 +421,9 @@
 def prepare(self, reader):
     block = reader.curr_block
     block.operations.append(self)
-    if self.has_jump():
-        new_block = reader.new_block()
-        reader.enter_next_block(new_block)
-        reader.get_block_at(self.arg)
+    new_block = reader.new_block()
+    reader.enter_next_block(new_block)
+    reader.get_block_at(self.arg)
 JUMP_FORWARD.prepare_flow = prepare
 
 @bc_reader.register_opcode

From noreply at buildbot.pypy.org  Mon Nov 24 17:30:18 2014
From: noreply at buildbot.pypy.org (rlamy)
Date: Mon, 24 Nov 2014 17:30:18 +0100 (CET)
Subject: [pypy-commit] pypy framestate: wip
Message-ID: <20141124163018.797B11C3271@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy 
Branch: framestate
Changeset: r74704:fa3cd9adfed5
Date: 2014-11-24 02:19 +0000
http://bitbucket.org/pypy/pypy/changeset/fa3cd9adfed5/

Log:	wip

diff --git a/rpython/flowspace/bytecode.py b/rpython/flowspace/bytecode.py
--- a/rpython/flowspace/bytecode.py
+++ b/rpython/flowspace/bytecode.py
@@ -221,9 +221,9 @@
     def next_pos(self, opcode):
         return self._next_pos[opcode.offset]
 
-    def add_jump(self, block, target_block):
+    def add_jump(self, block, target_block, target_offset):
         last_op = block.operations[-1]
-        self._next_pos[last_op.offset] = target_block.startpos
+        self._next_pos[last_op.offset] = target_offset
         block.set_exits([target_block])
 
 
@@ -401,7 +401,7 @@
     block = reader.curr_block
     graph = reader.graph
     target_block = reader.get_block_at(self.arg)
-    graph.add_jump(block, target_block)
+    graph.add_jump(block, target_block, self.arg)
 
 def prepare(self, reader):
     block = reader.curr_block
@@ -416,7 +416,7 @@
     block = reader.curr_block
     graph = reader.graph
     target_block = reader.get_block_at(self.arg)
-    graph.add_jump(block, target_block)
+    graph.add_jump(block, target_block, self.arg)
 
 def prepare(self, reader):
     block = reader.curr_block

From noreply at buildbot.pypy.org  Mon Nov 24 17:30:19 2014
From: noreply at buildbot.pypy.org (rlamy)
Date: Mon, 24 Nov 2014 17:30:19 +0100 (CET)
Subject: [pypy-commit] pypy framestate: almost kill JUMP_XXX.bc_flow()
Message-ID: <20141124163019.9EA1D1C3271@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy 
Branch: framestate
Changeset: r74705:48d5421e14d0
Date: 2014-11-24 02:55 +0000
http://bitbucket.org/pypy/pypy/changeset/48d5421e14d0/

Log:	almost kill JUMP_XXX.bc_flow()

diff --git a/rpython/flowspace/bytecode.py b/rpython/flowspace/bytecode.py
--- a/rpython/flowspace/bytecode.py
+++ b/rpython/flowspace/bytecode.py
@@ -398,32 +398,30 @@
 
 @flow_opcode
 def JUMP_ABSOLUTE(self, reader):
-    block = reader.curr_block
-    graph = reader.graph
-    target_block = reader.get_block_at(self.arg)
-    graph.add_jump(block, target_block, self.arg)
+    reader.graph._next_pos[self.offset] = self.arg
 
 def prepare(self, reader):
     block = reader.curr_block
+    graph = reader.graph
     block.operations.append(self)
     new_block = reader.new_block()
     reader.enter_next_block(new_block)
-    reader.get_block_at(self.arg)
+    target_block = reader.get_block_at(self.arg)
+    graph.add_jump(block, target_block, self.arg)
 JUMP_ABSOLUTE.prepare_flow = prepare
 
 @flow_opcode
 def JUMP_FORWARD(self, reader):
-    block = reader.curr_block
-    graph = reader.graph
-    target_block = reader.get_block_at(self.arg)
-    graph.add_jump(block, target_block, self.arg)
+    reader.graph._next_pos[self.offset] = self.arg
 
 def prepare(self, reader):
     block = reader.curr_block
+    graph = reader.graph
     block.operations.append(self)
     new_block = reader.new_block()
     reader.enter_next_block(new_block)
-    reader.get_block_at(self.arg)
+    target_block = reader.get_block_at(self.arg)
+    graph.add_jump(block, target_block, self.arg)
 JUMP_FORWARD.prepare_flow = prepare
 
 @bc_reader.register_opcode

From noreply at buildbot.pypy.org  Mon Nov 24 17:30:20 2014
From: noreply at buildbot.pypy.org (rlamy)
Date: Mon, 24 Nov 2014 17:30:20 +0100 (CET)
Subject: [pypy-commit] pypy framestate: move graph._next_pos init to
 _iter_instr() and really kill JUMP_XXX.bc_flow()
Message-ID: <20141124163020.C7DFF1C3271@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy 
Branch: framestate
Changeset: r74706:f048ea78f140
Date: 2014-11-24 03:09 +0000
http://bitbucket.org/pypy/pypy/changeset/f048ea78f140/

Log:	move graph._next_pos init to _iter_instr() and really kill
	JUMP_XXX.bc_flow()

diff --git a/rpython/flowspace/bytecode.py b/rpython/flowspace/bytecode.py
--- a/rpython/flowspace/bytecode.py
+++ b/rpython/flowspace/bytecode.py
@@ -136,7 +136,9 @@
     def _iter_instr(self, code):
         self.offset = 0
         i = 0
+        last_offset = -1
         while self.offset < len(code.co_code):
+            self.graph._next_pos.setdefault(last_offset, self.offset)
             if self.offset in self.pending_blocks:
                 new_block = self.pending_blocks[self.offset]
                 if not self.curr_block.operations:
@@ -144,6 +146,7 @@
                 self.enter_next_block(new_block)
             next_offset, instr = self.read(code, self.offset)
             yield instr
+            last_offset = instr.offset
             self.offset = next_offset
             i += 1
 
@@ -186,14 +189,11 @@
         self.blocks = [SimpleBlock([])]
         self.curr_block = self.blocks[0]
         self.graph = graph = BytecodeGraph(self.blocks[0])
-        last_offset = -1
         for instr in self._iter_instr(code):
             offsets.append(instr.offset)
             block = self.curr_block
             graph.pos_index[instr.offset] = block, len(block.operations)
-            graph._next_pos[last_offset] = instr.offset
             instr.prepare_flow(self)
-            last_offset = instr.offset
 
         graph._next_pos[offsets[-1]] = len(code.co_code)
         for block in self.blocks:
@@ -398,7 +398,7 @@
 
 @flow_opcode
 def JUMP_ABSOLUTE(self, reader):
-    reader.graph._next_pos[self.offset] = self.arg
+    pass
 
 def prepare(self, reader):
     block = reader.curr_block
@@ -412,7 +412,7 @@
 
 @flow_opcode
 def JUMP_FORWARD(self, reader):
-    reader.graph._next_pos[self.offset] = self.arg
+    pass
 
 def prepare(self, reader):
     block = reader.curr_block

From noreply at buildbot.pypy.org  Mon Nov 24 17:30:21 2014
From: noreply at buildbot.pypy.org (rlamy)
Date: Mon, 24 Nov 2014 17:30:21 +0100 (CET)
Subject: [pypy-commit] pypy framestate: Merge prepare_flow() and bc_flow()
Message-ID: <20141124163021.E9B081C3271@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy 
Branch: framestate
Changeset: r74707:d1f78f96c21e
Date: 2014-11-24 02:56 +0000
http://bitbucket.org/pypy/pypy/changeset/d1f78f96c21e/

Log:	Merge prepare_flow() and bc_flow()

diff --git a/rpython/flowspace/bytecode.py b/rpython/flowspace/bytecode.py
--- a/rpython/flowspace/bytecode.py
+++ b/rpython/flowspace/bytecode.py
@@ -140,10 +140,14 @@
         while self.offset < len(code.co_code):
             self.graph._next_pos.setdefault(last_offset, self.offset)
             if self.offset in self.pending_blocks:
-                new_block = self.pending_blocks[self.offset]
+                next_block = self.pending_blocks[self.offset]
                 if not self.curr_block.operations:
+                    import pdb; pdb.set_trace
                     self.blocks.pop()
-                self.enter_next_block(new_block)
+                self.enter_next_block(next_block)
+            elif self.needs_new_block:
+                next_block = self.new_block()
+                self.enter_next_block(next_block)
             next_offset, instr = self.read(code, self.offset)
             yield instr
             last_offset = instr.offset
@@ -175,31 +179,37 @@
             self.blocks[i_block:i_block + 1] = split
             return split[-1]
         else:
+            if offset in self.pending_blocks:
+                return self.pending_blocks[offset]
             new_block = self.new_block()
             self.pending_blocks[offset] = new_block
-            return self.new_block()
+            return new_block
 
     def enter_next_block(self, block):
         self.curr_block = block
         self.blocks.append(block)
+        self.needs_new_block = False
+
+    def end_block(self):
+        self.needs_new_block = True
 
     def build_flow(self, code):
         offsets = []
         self.pending_blocks = {}
         self.blocks = [SimpleBlock([])]
         self.curr_block = self.blocks[0]
+        self.needs_new_block = False
         self.graph = graph = BytecodeGraph(self.blocks[0])
         for instr in self._iter_instr(code):
             offsets.append(instr.offset)
             block = self.curr_block
             graph.pos_index[instr.offset] = block, len(block.operations)
-            instr.prepare_flow(self)
+            instr.bc_flow(self)
 
         graph._next_pos[offsets[-1]] = len(code.co_code)
-        for block in self.blocks:
-            self.curr_block = block
-            for i, op in enumerate(block.operations):
-                op.bc_flow(self)
+        for b in self.blocks:
+            for x in b._exits:
+                assert x in self.blocks
         return graph
 
     def build_code(self, code):
@@ -298,17 +308,13 @@
     def eval(self, ctx):
         pass
 
-    def prepare_flow(self, reader):
-        block = reader.curr_block
-        block.operations.append(self)
+    def bc_flow(self, reader):
+        reader.curr_block.operations.append(self)
         if self.has_jump():
             new_block = reader.new_block()
             reader.enter_next_block(new_block)
             reader.get_block_at(self.arg)
 
-    def bc_flow(self, reader):
-        pass
-
     def has_jump(self):
         return self.num in opcode.hasjrel or self.num in opcode.hasjabs
 
@@ -351,39 +357,29 @@
 
 @flow_opcode
 def POP_JUMP_IF_FALSE(self, reader):
+    reader.curr_block.operations.append(self)
+    on_True = reader.new_block()
+    on_False = reader.get_block_at(self.arg)
     block = reader.curr_block
     graph = reader.graph
-    on_False = reader.get_block_at(self.arg)
-    on_True = reader.get_block_at(graph.next_pos(self))
     block.operations[-1] = SWITCH_BOOL(on_False, on_True, offset=self.offset)
     block.set_exits([on_False, on_True])
-
-def prepare(self, reader):
-    block = reader.curr_block
-    block.operations.append(self)
-    new_block = reader.new_block()
-    reader.enter_next_block(new_block)
-    reader.get_block_at(self.arg)
-POP_JUMP_IF_FALSE.prepare_flow = prepare
+    reader.enter_next_block(on_True)
 
 @flow_opcode
 def POP_JUMP_IF_TRUE(self, reader):
+    reader.curr_block.operations.append(self)
+    on_False = reader.new_block()
+    on_True = reader.get_block_at(self.arg)
     block = reader.curr_block
     graph = reader.graph
-    on_True = reader.get_block_at(self.arg)
-    on_False = reader.get_block_at(graph.next_pos(self))
     block.operations[-1] = SWITCH_BOOL(on_False, on_True, offset=self.offset)
     block.set_exits([on_False, on_True])
-
-def prepare(self, reader):
-    block = reader.curr_block
-    block.operations.append(self)
-    new_block = reader.new_block()
-    reader.enter_next_block(new_block)
-    reader.get_block_at(self.arg)
-POP_JUMP_IF_TRUE.prepare_flow = prepare
+    reader.enter_next_block(on_False)
 
 class SWITCH_BOOL(BCInstruction):
+    name = 'Switch'
+    arg = -42
     def __init__(self, on_False, on_True, offset=-1):
         self.on_False = on_False
         self.on_True = on_True
@@ -398,31 +394,21 @@
 
 @flow_opcode
 def JUMP_ABSOLUTE(self, reader):
-    pass
-
-def prepare(self, reader):
+    reader.curr_block.operations.append(self)
+    target_block = reader.get_block_at(self.arg)
     block = reader.curr_block
     graph = reader.graph
-    block.operations.append(self)
-    new_block = reader.new_block()
-    reader.enter_next_block(new_block)
-    target_block = reader.get_block_at(self.arg)
     graph.add_jump(block, target_block, self.arg)
-JUMP_ABSOLUTE.prepare_flow = prepare
+    reader.end_block()
 
 @flow_opcode
 def JUMP_FORWARD(self, reader):
-    pass
-
-def prepare(self, reader):
+    reader.curr_block.operations.append(self)
+    target_block = reader.get_block_at(self.arg)
     block = reader.curr_block
     graph = reader.graph
-    block.operations.append(self)
-    new_block = reader.new_block()
-    reader.enter_next_block(new_block)
-    target_block = reader.get_block_at(self.arg)
     graph.add_jump(block, target_block, self.arg)
-JUMP_FORWARD.prepare_flow = prepare
+    reader.end_block()
 
 @bc_reader.register_opcode
 class SETUP_EXCEPT(BCInstruction):

From noreply at buildbot.pypy.org  Mon Nov 24 17:30:23 2014
From: noreply at buildbot.pypy.org (rlamy)
Date: Mon, 24 Nov 2014 17:30:23 +0100 (CET)
Subject: [pypy-commit] pypy framestate: cleanup
Message-ID: <20141124163023.149C31C3271@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy 
Branch: framestate
Changeset: r74708:9219b6d856ab
Date: 2014-11-24 03:54 +0000
http://bitbucket.org/pypy/pypy/changeset/9219b6d856ab/

Log:	cleanup

diff --git a/rpython/flowspace/bytecode.py b/rpython/flowspace/bytecode.py
--- a/rpython/flowspace/bytecode.py
+++ b/rpython/flowspace/bytecode.py
@@ -194,19 +194,17 @@
         self.needs_new_block = True
 
     def build_flow(self, code):
-        offsets = []
         self.pending_blocks = {}
         self.blocks = [SimpleBlock([])]
         self.curr_block = self.blocks[0]
         self.needs_new_block = False
         self.graph = graph = BytecodeGraph(self.blocks[0])
         for instr in self._iter_instr(code):
-            offsets.append(instr.offset)
             block = self.curr_block
             graph.pos_index[instr.offset] = block, len(block.operations)
             instr.bc_flow(self)
 
-        graph._next_pos[offsets[-1]] = len(code.co_code)
+        graph._next_pos[instr.offset] = len(code.co_code)
         for b in self.blocks:
             for x in b._exits:
                 assert x in self.blocks

From noreply at buildbot.pypy.org  Mon Nov 24 17:30:24 2014
From: noreply at buildbot.pypy.org (rlamy)
Date: Mon, 24 Nov 2014 17:30:24 +0100 (CET)
Subject: [pypy-commit] pypy framestate: Add DUP_TOP, POP_TOP BCInstructions
Message-ID: <20141124163024.39C7A1C3271@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy 
Branch: framestate
Changeset: r74709:f6a875ed479f
Date: 2014-11-24 05:05 +0000
http://bitbucket.org/pypy/pypy/changeset/f6a875ed479f/

Log:	Add DUP_TOP, POP_TOP BCInstructions

diff --git a/rpython/flowspace/bytecode.py b/rpython/flowspace/bytecode.py
--- a/rpython/flowspace/bytecode.py
+++ b/rpython/flowspace/bytecode.py
@@ -353,6 +353,17 @@
     def eval(self, ctx):
         ctx.pushvalue(const(self.arg))
 
+ at bc_reader.register_opcode
+class DUP_TOP(BCInstruction):
+    def eval(self, ctx):
+        w_1 = ctx.peekvalue()
+        ctx.pushvalue(w_1)
+
+ at bc_reader.register_opcode
+class POP_TOP(BCInstruction):
+    def eval(self, ctx):
+        ctx.popvalue()
+
 @flow_opcode
 def POP_JUMP_IF_FALSE(self, reader):
     reader.curr_block.operations.append(self)
diff --git a/rpython/flowspace/flowcontext.py b/rpython/flowspace/flowcontext.py
--- a/rpython/flowspace/flowcontext.py
+++ b/rpython/flowspace/flowcontext.py
@@ -818,9 +818,6 @@
         raise FlowingError(
             "Attempting to modify global variable  %r." % (varname))
 
-    def POP_TOP(self, oparg):
-        self.popvalue()
-
     def ROT_TWO(self, oparg):
         w_1 = self.popvalue()
         w_2 = self.popvalue()
@@ -845,10 +842,6 @@
         self.pushvalue(w_3)
         self.pushvalue(w_2)
 
-    def DUP_TOP(self, oparg):
-        w_1 = self.peekvalue()
-        self.pushvalue(w_1)
-
     def DUP_TOPX(self, itemcount):
         delta = itemcount - 1
         while True:

From noreply at buildbot.pypy.org  Mon Nov 24 17:30:25 2014
From: noreply at buildbot.pypy.org (rlamy)
Date: Mon, 24 Nov 2014 17:30:25 +0100 (CET)
Subject: [pypy-commit] pypy framestate: Add reliable way of finding the next
	block
Message-ID: <20141124163025.6385A1C3271@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy 
Branch: framestate
Changeset: r74710:d8c996cb6344
Date: 2014-11-24 16:19 +0000
http://bitbucket.org/pypy/pypy/changeset/d8c996cb6344/

Log:	Add reliable way of finding the next block

diff --git a/rpython/flowspace/bytecode.py b/rpython/flowspace/bytecode.py
--- a/rpython/flowspace/bytecode.py
+++ b/rpython/flowspace/bytecode.py
@@ -146,9 +146,10 @@
                     self.blocks.pop()
                 self.enter_next_block(next_block)
             elif self.needs_new_block:
-                next_block = self.new_block()
+                next_block = self.get_next_block()
                 self.enter_next_block(next_block)
             next_offset, instr = self.read(code, self.offset)
+            self.next_offset = next_offset
             yield instr
             last_offset = instr.offset
             self.offset = next_offset
@@ -169,7 +170,7 @@
 
     def get_block_at(self, offset):
         """Get or create the block starting at ``offset``"""
-        if offset <= self.offset:
+        if offset < self.next_offset:
             i_block, i_instr = self.find_position(offset)
             split = self.blocks[i_block].split_at(i_instr)
             if len(split) == 2:
@@ -185,6 +186,10 @@
             self.pending_blocks[offset] = new_block
             return new_block
 
+    def get_next_block(self):
+        """Find or create the block starting at the next offset"""
+        return self.get_block_at(self.next_offset)
+
     def enter_next_block(self, block):
         self.curr_block = block
         self.blocks.append(block)
@@ -309,7 +314,7 @@
     def bc_flow(self, reader):
         reader.curr_block.operations.append(self)
         if self.has_jump():
-            new_block = reader.new_block()
+            new_block = reader.get_next_block()
             reader.enter_next_block(new_block)
             reader.get_block_at(self.arg)
 
@@ -367,7 +372,7 @@
 @flow_opcode
 def POP_JUMP_IF_FALSE(self, reader):
     reader.curr_block.operations.append(self)
-    on_True = reader.new_block()
+    on_True = reader.get_next_block()
     on_False = reader.get_block_at(self.arg)
     block = reader.curr_block
     graph = reader.graph
@@ -378,7 +383,7 @@
 @flow_opcode
 def POP_JUMP_IF_TRUE(self, reader):
     reader.curr_block.operations.append(self)
-    on_False = reader.new_block()
+    on_False = reader.get_next_block()
     on_True = reader.get_block_at(self.arg)
     block = reader.curr_block
     graph = reader.graph

From noreply at buildbot.pypy.org  Mon Nov 24 18:25:38 2014
From: noreply at buildbot.pypy.org (rlamy)
Date: Mon, 24 Nov 2014 18:25:38 +0100 (CET)
Subject: [pypy-commit] pypy framestate: Add JUMP_IF_FALSE_OR_POP,
	JUMP_IF_TRUE_OR_POP
Message-ID: <20141124172538.517FF1C1366@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy 
Branch: framestate
Changeset: r74711:177e61c285a5
Date: 2014-11-24 17:03 +0000
http://bitbucket.org/pypy/pypy/changeset/177e61c285a5/

Log:	Add JUMP_IF_FALSE_OR_POP, JUMP_IF_TRUE_OR_POP

diff --git a/rpython/flowspace/bytecode.py b/rpython/flowspace/bytecode.py
--- a/rpython/flowspace/bytecode.py
+++ b/rpython/flowspace/bytecode.py
@@ -294,6 +294,7 @@
 
 
 OPNAMES = host_bytecode_spec.method_names
+NO_ARG = -1
 
 class BCInstruction(object):
     """
@@ -375,7 +376,6 @@
     on_True = reader.get_next_block()
     on_False = reader.get_block_at(self.arg)
     block = reader.curr_block
-    graph = reader.graph
     block.operations[-1] = SWITCH_BOOL(on_False, on_True, offset=self.offset)
     block.set_exits([on_False, on_True])
     reader.enter_next_block(on_True)
@@ -386,14 +386,48 @@
     on_False = reader.get_next_block()
     on_True = reader.get_block_at(self.arg)
     block = reader.curr_block
-    graph = reader.graph
     block.operations[-1] = SWITCH_BOOL(on_False, on_True, offset=self.offset)
     block.set_exits([on_False, on_True])
     reader.enter_next_block(on_False)
 
+ at bc_reader.register_opcode
+class JUMP_IF_FALSE_OR_POP(BCInstruction):
+    def bc_flow(self, reader):
+        block = reader.curr_block
+        block.operations.append(self)
+        self.on_True = reader.get_next_block()
+        self.on_False = reader.get_block_at(self.arg)
+        block.set_exits([self.on_False, self.on_True])
+        reader.enter_next_block(self.on_True)
+
+    def eval(self, ctx):
+        w_value = ctx.peekvalue()
+        if not ctx.guessbool(op.bool(w_value).eval(ctx)):
+            return self.on_False
+        ctx.popvalue()
+        return self.on_True
+
+ at bc_reader.register_opcode
+class JUMP_IF_TRUE_OR_POP(BCInstruction):
+    def bc_flow(self, reader):
+        block = reader.curr_block
+        block.operations.append(self)
+        self.on_True = reader.get_block_at(self.arg)
+        self.on_False = reader.get_next_block()
+        block.set_exits([self.on_False, self.on_True])
+        reader.enter_next_block(self.on_False)
+
+    def eval(self, ctx):
+        w_value = ctx.peekvalue()
+        if ctx.guessbool(op.bool(w_value).eval(ctx)):
+            return self.on_True
+        ctx.popvalue()
+        return self.on_False
+
+
 class SWITCH_BOOL(BCInstruction):
-    name = 'Switch'
-    arg = -42
+    name = 'SWITCH_BOOL'
+    arg = NO_ARG
     def __init__(self, on_False, on_True, offset=-1):
         self.on_False = on_False
         self.on_True = on_True
diff --git a/rpython/flowspace/flowcontext.py b/rpython/flowspace/flowcontext.py
--- a/rpython/flowspace/flowcontext.py
+++ b/rpython/flowspace/flowcontext.py
@@ -698,19 +698,6 @@
         if self.guessbool(op.bool(w_cond).eval(self)):
             return target
 
-    def JUMP_IF_FALSE_OR_POP(self, target):
-        w_value = self.peekvalue()
-        if not self.guessbool(op.bool(w_value).eval(self)):
-            return target
-        self.popvalue()
-
-    def JUMP_IF_TRUE_OR_POP(self, target):
-        w_value = self.peekvalue()
-        if self.guessbool(op.bool(w_value).eval(self)):
-            return target
-            return target
-        self.popvalue()
-
     def JUMP_IF_NOT_DEBUG(self, target):
         pass
 

From noreply at buildbot.pypy.org  Mon Nov 24 18:25:39 2014
From: noreply at buildbot.pypy.org (rlamy)
Date: Mon, 24 Nov 2014 18:25:39 +0100 (CET)
Subject: [pypy-commit] pypy framestate: more validation of bc_graph: check
 that all blocks have exits
Message-ID: <20141124172539.987831C1366@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy 
Branch: framestate
Changeset: r74712:67c2646feb72
Date: 2014-11-24 17:25 +0000
http://bitbucket.org/pypy/pypy/changeset/67c2646feb72/

Log:	more validation of bc_graph: check that all blocks have exits

diff --git a/rpython/flowspace/bytecode.py b/rpython/flowspace/bytecode.py
--- a/rpython/flowspace/bytecode.py
+++ b/rpython/flowspace/bytecode.py
@@ -191,6 +191,8 @@
         return self.get_block_at(self.next_offset)
 
     def enter_next_block(self, block):
+        if not self.curr_block._exits:
+            self.curr_block.set_exits([block])
         self.curr_block = block
         self.blocks.append(block)
         self.needs_new_block = False
@@ -208,12 +210,17 @@
             block = self.curr_block
             graph.pos_index[instr.offset] = block, len(block.operations)
             instr.bc_flow(self)
+        graph._next_pos[instr.offset] = len(code.co_code)
+        self.check_graph()
+        return graph
 
-        graph._next_pos[instr.offset] = len(code.co_code)
+    def check_graph(self):
         for b in self.blocks:
+            if not b._exits:
+                assert any(instr.name in ('RETURN_VALUE', 'RAISE_VARARGS')
+                        for instr in b.operations)
             for x in b._exits:
                 assert x in self.blocks
-        return graph
 
     def build_code(self, code):
         return HostCode._from_code(code)
@@ -275,9 +282,10 @@
             return [self]
         assert 0 < i < len(self.operations)
         tail = self.operations[i:]
-        assert tail
         del self.operations[i:]
         new_block = SimpleBlock(tail)
+        new_block.set_exits(self._exits)
+        self.set_exits([new_block])
         return [self, new_block]
 
 

From noreply at buildbot.pypy.org  Mon Nov 24 19:07:59 2014
From: noreply at buildbot.pypy.org (rlamy)
Date: Mon, 24 Nov 2014 19:07:59 +0100 (CET)
Subject: [pypy-commit] pypy framestate: reimplement bc_graph.next_pos using
 pos_index and block exits
Message-ID: <20141124180759.4D1091C3271@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy 
Branch: framestate
Changeset: r74713:d8c9b7d1985f
Date: 2014-11-24 17:55 +0000
http://bitbucket.org/pypy/pypy/changeset/d8c9b7d1985f/

Log:	reimplement bc_graph.next_pos using pos_index and block exits

diff --git a/rpython/flowspace/bytecode.py b/rpython/flowspace/bytecode.py
--- a/rpython/flowspace/bytecode.py
+++ b/rpython/flowspace/bytecode.py
@@ -192,6 +192,7 @@
 
     def enter_next_block(self, block):
         if not self.curr_block._exits:
+            assert block is not self.curr_block
             self.curr_block.set_exits([block])
         self.curr_block = block
         self.blocks.append(block)
@@ -238,8 +239,15 @@
         bc_block, i = self.pos_index[pos]
         return bc_block[i]
 
-    def next_pos(self, opcode):
-        return self._next_pos[opcode.offset]
+    def next_pos(self, instr):
+        block, i = self.pos_index[instr.offset]
+        i = i + 1
+        if i >= len(block.operations):
+            assert len(block._exits) == 1
+            assert block._exits[0] is not block
+            return block._exits[0].startpos
+        else:
+            return block.operations[i].offset
 
     def add_jump(self, block, target_block, target_offset):
         last_op = block.operations[-1]
@@ -323,8 +331,7 @@
     def bc_flow(self, reader):
         reader.curr_block.operations.append(self)
         if self.has_jump():
-            new_block = reader.get_next_block()
-            reader.enter_next_block(new_block)
+            reader.end_block()
             reader.get_block_at(self.arg)
 
     def has_jump(self):
@@ -386,7 +393,6 @@
     block = reader.curr_block
     block.operations[-1] = SWITCH_BOOL(on_False, on_True, offset=self.offset)
     block.set_exits([on_False, on_True])
-    reader.enter_next_block(on_True)
 
 @flow_opcode
 def POP_JUMP_IF_TRUE(self, reader):
@@ -396,7 +402,6 @@
     block = reader.curr_block
     block.operations[-1] = SWITCH_BOOL(on_False, on_True, offset=self.offset)
     block.set_exits([on_False, on_True])
-    reader.enter_next_block(on_False)
 
 @bc_reader.register_opcode
 class JUMP_IF_FALSE_OR_POP(BCInstruction):
@@ -406,7 +411,6 @@
         self.on_True = reader.get_next_block()
         self.on_False = reader.get_block_at(self.arg)
         block.set_exits([self.on_False, self.on_True])
-        reader.enter_next_block(self.on_True)
 
     def eval(self, ctx):
         w_value = ctx.peekvalue()
@@ -423,7 +427,6 @@
         self.on_True = reader.get_block_at(self.arg)
         self.on_False = reader.get_next_block()
         block.set_exits([self.on_False, self.on_True])
-        reader.enter_next_block(self.on_False)
 
     def eval(self, ctx):
         w_value = ctx.peekvalue()

From noreply at buildbot.pypy.org  Mon Nov 24 19:08:00 2014
From: noreply at buildbot.pypy.org (rlamy)
Date: Mon, 24 Nov 2014 19:08:00 +0100 (CET)
Subject: [pypy-commit] pypy framestate: kill ._next_pos
Message-ID: <20141124180800.758121C3271@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy 
Branch: framestate
Changeset: r74714:1a262bc9c1bf
Date: 2014-11-24 18:07 +0000
http://bitbucket.org/pypy/pypy/changeset/1a262bc9c1bf/

Log:	kill ._next_pos

diff --git a/rpython/flowspace/bytecode.py b/rpython/flowspace/bytecode.py
--- a/rpython/flowspace/bytecode.py
+++ b/rpython/flowspace/bytecode.py
@@ -136,9 +136,7 @@
     def _iter_instr(self, code):
         self.offset = 0
         i = 0
-        last_offset = -1
         while self.offset < len(code.co_code):
-            self.graph._next_pos.setdefault(last_offset, self.offset)
             if self.offset in self.pending_blocks:
                 next_block = self.pending_blocks[self.offset]
                 if not self.curr_block.operations:
@@ -151,7 +149,6 @@
             next_offset, instr = self.read(code, self.offset)
             self.next_offset = next_offset
             yield instr
-            last_offset = instr.offset
             self.offset = next_offset
             i += 1
 
@@ -211,7 +208,6 @@
             block = self.curr_block
             graph.pos_index[instr.offset] = block, len(block.operations)
             instr.bc_flow(self)
-        graph._next_pos[instr.offset] = len(code.co_code)
         self.check_graph()
         return graph
 
@@ -233,7 +229,6 @@
         self.entry = EntryBlock()
         self.entry.set_exits([startblock])
         self.pos_index = {}
-        self._next_pos = {}
 
     def read(self, pos):
         bc_block, i = self.pos_index[pos]
@@ -249,11 +244,6 @@
         else:
             return block.operations[i].offset
 
-    def add_jump(self, block, target_block, target_offset):
-        last_op = block.operations[-1]
-        self._next_pos[last_op.offset] = target_offset
-        block.set_exits([target_block])
-
 
 class BytecodeBlock(object):
     """Base class for opcode blocks"""
@@ -453,20 +443,18 @@
 
 @flow_opcode
 def JUMP_ABSOLUTE(self, reader):
-    reader.curr_block.operations.append(self)
+    block = reader.curr_block
+    block.operations.append(self)
     target_block = reader.get_block_at(self.arg)
-    block = reader.curr_block
-    graph = reader.graph
-    graph.add_jump(block, target_block, self.arg)
+    block.set_exits([target_block])
     reader.end_block()
 
 @flow_opcode
 def JUMP_FORWARD(self, reader):
-    reader.curr_block.operations.append(self)
+    block = reader.curr_block
+    block.operations.append(self)
     target_block = reader.get_block_at(self.arg)
-    block = reader.curr_block
-    graph = reader.graph
-    graph.add_jump(block, target_block, self.arg)
+    block.set_exits([target_block])
     reader.end_block()
 
 @bc_reader.register_opcode

From noreply at buildbot.pypy.org  Mon Nov 24 21:35:16 2014
From: noreply at buildbot.pypy.org (rlamy)
Date: Mon, 24 Nov 2014 21:35:16 +0100 (CET)
Subject: [pypy-commit] pypy framestate: Add dumb way of dumping BC graphs
Message-ID: <20141124203516.E1D171D2513@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy 
Branch: framestate
Changeset: r74715:41cb87605dc8
Date: 2014-11-24 20:35 +0000
http://bitbucket.org/pypy/pypy/changeset/41cb87605dc8/

Log:	Add dumb way of dumping BC graphs

diff --git a/rpython/flowspace/bytecode.py b/rpython/flowspace/bytecode.py
--- a/rpython/flowspace/bytecode.py
+++ b/rpython/flowspace/bytecode.py
@@ -244,6 +244,11 @@
         else:
             return block.operations[i].offset
 
+    def dump(self):
+        all_blocks = set(x[0] for x in self.pos_index.values())
+        blocks = sorted(all_blocks, key=lambda b: b.startpos)
+        return [b.operations for b in blocks]
+
 
 class BytecodeBlock(object):
     """Base class for opcode blocks"""
diff --git a/rpython/flowspace/test/test_bytecode.py b/rpython/flowspace/test/test_bytecode.py
new file mode 100644
--- /dev/null
+++ b/rpython/flowspace/test/test_bytecode.py
@@ -0,0 +1,13 @@
+"""Unit tests for rpython.flowspace.bytecode"""
+
+from rpython.flowspace.bytecode import bc_reader
+
+def test_graph_dump():
+    def f(x):
+        if x:
+            return 1
+        else:
+            return 0
+    bc_graph = bc_reader.build_flow(bc_reader.build_code(f.__code__))
+    print bc_graph.dump()
+    assert [lst[0].offset for lst in bc_graph.dump()] == [0, 6, 10]

From noreply at buildbot.pypy.org  Tue Nov 25 15:47:27 2014
From: noreply at buildbot.pypy.org (rlamy)
Date: Tue, 25 Nov 2014 15:47:27 +0100 (CET)
Subject: [pypy-commit] pypy framestate: Don't tie the FlowContext's notion
 of position with offsets
Message-ID: <20141125144727.684601C1366@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy 
Branch: framestate
Changeset: r74716:355495340c68
Date: 2014-11-24 20:49 +0000
http://bitbucket.org/pypy/pypy/changeset/355495340c68/

Log:	Don't tie the FlowContext's notion of position with offsets

diff --git a/rpython/flowspace/bytecode.py b/rpython/flowspace/bytecode.py
--- a/rpython/flowspace/bytecode.py
+++ b/rpython/flowspace/bytecode.py
@@ -244,6 +244,12 @@
         else:
             return block.operations[i].offset
 
+    def get_position(self, offset):
+        return offset
+
+    def get_offset(self, position):
+        return position
+
     def dump(self):
         all_blocks = set(x[0] for x in self.pos_index.values())
         blocks = sorted(all_blocks, key=lambda b: b.startpos)
diff --git a/rpython/flowspace/flowcontext.py b/rpython/flowspace/flowcontext.py
--- a/rpython/flowspace/flowcontext.py
+++ b/rpython/flowspace/flowcontext.py
@@ -288,9 +288,9 @@
     def dropvaluesuntil(self, finaldepth):
         del self.stack[finaldepth:]
 
-    def getstate(self, next_offset):
+    def getstate(self, position):
         return FrameState(self.locals_w[:], self.stack[:],
-                self.last_exception, self.blockstack[:], next_offset)
+                self.last_exception, self.blockstack[:], position)
 
     def setstate(self, state):
         """ Reset the context to the given frame state. """
@@ -347,15 +347,16 @@
 
     def record_block(self, block):
         self.setstate(block.framestate)
-        next_offset = block.framestate.next_offset
         self.recorder = block.make_recorder()
         bc_graph = self.pycode.graph
+        next_offset = bc_graph.get_offset(block.framestate.position)
         try:
             while True:
                 instr = bc_graph.read(next_offset)
                 self.last_offset = instr.offset
                 next_offset = self.handle_bytecode(instr)
-                self.recorder.final_state = self.getstate(next_offset)
+                position = bc_graph.get_position(next_offset)
+                self.recorder.final_state = self.getstate(position)
 
         except RaiseImplicit as e:
             w_exc = e.w_exc
@@ -393,10 +394,10 @@
         self.recorder = None
 
     def mergeblock(self, currentblock, currentstate):
-        next_offset = currentstate.next_offset
+        position = currentstate.position
         # can 'currentstate' be merged with one of the blocks that
         # already exist for this bytecode position?
-        candidates = self.joinpoints.setdefault(next_offset, [])
+        candidates = self.joinpoints.setdefault(position, [])
         for block in candidates:
             newstate = block.framestate.union(currentstate)
             if newstate is not None:
diff --git a/rpython/flowspace/framestate.py b/rpython/flowspace/framestate.py
--- a/rpython/flowspace/framestate.py
+++ b/rpython/flowspace/framestate.py
@@ -16,12 +16,12 @@
 
 
 class FrameState(object):
-    def __init__(self, locals_w, stack, last_exception, blocklist, next_offset):
+    def __init__(self, locals_w, stack, last_exception, blocklist, position):
         self.locals_w = locals_w
         self.stack = stack
         self.last_exception = last_exception
         self.blocklist = blocklist
-        self.next_offset = next_offset
+        self.position = position
         self._mergeable = None
 
     @property
@@ -44,7 +44,7 @@
         if exc is not None:
             exc = FSException(_copy(exc.w_type), _copy(exc.w_value))
         return FrameState(map(_copy, self.locals_w), map(_copy, self.stack),
-                exc, self.blocklist, self.next_offset)
+                exc, self.blocklist, self.position)
 
     def getvariables(self):
         return [w for w in self.mergeable if isinstance(w, Variable)]
@@ -55,7 +55,7 @@
         # safety check, don't try to compare states with different
         # nonmergeable states
         assert self.blocklist == other.blocklist
-        assert self.next_offset == other.next_offset
+        assert self.position == other.position
         for w1, w2 in zip(self.mergeable, other.mergeable):
             if not (w1 == w2 or (isinstance(w1, Variable) and
                                  isinstance(w2, Variable))):
@@ -86,7 +86,7 @@
                         union(args1[1], args2[1]))
         except UnionError:
             return None
-        return FrameState(locals, stack, exc, self.blocklist, self.next_offset)
+        return FrameState(locals, stack, exc, self.blocklist, self.position)
 
     def getoutputargs(self, targetstate):
         "Return the output arguments needed to link self to targetstate."

From noreply at buildbot.pypy.org  Tue Nov 25 15:47:28 2014
From: noreply at buildbot.pypy.org (rlamy)
Date: Tue, 25 Nov 2014 15:47:28 +0100 (CET)
Subject: [pypy-commit] pypy framestate: Move PyGraph creation inside
	FlowContext.__init__
Message-ID: <20141125144728.9E8F41C1366@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy 
Branch: framestate
Changeset: r74717:d8f08299732a
Date: 2014-11-25 06:18 +0000
http://bitbucket.org/pypy/pypy/changeset/d8f08299732a/

Log:	Move PyGraph creation inside FlowContext.__init__

diff --git a/rpython/flowspace/flowcontext.py b/rpython/flowspace/flowcontext.py
--- a/rpython/flowspace/flowcontext.py
+++ b/rpython/flowspace/flowcontext.py
@@ -7,7 +7,6 @@
 import __builtin__
 
 from rpython.tool.error import source_lines
-from rpython.translator.simplify import eliminate_empty_blocks
 from rpython.rlib import rstackovf
 from rpython.flowspace.argument import CallSpec
 from rpython.flowspace.model import (Constant, Variable, Block, Link,
@@ -15,9 +14,10 @@
 from rpython.flowspace.framestate import FrameState
 from rpython.flowspace.specialcase import (rpython_print_item,
     rpython_print_newline)
-from rpython.flowspace.bytecode import BytecodeBlock
 from rpython.flowspace.operation import op
-from rpython.flowspace.bytecode import BytecodeCorruption
+from rpython.flowspace.bytecode import (
+    BytecodeBlock, BytecodeCorruption, bc_reader)
+from rpython.flowspace.pygraph import PyGraph
 
 w_None = const(None)
 
@@ -227,9 +227,10 @@
 
 
 class FlowContext(object):
-    def __init__(self, graph, code):
+    def __init__(self, func):
+        code = bc_reader.build_code(func.func_code)
+        graph = PyGraph(func, code)
         self.graph = graph
-        func = graph.func
         self.pycode = code
         self.w_globals = Constant(func.func_globals)
         self.blockstack = []
diff --git a/rpython/flowspace/objspace.py b/rpython/flowspace/objspace.py
--- a/rpython/flowspace/objspace.py
+++ b/rpython/flowspace/objspace.py
@@ -3,12 +3,9 @@
 
 from inspect import CO_NEWLOCALS, isgeneratorfunction
 
-from rpython.flowspace.model import checkgraph
-from rpython.flowspace.bytecode import bc_reader
 from rpython.flowspace.flowcontext import (FlowContext, fixeggblocks)
 from rpython.flowspace.generator import (tweak_generator_graph,
         make_generator_entry_graph)
-from rpython.flowspace.pygraph import PyGraph
 
 
 def _assert_rpythonic(func):
@@ -36,11 +33,10 @@
     if (isgeneratorfunction(func) and
             not hasattr(func, '_generator_next_method_of_')):
         return make_generator_entry_graph(func)
-    code = bc_reader.build_code(func.func_code)
-    graph = PyGraph(func, code)
-    ctx = FlowContext(graph, code)
+    ctx = FlowContext(func)
     ctx.build_flow()
+    graph = ctx.graph
     fixeggblocks(graph)
-    if code.is_generator:
+    if ctx.pycode.is_generator:
         tweak_generator_graph(graph)
     return graph
diff --git a/rpython/flowspace/test/test_framestate.py b/rpython/flowspace/test/test_framestate.py
--- a/rpython/flowspace/test/test_framestate.py
+++ b/rpython/flowspace/test/test_framestate.py
@@ -10,9 +10,9 @@
             func = func.im_func
         except AttributeError:
             pass
-        code = bc_reader.build_code(func.func_code)
-        graph = PyGraph(func, code)
-        ctx = FlowContext(graph, code)
+        ctx = FlowContext(func)
+        ctx.build_flow()
+        graph = ctx.graph
         # hack the frame
         ctx.setstate(graph.startblock.framestate)
         ctx.locals_w[-1] = Constant(None)

From noreply at buildbot.pypy.org  Tue Nov 25 23:28:04 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Tue, 25 Nov 2014 23:28:04 +0100 (CET)
Subject: [pypy-commit] pypy portable-threadlocal: Implement this
 thread-local variant in the JIT.
Message-ID: <20141125222804.9AE3F1C1366@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: portable-threadlocal
Changeset: r74718:d739638f9b7f
Date: 2014-11-25 22:48 +0100
http://bitbucket.org/pypy/pypy/changeset/d739638f9b7f/

Log:	Implement this thread-local variant in the JIT.

diff --git a/rpython/jit/backend/arm/opassembler.py b/rpython/jit/backend/arm/opassembler.py
--- a/rpython/jit/backend/arm/opassembler.py
+++ b/rpython/jit/backend/arm/opassembler.py
@@ -1274,8 +1274,10 @@
         regalloc.rm.possibly_free_var(dstaddr_box)
         return fcond
 
-    def emit_opx_threadlocalref_addr(self, op, arglocs, regalloc, fcond):
-        res, = arglocs
+    def emit_opx_threadlocalref_get(self, op, arglocs, regalloc, fcond):
+        ofs0, res = arglocs
+        assert ofs0.is_imm()
         ofs = self.saved_threadlocal_addr
         self.load_reg(self.mc, res, r.sp, ofs)
+        self.load_reg(self.mc, res, res, ofs0.value)
         return fcond
diff --git a/rpython/jit/backend/arm/regalloc.py b/rpython/jit/backend/arm/regalloc.py
--- a/rpython/jit/backend/arm/regalloc.py
+++ b/rpython/jit/backend/arm/regalloc.py
@@ -559,8 +559,8 @@
                 args = self._prepare_op_math_sqrt(op, fcond)
                 self.perform_extra(op, args, fcond)
                 return
-            elif oopspecindex == EffectInfo.OS_THREADLOCALREF_ADDR:
-                args = self._prepare_threadlocalref_addr(op, fcond)
+            elif oopspecindex == EffectInfo.OS_THREADLOCALREF_GET:
+                args = self._prepare_threadlocalref_get(op, fcond)
                 self.perform_extra(op, args, fcond)
                 return
             #elif oopspecindex == EffectInfo.OS_MATH_READ_TIMESTAMP:
@@ -619,9 +619,10 @@
         res = self.force_allocate_reg(op.result)
         return [loc0, res]
 
-    def _prepare_threadlocalref_addr(self, op, fcond):
+    def _prepare_threadlocalref_get(self, op, fcond):
+        ofs0 = imm(op.getarg(1).getint())
         res = self.force_allocate_reg(op.result)
-        return [res]
+        return [ofs0, res]
 
     def _prepare_guard(self, op, args=None):
         if args is None:
diff --git a/rpython/jit/backend/llsupport/llmodel.py b/rpython/jit/backend/llsupport/llmodel.py
--- a/rpython/jit/backend/llsupport/llmodel.py
+++ b/rpython/jit/backend/llsupport/llmodel.py
@@ -220,8 +220,8 @@
         # The JIT backend must generate functions with the following
         # signature: it takes the jitframe and the threadlocal_addr
         # as arguments, and it returns the (possibly reallocated) jitframe.
-        # The backend can optimize OS_THREADLOCALREF_ADDR calls to return
-        # this threadlocal_addr, but only if 'translate_support_code':
+        # The backend can optimize OS_THREADLOCALREF_GET calls to return a
+        # field of this threadlocal_addr, but only if 'translate_support_code':
         # in untranslated tests, threadlocal_addr is a dummy NULL.
         FUNCPTR = lltype.Ptr(lltype.FuncType([llmemory.GCREF, llmemory.Address],
                                              llmemory.GCREF))
diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py
--- a/rpython/jit/backend/x86/assembler.py
+++ b/rpython/jit/backend/x86/assembler.py
@@ -2322,38 +2322,16 @@
         assert isinstance(reg, RegLoc)
         self.mc.MOV_rr(reg.value, ebp.value)
 
-    def threadlocalref_addr(self, resloc):
-        # This simply loads the stack location THREADLOCAL_OFS into a
-        # register.  It is only supported if 'translate_support_code' is
+    def threadlocalref_get(self, offset, resloc):
+        # This loads the stack location THREADLOCAL_OFS into a
+        # register, and then read the word at the given offset.
+        # It is only supported if 'translate_support_code' is
         # true; otherwise, the original call to the piece of assembler
         # was done with a dummy NULL value.
         assert self.cpu.translate_support_code
         assert isinstance(resloc, RegLoc)
         self.mc.MOV_rs(resloc.value, THREADLOCAL_OFS)
-
-    def get_set_errno(self, op, loc, issue_a_write):
-        # this function is only called on Linux
-        from rpython.jit.backend.x86 import stmtlocal
-        addr = stmtlocal.get_errno_tl()
-        assert rx86.fits_in_32bits(addr)
-        mc = self.mc
-        mc.writechar(stmtlocal.SEGMENT_TL)     # prefix: %fs or %gs
-        # !!important: the *next* instruction must be the one using 'addr'!!
-        if issue_a_write:
-            if isinstance(loc, RegLoc):
-                mc.MOV32_jr(addr, loc.value)       # memory write from reg
-            else:
-                assert isinstance(loc, ImmedLoc)
-                newvalue = loc.value
-                newvalue = rffi.cast(rffi.INT, newvalue)
-                newvalue = rffi.cast(lltype.Signed, newvalue)
-                mc.MOV32_ji(addr, newvalue)        # memory write immediate
-        else:
-            assert isinstance(loc, RegLoc)
-            if IS_X86_32:
-                mc.MOV_rj(loc.value, addr)         # memory read
-            elif IS_X86_64:
-                mc.MOVSX32_rj(loc.value, addr)     # memory read, sign-extend
+        self.mc.MOV_rm(resloc.value, (resloc.value, offset))
 
     def genop_discard_zero_array(self, op, arglocs):
         (base_loc, startindex_loc, bytes_loc,
diff --git a/rpython/jit/backend/x86/regalloc.py b/rpython/jit/backend/x86/regalloc.py
--- a/rpython/jit/backend/x86/regalloc.py
+++ b/rpython/jit/backend/x86/regalloc.py
@@ -693,10 +693,11 @@
         loc0 = self.xrm.force_result_in_reg(op.result, op.getarg(1))
         self.perform_math(op, [loc0], loc0)
 
-    def _consider_threadlocalref_addr(self, op):
+    def _consider_threadlocalref_get(self, op):
         if self.translate_support_code:
+            offset = op.getarg(1).getint()   # getarg(0) == 'threadlocalref_get'
             resloc = self.force_allocate_reg(op.result)
-            self.assembler.threadlocalref_addr(resloc)
+            self.assembler.threadlocalref_get(offset, resloc)
         else:
             self._consider_call(op)
 
@@ -777,8 +778,8 @@
                         return
             if oopspecindex == EffectInfo.OS_MATH_SQRT:
                 return self._consider_math_sqrt(op)
-            if oopspecindex == EffectInfo.OS_THREADLOCALREF_ADDR:
-                return self._consider_threadlocalref_addr(op)
+            if oopspecindex == EffectInfo.OS_THREADLOCALREF_GET:
+                return self._consider_threadlocalref_get(op)
             if oopspecindex == EffectInfo.OS_MATH_READ_TIMESTAMP:
                 return self._consider_math_read_timestamp(op)
         self._consider_call(op)
diff --git a/rpython/jit/codewriter/effectinfo.py b/rpython/jit/codewriter/effectinfo.py
--- a/rpython/jit/codewriter/effectinfo.py
+++ b/rpython/jit/codewriter/effectinfo.py
@@ -22,7 +22,7 @@
     OS_STR2UNICODE              = 2    # "str.str2unicode"
     OS_SHRINK_ARRAY             = 3    # rgc.ll_shrink_array
     OS_DICT_LOOKUP              = 4    # ll_dict_lookup
-    OS_THREADLOCALREF_ADDR      = 5    # llop.threadlocalref_addr
+    OS_THREADLOCALREF_GET       = 5    # llop.threadlocalref_get
     OS_NOT_IN_TRACE             = 8    # for calls not recorded in the jit trace
     #
     OS_STR_CONCAT               = 22   # "stroruni.concat"
diff --git a/rpython/jit/codewriter/jtransform.py b/rpython/jit/codewriter/jtransform.py
--- a/rpython/jit/codewriter/jtransform.py
+++ b/rpython/jit/codewriter/jtransform.py
@@ -1999,11 +1999,16 @@
                              None)
         return [op0, op1]
 
-    def rewrite_op_threadlocalref_addr(self, op):
-        op1 = self.prepare_builtin_call(op, 'threadlocalref_addr', [])
+    def rewrite_op_threadlocalref_get(self, op):
+        # only supports RESTYPE being exactly one word.
+        RESTYPE = op.result.concretetype
+        assert (RESTYPE in (lltype.Signed, lltype.Unsigned, llmemory.Address)
+                or isinstance(RESTYPE, lltype.Ptr))
+        c_offset, = op.args
+        op1 = self.prepare_builtin_call(op, 'threadlocalref_get', [c_offset])
         return self.handle_residual_call(op1,
-            oopspecindex=EffectInfo.OS_THREADLOCALREF_ADDR,
-            extraeffect=EffectInfo.EF_CANNOT_RAISE)
+            oopspecindex=EffectInfo.OS_THREADLOCALREF_GET,
+            extraeffect=EffectInfo.EF_LOOPINVARIANT)
 
 # ____________________________________________________________
 
diff --git a/rpython/jit/codewriter/support.py b/rpython/jit/codewriter/support.py
--- a/rpython/jit/codewriter/support.py
+++ b/rpython/jit/codewriter/support.py
@@ -702,8 +702,9 @@
     build_ll_1_raw_free_no_track_allocation = (
         build_raw_free_builder(track_allocation=False))
 
-    def _ll_0_threadlocalref_addr():
-        return llop.threadlocalref_addr(llmemory.Address)
+    def _ll_1_threadlocalref_get(TP, offset):
+        return llop.threadlocalref_get(TP, offset)
+    _ll_1_threadlocalref_get.need_result_type = 'exact'   # don't deref
 
     def _ll_1_weakref_create(obj):
         return llop.weakref_create(llmemory.WeakRefPtr, obj)
@@ -816,8 +817,18 @@
     s_result = lltype_to_annotation(ll_res)
     impl = setup_extra_builtin(rtyper, oopspec_name, len(args_s), extra)
     if getattr(impl, 'need_result_type', False):
-        bk = rtyper.annotator.bookkeeper
-        args_s.insert(0, annmodel.SomePBC([bk.getdesc(deref(ll_res))]))
+        if hasattr(rtyper, 'annotator'):
+            bk = rtyper.annotator.bookkeeper
+            ll_restype = ll_res
+            if impl.need_result_type != 'exact':
+                ll_restype = deref(ll_restype)
+            desc = bk.getdesc(ll_restype)
+        else:
+            class TestingDesc(object):
+                knowntype = int
+                pyobj = None
+            desc = TestingDesc()
+        args_s.insert(0, annmodel.SomePBC([desc]))
     #
     if hasattr(rtyper, 'annotator'):  # regular case
         mixlevelann = MixLevelHelperAnnotator(rtyper)
diff --git a/rpython/jit/codewriter/test/test_jtransform.py b/rpython/jit/codewriter/test/test_jtransform.py
--- a/rpython/jit/codewriter/test/test_jtransform.py
+++ b/rpython/jit/codewriter/test/test_jtransform.py
@@ -148,7 +148,7 @@
              EI.OS_UNIEQ_LENGTHOK:       ([PUNICODE, PUNICODE], INT),
              EI.OS_RAW_MALLOC_VARSIZE_CHAR: ([INT], ARRAYPTR),
              EI.OS_RAW_FREE:             ([ARRAYPTR], lltype.Void),
-             EI.OS_THREADLOCALREF_ADDR:  ([], llmemory.Address),
+             EI.OS_THREADLOCALREF_GET:   ([INT], INT),   # for example
             }
             argtypes = argtypes[oopspecindex]
             assert argtypes[0] == [v.concretetype for v in op.args[1:]]
@@ -157,9 +157,10 @@
                 assert extraeffect == EI.EF_ELIDABLE_CAN_RAISE
             elif oopspecindex == EI.OS_RAW_MALLOC_VARSIZE_CHAR:
                 assert extraeffect == EI.EF_CAN_RAISE
-            elif oopspecindex in (EI.OS_RAW_FREE,
-                                  EI.OS_THREADLOCALREF_ADDR):
+            elif oopspecindex == EI.OS_RAW_FREE:
                 assert extraeffect == EI.EF_CANNOT_RAISE
+            elif oopspecindex == EI.OS_THREADLOCALREF_GET:
+                assert extraeffect == EI.EF_LOOPINVARIANT
             else:
                 assert extraeffect == EI.EF_ELIDABLE_CANNOT_RAISE
         return 'calldescr-%d' % oopspecindex
@@ -1341,16 +1342,20 @@
     assert op1.result is None
     assert op2 is None
 
-def test_threadlocalref_addr():
-    OS_THREADLOCALREF_ADDR = effectinfo.EffectInfo.OS_THREADLOCALREF_ADDR
-    v = varoftype(llmemory.Address)
-    op = SpaceOperation('threadlocalref_addr', [], v)
+def test_threadlocalref_get():
+    from rpython.rlib.rthread import ThreadLocalField
+    tlfield = ThreadLocalField(lltype.Signed, 'foobar_test_')
+    OS_THREADLOCALREF_GET = effectinfo.EffectInfo.OS_THREADLOCALREF_GET
+    c = const(tlfield.offset)
+    v = varoftype(lltype.Signed)
+    op = SpaceOperation('threadlocalref_get', [c], v)
     tr = Transformer(FakeCPU(), FakeBuiltinCallControl())
     op0 = tr.rewrite_operation(op)
-    assert op0.opname == 'residual_call_r_i'
-    assert op0.args[0].value == 'threadlocalref_addr' # pseudo-function as str
-    assert op0.args[1] == ListOfKind("ref", [])
-    assert op0.args[2] == 'calldescr-%d' % OS_THREADLOCALREF_ADDR
+    assert op0.opname == 'residual_call_ir_i'
+    assert op0.args[0].value == 'threadlocalref_get' # pseudo-function as str
+    assert op0.args[1] == ListOfKind("int", [c])
+    assert op0.args[2] == ListOfKind("ref", [])
+    assert op0.args[3] == 'calldescr-%d' % OS_THREADLOCALREF_GET
     assert op0.result == v
 
 def test_unknown_operation():
diff --git a/rpython/jit/metainterp/test/test_threadlocal.py b/rpython/jit/metainterp/test/test_threadlocal.py
--- a/rpython/jit/metainterp/test/test_threadlocal.py
+++ b/rpython/jit/metainterp/test/test_threadlocal.py
@@ -1,19 +1,20 @@
 import py
+from rpython.rlib import rthread
 from rpython.jit.metainterp.test.support import LLJitMixin
-from rpython.rtyper.lltypesystem import llmemory
+from rpython.rtyper.lltypesystem import lltype
 from rpython.rtyper.lltypesystem.lloperation import llop
 
 
 class ThreadLocalTest(object):
 
     def test_threadlocalref_get(self):
+        tlfield = rthread.ThreadLocalField(lltype.Signed, 'foobar_test_')
+
         def f():
-            addr1 = llop.threadlocalref_addr(llmemory.Address)
-            # a "does not crash" test only
-            return 1
+            return tlfield.getraw()
 
         res = self.interp_operations(f, [])
-        assert res == 1
+        assert res == 0x544c    # magic value returned by llinterp
 
 
 class TestLLtype(ThreadLocalTest, LLJitMixin):
diff --git a/rpython/rtyper/llinterp.py b/rpython/rtyper/llinterp.py
--- a/rpython/rtyper/llinterp.py
+++ b/rpython/rtyper/llinterp.py
@@ -921,14 +921,12 @@
 
     def op_threadlocalref_addr(self):
         raise NotImplementedError("threadlocalref_addr")
-        ## class FakeThreadLocalAddr(object):
-        ##     is_fake_thread_local_addr = True
-        ##     _TYPE = llmemory.Address
-        ##     def _cast_to_int(self, symbolic=None):
-        ##         return FakeThreadLocalAddrAsInt()
-        ## class FakeThreadLocalAddrAsInt(object):
-        ##     _TYPE = lltype.Signed
-        ## return FakeThreadLocalAddr()
+
+    def op_threadlocalref_get(self, offset):
+        if (type(offset) is CDefinedIntSymbolic and
+                offset.expr == 'RPY_TLOFS_foobar_test_'):   # used in tests
+            return 0x544c
+        raise NotImplementedError("threadlocalref_get")
 
     # __________________________________________________________
     # operations on addresses
diff --git a/rpython/translator/c/funcgen.py b/rpython/translator/c/funcgen.py
--- a/rpython/translator/c/funcgen.py
+++ b/rpython/translator/c/funcgen.py
@@ -902,15 +902,20 @@
             return None    # use the default
 
     def OP_THREADLOCALREF_GET(self, op):
-        assert isinstance(op.args[0], Constant)
-        assert isinstance(op.args[0].value, CDefinedIntSymbolic)
-        fieldname = op.args[0].value.expr
-        assert fieldname.startswith('RPY_TLOFS_')
-        fieldname = fieldname[10:]
         typename = self.db.gettype(op.result.concretetype)
-        return '%s = (%s)RPY_THREADLOCALREF_GET(%s);' % (
-            self.expr(op.result),
-            cdecl(typename, ''),
-            fieldname)
+        if isinstance(op.args[0], Constant):
+            assert isinstance(op.args[0].value, CDefinedIntSymbolic)
+            fieldname = op.args[0].value.expr
+            assert fieldname.startswith('RPY_TLOFS_')
+            fieldname = fieldname[10:]
+            return '%s = (%s)RPY_THREADLOCALREF_GET(%s);' % (
+                self.expr(op.result),
+                cdecl(typename, ''),
+                fieldname)
+        else:
+            return 'OP_THREADLOCALREF_GET_NONCONST(%s, %s, %s);' % (
+                cdecl(typename, ''),
+                self.expr(op.args[0]),
+                self.expr(op.result))
 
 assert not USESLOTS or '__dict__' not in dir(FunctionCodeGenerator)
diff --git a/rpython/translator/c/src/threadlocal.h b/rpython/translator/c/src/threadlocal.h
--- a/rpython/translator/c/src/threadlocal.h
+++ b/rpython/translator/c/src/threadlocal.h
@@ -86,4 +86,13 @@
 /* ------------------------------------------------------------ */
 
 
+/* only for the fall-back path in the JIT */
+#define OP_THREADLOCALREF_GET_NONCONST(RESTYPE, offset, r)      \
+    do {                                                        \
+        char *a;                                                \
+        OP_THREADLOCALREF_ADDR(a);                              \
+        r = *(RESTYPE *)(a + offset);                           \
+    } while (0)
+
+
 #endif /* _SRC_THREADLOCAL_H */

From noreply at buildbot.pypy.org  Wed Nov 26 00:00:58 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Wed, 26 Nov 2014 00:00:58 +0100 (CET)
Subject: [pypy-commit] pypy default: Tweak the repr of loops shown e.g. in
	test_pypy_c/.
Message-ID: <20141125230058.435311C1366@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r74719:afc446fb831e
Date: 2014-11-25 23:49 +0100
http://bitbucket.org/pypy/pypy/changeset/afc446fb831e/

Log:	Tweak the repr of loops shown e.g. in test_pypy_c/.

diff --git a/rpython/tool/jitlogparser/parser.py b/rpython/tool/jitlogparser/parser.py
--- a/rpython/tool/jitlogparser/parser.py
+++ b/rpython/tool/jitlogparser/parser.py
@@ -216,7 +216,7 @@
     line_starts_here = property(getline_starts_here)
 
     def __repr__(self):
-        return "[%s]" % ", ".join([repr(op) for op in self.operations])
+        return "[%s\n]" % "\n    ".join([repr(op) for op in self.operations])
 
     def pretty_print(self, out):
         pass

From noreply at buildbot.pypy.org  Wed Nov 26 00:00:59 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Wed, 26 Nov 2014 00:00:59 +0100 (CET)
Subject: [pypy-commit] pypy portable-threadlocal: Don't use GCREF here,
 but only OBJECTPTR.  Avoids 'mark_opaque_ptr' in
Message-ID: <20141125230059.8DF9F1C1366@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: portable-threadlocal
Changeset: r74720:030b8db86648
Date: 2014-11-26 00:00 +0100
http://bitbucket.org/pypy/pypy/changeset/030b8db86648/

Log:	Don't use GCREF here, but only OBJECTPTR. Avoids 'mark_opaque_ptr'
	in the JIT which disables minor optimizations.

diff --git a/rpython/rlib/rthread.py b/rpython/rlib/rthread.py
--- a/rpython/rlib/rthread.py
+++ b/rpython/rlib/rthread.py
@@ -320,10 +320,11 @@
 
         def get():
             if we_are_translated():
-                from rpython.rtyper.annlowlevel import cast_gcref_to_instance
+                from rpython.rtyper import rclass
+                from rpython.rtyper.annlowlevel import cast_base_ptr_to_instance
                 _threadlocalref_seeme(self)
-                gcref = llop.threadlocalref_get(llmemory.GCREF, offset)
-                return cast_gcref_to_instance(Cls, gcref)
+                ptr = llop.threadlocalref_get(rclass.OBJECTPTR, offset)
+                return cast_base_ptr_to_instance(Cls, ptr)
             else:
                 return getattr(self.local, 'value', None)
 

From noreply at buildbot.pypy.org  Wed Nov 26 00:02:08 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Wed, 26 Nov 2014 00:02:08 +0100 (CET)
Subject: [pypy-commit] pypy portable-threadlocal: hg merge default
Message-ID: <20141125230208.B314D1C1366@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: portable-threadlocal
Changeset: r74721:ac12cf96b219
Date: 2014-11-26 00:01 +0100
http://bitbucket.org/pypy/pypy/changeset/ac12cf96b219/

Log:	hg merge default

diff --git a/pypy/interpreter/pyframe.py b/pypy/interpreter/pyframe.py
--- a/pypy/interpreter/pyframe.py
+++ b/pypy/interpreter/pyframe.py
@@ -131,7 +131,6 @@
         # class bodies only have CO_NEWLOCALS.
         # CO_NEWLOCALS: make a locals dict unless optimized is also set
         # CO_OPTIMIZED: no locals dict needed at all
-        # NB: this method is overridden in nestedscope.py
         flags = code.co_flags
         if not (flags & pycode.CO_OPTIMIZED):
             if flags & pycode.CO_NEWLOCALS:
diff --git a/pypy/module/pypyjit/test_pypy_c/model.py b/pypy/module/pypyjit/test_pypy_c/model.py
--- a/pypy/module/pypyjit/test_pypy_c/model.py
+++ b/pypy/module/pypyjit/test_pypy_c/model.py
@@ -275,9 +275,19 @@
 
     @classmethod
     def parse_op(cls, line):
-        # strip comment
+        # strip comment after '#', but not if it appears inside parentheses
         if '#' in line:
-            line = line[:line.index('#')]
+            nested = 0
+            for i, c in enumerate(line):
+                if c == '(':
+                    nested += 1
+                elif c == ')':
+                    assert nested > 0, "more ')' than '(' in %r" % (line,)
+                    nested -= 1
+                elif c == '#' and nested == 0:
+                    line = line[:i]
+                    break
+        #
         if line.strip() == 'guard_not_invalidated?':
             return 'guard_not_invalidated', None, [], '...', False
         # find the resvar, if any
@@ -314,7 +324,7 @@
         # to repeat it every time
         ticker_check = """
             guard_not_invalidated?
-            ticker0 = getfield_raw(ticker_address, descr=)
+            ticker0 = getfield_raw(#, descr=)
             ticker_cond0 = int_lt(ticker0, 0)
             guard_false(ticker_cond0, descr=...)
         """
@@ -323,9 +333,9 @@
         # this is the ticker check generated if we have threads
         thread_ticker_check = """
             guard_not_invalidated?
-            ticker0 = getfield_raw(ticker_address, descr=)
-            ticker1 = int_sub(ticker0, _)
-            setfield_raw(ticker_address, ticker1, descr=)
+            ticker0 = getfield_raw(#, descr=)
+            ticker1 = int_sub(ticker0, #)
+            setfield_raw(#, ticker1, descr=)
             ticker_cond0 = int_lt(ticker1, 0)
             guard_false(ticker_cond0, descr=...)
         """
@@ -333,7 +343,7 @@
         #
         # this is the ticker check generated in PyFrame.handle_operation_error
         exc_ticker_check = """
-            ticker2 = getfield_raw(ticker_address, descr=)
+            ticker2 = getfield_raw(#, descr=)
             ticker_cond1 = int_lt(ticker2, 0)
             guard_false(ticker_cond1, descr=...)
         """
@@ -351,18 +361,31 @@
 
     @staticmethod
     def as_numeric_const(v1):
+        # returns one of:  ('int', value)  ('float', value)  None
         try:
-            return int(v1)
-        except (ValueError, TypeError):
-            return None
+            return ('int', int(v1))
+        except ValueError:
+            pass
+        if '.' in v1:
+            try:
+                return ('float', float(v1))
+            except ValueError:
+                pass
+        return None
 
     def match_var(self, v1, exp_v2):
         assert v1 != '_'
-        if exp_v2 == '_':
+        if exp_v2 == '_':           # accept anything
             return True
+        if exp_v2 is None:
+            return v1 is None
+        assert exp_v2 != '...'      # bogus use of '...' in the expected code
         n1 = self.as_numeric_const(v1)
+        if exp_v2 == '#':           # accept any (integer or float) number
+            return n1 is not None
         n2 = self.as_numeric_const(exp_v2)
-        if n1 is not None and n2 is not None:
+        if n1 is not None or n2 is not None:
+            # at least one is a number; check that both are, and are equal
             return n1 == n2
         if self.is_const(v1) or self.is_const(exp_v2):
             return v1[:-1].startswith(exp_v2[:-1])
@@ -382,10 +405,13 @@
     def match_op(self, op, (exp_opname, exp_res, exp_args, exp_descr, _)):
         self._assert(op.name == exp_opname, "operation mismatch")
         self.match_var(op.res, exp_res)
-        if exp_args != ['...']:
+        if exp_args[-1:] == ['...']:      # exp_args ends with '...'
+            exp_args = exp_args[:-1]
+            self._assert(len(op.args) >= len(exp_args), "not enough arguments")
+        else:
             self._assert(len(op.args) == len(exp_args), "wrong number of arguments")
-            for arg, exp_arg in zip(op.args, exp_args):
-                self._assert(self.match_var(arg, exp_arg), "variable mismatch: %r instead of %r" % (arg, exp_arg))
+        for arg, exp_arg in zip(op.args, exp_args):
+            self._assert(self.match_var(arg, exp_arg), "variable mismatch: %r instead of %r" % (arg, exp_arg))
         self.match_descr(op.descr, exp_descr)
 
 
diff --git a/pypy/module/pypyjit/test_pypy_c/test_00_model.py b/pypy/module/pypyjit/test_pypy_c/test_00_model.py
--- a/pypy/module/pypyjit/test_pypy_c/test_00_model.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_00_model.py
@@ -158,6 +158,24 @@
         assert match_var('v0', 'V0')
         assert match_var('ConstPtr(ptr0)', '_')
         py.test.raises(AssertionError, "match_var('_', 'v0')")
+        #
+        # numerics
+        assert match_var('1234', '1234')
+        assert not match_var('1234', '1235')
+        assert not match_var('v0', '1234')
+        assert not match_var('1234', 'v0')
+        assert match_var('1234', '#')        # the '#' char matches any number
+        assert not match_var('v0', '#')
+        assert match_var('1234', '_')        # the '_' char matches anything
+        #
+        # float numerics
+        assert match_var('0.000000', '0.0')
+        assert not match_var('0.000000', '0')
+        assert not match_var('0', '0.0')
+        assert not match_var('v0', '0.0')
+        assert not match_var('0.0', 'v0')
+        assert match_var('0.0', '#')
+        assert match_var('0.0', '_')
 
     def test_parse_op(self):
         res = OpMatcher.parse_op("  a =   int_add(  b,  3 ) # foo")
@@ -210,6 +228,19 @@
         """
         assert not self.match(loop, expected)
 
+    def test_dotdotdot_in_operation(self):
+        loop = """
+            [i0, i1]
+            jit_debug(i0, 1, ConstClass(myclass), i1)
+        """
+        assert self.match(loop, "jit_debug(...)")
+        assert self.match(loop, "jit_debug(i0, ...)")
+        assert self.match(loop, "jit_debug(i0, 1, ...)")
+        assert self.match(loop, "jit_debug(i0, 1, _, ...)")
+        assert self.match(loop, "jit_debug(i0, 1, _, i1, ...)")
+        py.test.raises(AssertionError, self.match,
+                       loop, "jit_debug(i0, 1, ..., i1)")
+
     def test_match_descr(self):
         loop = """
             [p0]
@@ -232,7 +263,7 @@
             jump(i4)
         """
         expected = """
-            i1 = int_add(0, 1)
+            i1 = int_add(i0, 1)
             ...
             i4 = int_mul(i1, 1000)
             jump(i4, descr=...)
@@ -249,7 +280,7 @@
             jump(i4, descr=...)
         """
         expected = """
-            i1 = int_add(0, 1)
+            i1 = int_add(i0, 1)
             ...
             _ = int_mul(_, 1000)
             jump(i4, descr=...)
@@ -268,7 +299,7 @@
             jump(i4)
         """
         expected = """
-            i1 = int_add(0, 1)
+            i1 = int_add(i0, 1)
             ...
         """
         assert self.match(loop, expected)
diff --git a/pypy/module/pypyjit/test_pypy_c/test_buffers.py b/pypy/module/pypyjit/test_pypy_c/test_buffers.py
--- a/pypy/module/pypyjit/test_pypy_c/test_buffers.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_buffers.py
@@ -21,7 +21,7 @@
             i65 = getfield_gc(p18, descr=...)
             i67 = int_gt(0, i65)
             guard_false(i67, descr=...)
-            i69 = int_gt(., i65)
+            i69 = int_gt(#, i65)
             guard_true(i69, descr=...)
             --TICK--
         """)
@@ -56,7 +56,7 @@
             guard_false(i99, descr=...)
             i100 = int_lshift(i98, 24)
             i101 = int_or(i97, i100)
-            i102 = getfield_raw(\d+, descr=)
+            i102 = getfield_raw(#, descr=)
             i103 = int_lt(i102, 0)
             guard_false(i103, descr=...)
         """)
diff --git a/pypy/module/pypyjit/test_pypy_c/test_call.py b/pypy/module/pypyjit/test_pypy_c/test_call.py
--- a/pypy/module/pypyjit/test_pypy_c/test_call.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_call.py
@@ -395,7 +395,7 @@
             setarrayitem_gc(p24, 0, p26, descr=)
             setfield_gc(p22, p24, descr=)
             }}}
-            p32 = call_may_force(..., p18, p22, descr=)
+            p32 = call_may_force(_, p18, p22, descr=)
             ...
         """)
 
diff --git a/pypy/module/pypyjit/test_pypy_c/test_containers.py b/pypy/module/pypyjit/test_pypy_c/test_containers.py
--- a/pypy/module/pypyjit/test_pypy_c/test_containers.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_containers.py
@@ -82,7 +82,7 @@
             guard_no_exception(descr=...)
             i23 = call(ConstClass(ll_dict_lookup_trampoline), p13, p10, i12, descr=)
             guard_no_exception(descr=...)
-            i26 = int_and(i23, .*)
+            i26 = int_and(i23, #)
             i27 = int_is_true(i26)
             guard_false(i27, descr=...)
             p28 = getfield_gc(p13, descr=)
diff --git a/pypy/module/pypyjit/test_pypy_c/test_math.py b/pypy/module/pypyjit/test_pypy_c/test_math.py
--- a/pypy/module/pypyjit/test_pypy_c/test_math.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_math.py
@@ -21,7 +21,7 @@
             guard_true(i2, descr=...)
             guard_not_invalidated(descr=...)
             f1 = cast_int_to_float(i0)
-            i3 = float_le(f1, 0)
+            i3 = float_le(f1, 0.0)
             guard_false(i3, descr=...)
             f2 = call(ConstClass(log), f1, descr=)
             f3 = call(ConstClass(log10), f1, descr=)
@@ -56,7 +56,7 @@
             f3 = call(ConstClass(cos), f1, descr=)
             f4 = float_sub(f2, f3)
             f5 = float_add(f0, f4)
-            i7 = int_add(i0, f1)
+            i7 = int_add(i0, 1)
             --TICK--
             jump(..., descr=)
         """)
diff --git a/pypy/module/pypyjit/test_pypy_c/test_micronumpy.py b/pypy/module/pypyjit/test_pypy_c/test_micronumpy.py
--- a/pypy/module/pypyjit/test_pypy_c/test_micronumpy.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_micronumpy.py
@@ -104,7 +104,7 @@
             setarrayitem_gc(p150, 1, 0, descr=)
             setarrayitem_gc(p150, 0, 0, descr=)
             guard_not_invalidated(descr=...)
-            i154 = getfield_raw(ticker_address, descr=)
+            i154 = getfield_raw(#, descr=)
             i155 = int_lt(i154, 0)
             guard_false(i155, descr=...)
             p156 = new_with_vtable(...)
@@ -142,7 +142,7 @@
             raw_store(i103, i132, 42.000000, descr=)
             p152 = getfield_gc_pure(p126, descr=)
             i153 = int_add(i120, 1)
-            i154 = getfield_raw(ticker_address, descr=)
+            i154 = getfield_raw(#, descr=)
             setarrayitem_gc(p152, 1, 0, descr=)
             setarrayitem_gc(p152, 0, 0, descr=)
             i157 = int_lt(i154, 0)
diff --git a/pypy/module/pypyjit/test_pypy_c/test_string.py b/pypy/module/pypyjit/test_pypy_c/test_string.py
--- a/pypy/module/pypyjit/test_pypy_c/test_string.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_string.py
@@ -67,15 +67,15 @@
             i11 = int_lt(i6, i7)
             guard_true(i11, descr=...)
             guard_not_invalidated(descr=...)
-            i13 = int_eq(i6, %d)
+            i13 = int_eq(i6, %d)         # value provided below
             guard_false(i13, descr=...)
-            i15 = int_mod(i6, i8)
-            i17 = int_rshift(i15, %d)
-            i18 = int_and(i8, i17)
+            i15 = int_mod(i6, 10)
+            i17 = int_rshift(i15, %d)    # value provided below
+            i18 = int_and(10, i17)
             i19 = int_add(i15, i18)
             i21 = int_lt(i19, 0)
             guard_false(i21, descr=...)
-            i22 = int_ge(i19, i8)
+            i22 = int_ge(i19, 10)
             guard_false(i22, descr=...)
             i23 = strgetitem(p10, i19)
             p25 = newstr(1)
@@ -83,7 +83,7 @@
             p93 = call(ConstClass(fromstr), p25, 16, descr=)
             guard_no_exception(descr=...)
             i95 = getfield_gc_pure(p93, descr=)
-            i96 = int_gt(i95, .*)
+            i96 = int_gt(i95, #)
             guard_false(i96, descr=...)
             i94 = call(ConstClass(rbigint._toint_helper), p93, descr=)
             guard_no_exception(descr=...)
diff --git a/pypy/module/pypyjit/test_pypy_c/test_thread.py b/pypy/module/pypyjit/test_pypy_c/test_thread.py
--- a/pypy/module/pypyjit/test_pypy_c/test_thread.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_thread.py
@@ -64,7 +64,7 @@
         guard_true(i56, descr=...)
         p57 = force_token()
         setfield_gc(p0, p57, descr=)
-        i58 = call_release_gil(..., i37, 1, descr=)
+        i58 = call_release_gil(_, i37, 1, descr=)
         guard_not_forced(descr=...)
         guard_no_exception(descr=...)
         i59 = int_is_true(i58)
@@ -72,14 +72,14 @@
         i60 = int_sub(i44, 1)
         p62 = force_token()
         setfield_gc(p0, p62, descr=)
-        i63 = call_release_gil(..., i37, 0, descr=)
+        i63 = call_release_gil(_, i37, 0, descr=)
         guard_not_forced(descr=...)
         guard_no_exception(descr=...)
         i64 = int_is_true(i63)
         guard_false(i64, descr=...)
         p65 = force_token()
         setfield_gc(p0, p65, descr=)
-        call_release_gil(..., i37, descr=)
+        call_release_gil(_, i37, descr=)
         guard_not_forced(descr=...)
         guard_no_exception(descr=...)
         guard_not_invalidated(descr=...)
diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py
--- a/rpython/jit/backend/x86/assembler.py
+++ b/rpython/jit/backend/x86/assembler.py
@@ -1833,15 +1833,17 @@
             offset = self.cpu.get_ofs_of_frame_field('jf_guard_exc')
             mc.MOV_br(offset, ebx.value)
 
-        # now we return from the complete frame, which starts from
-        # _call_header_with_stack_check().  The LEA in _call_footer below
-        # throws away most of the frame, including all the PUSHes that we
-        # did just above.
+        # fill in the jf_descr and jf_gcmap fields of the frame according
+        # to which failure we are resuming from.  These are constants
+        # pushed on the stack just before we jump to the current helper,
+        # in generate_quick_failure().
         ofs = self.cpu.get_ofs_of_frame_field('jf_descr')
         ofs2 = self.cpu.get_ofs_of_frame_field('jf_gcmap')
         mc.POP_b(ofs2)
         mc.POP_b(ofs)
 
+        # now we return from the complete frame, which starts from
+        # _call_header_with_stack_check().  The _call_footer below does it.
         self._call_footer()
         rawstart = mc.materialize(self.cpu.asmmemmgr, [])
         self.failure_recovery_code[exc + 2 * withfloats] = rawstart
diff --git a/rpython/rtyper/normalizecalls.py b/rpython/rtyper/normalizecalls.py
--- a/rpython/rtyper/normalizecalls.py
+++ b/rpython/rtyper/normalizecalls.py
@@ -298,12 +298,16 @@
 
 # ____________________________________________________________
 
+class TooLateForNewSubclass(Exception):
+    pass
+
 class TotalOrderSymbolic(ComputedIntSymbolic):
 
     def __init__(self, orderwitness, peers):
         self.orderwitness = orderwitness
         self.peers = peers
         self.value = None
+        self._with_subclasses = None    # unknown
         peers.append(self)
 
     def __cmp__(self, other):
@@ -320,12 +324,34 @@
     def __rsub__(self, other):
         return other - self.compute_fn()
 
+    def check_any_subclass_in_peer_list(self, i):
+        # check if the next peer, in order, is or not the end
+        # marker for this start marker
+        assert self.peers[i] is self
+        return self.peers[i + 1].orderwitness != self.orderwitness + [MAX]
+
+    def number_with_subclasses(self):
+        # Return True or False depending on whether this is the
+        # subclassrange_min corresponding to a class which has subclasses
+        # or not.  If this is called and returns False, then adding later
+        # new subclasses will crash in compute_fn().
+        if self._with_subclasses is None:     # unknown so far
+            self.peers.sort()
+            i = self.peers.index(self)
+            self._with_subclasses = self.check_any_subclass_in_peer_list(i)
+        return self._with_subclasses
+
     def compute_fn(self):
         if self.value is None:
             self.peers.sort()
             for i, peer in enumerate(self.peers):
                 assert peer.value is None or peer.value == i
                 peer.value = i
+                #
+                if peer._with_subclasses is False:
+                    if peer.check_any_subclass_in_peer_list(i):
+                        raise TooLateForNewSubclass
+                #
             assert self.value is not None
         return self.value
 
diff --git a/rpython/rtyper/rclass.py b/rpython/rtyper/rclass.py
--- a/rpython/rtyper/rclass.py
+++ b/rpython/rtyper/rclass.py
@@ -1007,14 +1007,11 @@
         v_obj, v_cls = hop.inputargs(instance_repr, class_repr)
         if isinstance(v_cls, Constant):
             cls = v_cls.value
-            # XXX re-implement the following optimization
-            #if cls.subclassrange_max == cls.subclassrange_min:
-            #    # a class with no subclass
-            #    return hop.gendirectcall(rclass.ll_isinstance_exact, v_obj, v_cls)
-            #else:
-            minid = hop.inputconst(Signed, cls.subclassrange_min)
-            maxid = hop.inputconst(Signed, cls.subclassrange_max)
-            return hop.gendirectcall(ll_isinstance_const, v_obj, minid, maxid)
+            llf, llf_nonnull = make_ll_isinstance(self.rtyper, cls)
+            if hop.args_s[0].can_be_None:
+                return hop.gendirectcall(llf, v_obj)
+            else:
+                return hop.gendirectcall(llf_nonnull, v_obj)
         else:
             return hop.gendirectcall(ll_isinstance, v_obj, v_cls)
 
@@ -1128,16 +1125,26 @@
     obj_cls = obj.typeptr
     return ll_issubclass(obj_cls, cls)
 
-def ll_isinstance_const(obj, minid, maxid):
-    if not obj:
-        return False
-    return ll_issubclass_const(obj.typeptr, minid, maxid)
-
-def ll_isinstance_exact(obj, cls):
-    if not obj:
-        return False
-    obj_cls = obj.typeptr
-    return obj_cls == cls
+def make_ll_isinstance(rtyper, cls):
+    try:
+        return rtyper.isinstance_helpers[cls._obj]
+    except KeyError:
+        minid = cls.subclassrange_min
+        maxid = cls.subclassrange_max
+        if minid.number_with_subclasses():
+            def ll_isinstance_const_nonnull(obj):
+                objid = obj.typeptr.subclassrange_min
+                return llop.int_between(Bool, minid, objid, maxid)
+        else:
+            def ll_isinstance_const_nonnull(obj):
+                return obj.typeptr == cls
+        def ll_isinstance_const(obj):
+            if not obj:
+                return False
+            return ll_isinstance_const_nonnull(obj)
+        result = (ll_isinstance_const, ll_isinstance_const_nonnull)
+        rtyper.isinstance_helpers[cls._obj] = result
+        return result
 
 def ll_runtime_type_info(obj):
     return obj.typeptr.rtti
diff --git a/rpython/rtyper/rtyper.py b/rpython/rtyper/rtyper.py
--- a/rpython/rtyper/rtyper.py
+++ b/rpython/rtyper/rtyper.py
@@ -59,6 +59,7 @@
         self.typererror_count = 0
         # make the primitive_to_repr constant mapping
         self.primitive_to_repr = {}
+        self.isinstance_helpers = {}
         self.exceptiondata = ExceptionData(self)
         self.custom_trace_funcs = []
 
diff --git a/rpython/rtyper/test/test_normalizecalls.py b/rpython/rtyper/test/test_normalizecalls.py
--- a/rpython/rtyper/test/test_normalizecalls.py
+++ b/rpython/rtyper/test/test_normalizecalls.py
@@ -6,6 +6,7 @@
 from rpython.rtyper.test.test_llinterp import interpret
 from rpython.rtyper.lltypesystem import lltype
 from rpython.rtyper.normalizecalls import TotalOrderSymbolic, MAX
+from rpython.rtyper.normalizecalls import TooLateForNewSubclass
 
 
 def test_TotalOrderSymbolic():
@@ -21,6 +22,49 @@
     assert t1 <= 5
     assert t1.value == 0
 
+def test_TotalOrderSymbolic_with_subclasses():
+    lst = []
+    t3 = TotalOrderSymbolic([3, 4, 2, MAX], lst)
+    t1 = TotalOrderSymbolic([3, 4], lst)
+    t2 = TotalOrderSymbolic([3, 4, 2], lst)
+    t4 = TotalOrderSymbolic([3, 4, MAX], lst)
+    assert t1.number_with_subclasses()
+    assert not t2.number_with_subclasses()
+    assert [t.compute_fn() for t in [t1, t2, t3, t4]] == range(4)
+    #
+    lst = []
+    t1 = TotalOrderSymbolic([3, 4], lst)
+    t3 = TotalOrderSymbolic([3, 4, 2, MAX], lst)
+    t4 = TotalOrderSymbolic([3, 4, MAX], lst)
+    t2 = TotalOrderSymbolic([3, 4, 2], lst)
+    assert not t2.number_with_subclasses()
+    assert t1.number_with_subclasses()
+    assert [t.compute_fn() for t in [t1, t2, t3, t4]] == range(4)
+    #
+    lst = []
+    t1 = TotalOrderSymbolic([3, 4], lst)
+    t4 = TotalOrderSymbolic([3, 4, MAX], lst)
+    assert not t1.number_with_subclasses()
+    t2 = TotalOrderSymbolic([3, 4, 2], lst)
+    t3 = TotalOrderSymbolic([3, 4, 2, MAX], lst)
+    py.test.raises(TooLateForNewSubclass, t2.compute_fn)
+    #
+    lst = []
+    t1 = TotalOrderSymbolic([3, 4], lst)
+    t4 = TotalOrderSymbolic([3, 4, MAX], lst)
+    assert not t1.number_with_subclasses()
+    t2 = TotalOrderSymbolic([1], lst)
+    t3 = TotalOrderSymbolic([1, MAX], lst)
+    assert [t.compute_fn() for t in [t2, t3, t1, t4]] == range(4)
+    #
+    lst = []
+    t1 = TotalOrderSymbolic([3, 4], lst)
+    t4 = TotalOrderSymbolic([3, 4, MAX], lst)
+    assert not t1.number_with_subclasses()
+    t2 = TotalOrderSymbolic([6], lst)
+    t3 = TotalOrderSymbolic([6, MAX], lst)
+    assert [t.compute_fn() for t in [t1, t4, t2, t3]] == range(4)
+
 # ____________________________________________________________
 
 class TestNormalize(object):
diff --git a/rpython/tool/jitlogparser/parser.py b/rpython/tool/jitlogparser/parser.py
--- a/rpython/tool/jitlogparser/parser.py
+++ b/rpython/tool/jitlogparser/parser.py
@@ -216,7 +216,7 @@
     line_starts_here = property(getline_starts_here)
 
     def __repr__(self):
-        return "[%s]" % ", ".join([repr(op) for op in self.operations])
+        return "[%s\n]" % "\n    ".join([repr(op) for op in self.operations])
 
     def pretty_print(self, out):
         pass

From noreply at buildbot.pypy.org  Wed Nov 26 03:22:18 2014
From: noreply at buildbot.pypy.org (pjenvey)
Date: Wed, 26 Nov 2014 03:22:18 +0100 (CET)
Subject: [pypy-commit] pypy default: cleanup/arrange imports
Message-ID: <20141126022218.C87D31C02EE@cobra.cs.uni-duesseldorf.de>

Author: Philip Jenvey 
Branch: 
Changeset: r74722:290d6ee4dc2d
Date: 2014-11-25 18:22 -0800
http://bitbucket.org/pypy/pypy/changeset/290d6ee4dc2d/

Log:	cleanup/arrange imports

diff --git a/pypy/interpreter/astcompiler/assemble.py b/pypy/interpreter/astcompiler/assemble.py
--- a/pypy/interpreter/astcompiler/assemble.py
+++ b/pypy/interpreter/astcompiler/assemble.py
@@ -1,15 +1,13 @@
-"""
-Python control flow graph generation and bytecode assembly.
-"""
+"""Python control flow graph generation and bytecode assembly."""
 
-from pypy.interpreter.astcompiler import ast, symtable, misc
-from pypy.interpreter import pycode
+from rpython.rlib import rfloat
+from rpython.rlib.objectmodel import we_are_translated
+
+from pypy.interpreter.astcompiler import ast, misc, symtable
+from pypy.interpreter.error import OperationError
+from pypy.interpreter.pycode import PyCode
 from pypy.tool import stdlib_opcode as ops
 
-from pypy.interpreter.error import OperationError
-from rpython.rlib.objectmodel import we_are_translated
-from rpython.rlib import rfloat
-
 
 class Instruction(object):
     """Represents a single opcode."""
@@ -21,14 +19,12 @@
         self.has_jump = False
 
     def size(self):
-        """Return the size of bytes of this instruction when it is encoded."""
+        """Return the size of bytes of this instruction when it is
+        encoded.
+        """
         if self.opcode >= ops.HAVE_ARGUMENT:
-            if self.arg > 0xFFFF:
-                return 6
-            else:
-                return 3
-        else:
-            return 1
+            return (6 if self.arg > 0xFFFF else 3)
+        return 1
 
     def jump_to(self, target, absolute=False):
         """Indicate the target this jump instruction.
@@ -54,9 +50,9 @@
 class Block(object):
     """A basic control flow block.
 
-    It has one entry point and several possible exit points.  Its instructions
-    may be jumps to other blocks, or if control flow reaches the end of the
-    block, it continues to next_block.
+    It has one entry point and several possible exit points.  Its
+    instructions may be jumps to other blocks, or if control flow
+    reaches the end of the block, it continues to next_block.
     """
 
     def __init__(self):
@@ -71,10 +67,10 @@
             stack.append(nextblock)
 
     def post_order(self):
-        """Return this block and its children in post order.
-        This means that the graph of blocks is first cleaned up to
-        ignore back-edges, thus turning it into a DAG.  Then the DAG
-        is linearized.  For example:
+        """Return this block and its children in post order.  This means
+        that the graph of blocks is first cleaned up to ignore
+        back-edges, thus turning it into a DAG.  Then the DAG is
+        linearized.  For example:
 
                    A --> B -\           =>     [A, D, B, C]
                      \-> D ---> C
@@ -105,7 +101,9 @@
         return resultblocks
 
     def code_size(self):
-        """Return the encoded size of all the instructions in this block."""
+        """Return the encoded size of all the instructions in this
+        block.
+        """
         i = 0
         for instr in self.instructions:
             i += instr.size()
@@ -141,6 +139,7 @@
             i += 1
     return result
 
+
 def _list_to_dict(l, offset=0):
     result = {}
     index = offset
@@ -300,11 +299,11 @@
     def _resolve_block_targets(self, blocks):
         """Compute the arguments of jump instructions."""
         last_extended_arg_count = 0
-        # The reason for this loop is extended jumps.  EXTENDED_ARG extends the
-        # bytecode size, so it might invalidate the offsets we've already given.
-        # Thus we have to loop until the number of extended args is stable.  Any
-        # extended jump at all is extremely rare, so performance is not too
-        # concerning.
+        # The reason for this loop is extended jumps.  EXTENDED_ARG
+        # extends the bytecode size, so it might invalidate the offsets
+        # we've already given.  Thus we have to loop until the number of
+        # extended args is stable.  Any extended jump at all is
+        # extremely rare, so performance is not too concerning.
         while True:
             extended_arg_count = 0
             offset = 0
@@ -330,7 +329,8 @@
                                     instr.opcode = ops.JUMP_ABSOLUTE
                                     absolute = True
                                 elif target_op == ops.RETURN_VALUE:
-                                    # Replace JUMP_* to a RETURN into just a RETURN
+                                    # Replace JUMP_* to a RETURN into
+                                    # just a RETURN
                                     instr.opcode = ops.RETURN_VALUE
                                     instr.arg = 0
                                     instr.has_jump = False
@@ -345,7 +345,8 @@
                         instr.arg = jump_arg
                         if jump_arg > 0xFFFF:
                             extended_arg_count += 1
-            if extended_arg_count == last_extended_arg_count and not force_redo:
+            if (extended_arg_count == last_extended_arg_count and
+                not force_redo):
                 break
             else:
                 last_extended_arg_count = extended_arg_count
@@ -360,7 +361,7 @@
         while True:
             try:
                 w_key = space.next(w_iter)
-            except OperationError, e:
+            except OperationError as e:
                 if not e.match(space, space.w_StopIteration):
                     raise
                 break
@@ -435,15 +436,16 @@
                         continue
                     addr = offset - current_off
                     # Python assumes that lineno always increases with
-                    # increasing bytecode address (lnotab is unsigned char).
-                    # Depending on when SET_LINENO instructions are emitted this
-                    # is not always true.  Consider the code:
+                    # increasing bytecode address (lnotab is unsigned
+                    # char).  Depending on when SET_LINENO instructions
+                    # are emitted this is not always true.  Consider the
+                    # code:
                     #     a = (1,
                     #          b)
-                    # In the bytecode stream, the assignment to "a" occurs after
-                    # the loading of "b".  This works with the C Python compiler
-                    # because it only generates a SET_LINENO instruction for the
-                    # assignment.
+                    # In the bytecode stream, the assignment to "a"
+                    # occurs after the loading of "b".  This works with
+                    # the C Python compiler because it only generates a
+                    # SET_LINENO instruction for the assignment.
                     if line or addr:
                         while addr > 255:
                             push(chr(255))
@@ -486,22 +488,22 @@
         free_names = _list_from_dict(self.free_vars, len(cell_names))
         flags = self._get_code_flags() | self.compile_info.flags
         bytecode = ''.join([block.get_code() for block in blocks])
-        return pycode.PyCode(self.space,
-                             self.argcount,
-                             len(self.var_names),
-                             stack_depth,
-                             flags,
-                             bytecode,
-                             list(consts_w),
-                             names,
-                             var_names,
-                             self.compile_info.filename,
-                             self.name,
-                             self.first_lineno,
-                             lnotab,
-                             free_names,
-                             cell_names,
-                             self.compile_info.hidden_applevel)
+        return PyCode(self.space,
+                      self.argcount,
+                      len(self.var_names),
+                      stack_depth,
+                      flags,
+                      bytecode,
+                      list(consts_w),
+                      names,
+                      var_names,
+                      self.compile_info.filename,
+                      self.name,
+                      self.first_lineno,
+                      lnotab,
+                      free_names,
+                      cell_names,
+                      self.compile_info.hidden_applevel)
 
 
 def _list_from_dict(d, offset=0):
@@ -512,134 +514,134 @@
 
 
 _static_opcode_stack_effects = {
-    ops.NOP : 0,
-    ops.STOP_CODE : 0,
+    ops.NOP: 0,
+    ops.STOP_CODE: 0,
 
-    ops.POP_TOP : -1,
-    ops.ROT_TWO : 0,
-    ops.ROT_THREE : 0,
-    ops.ROT_FOUR : 0,
-    ops.DUP_TOP : 1,
+    ops.POP_TOP: -1,
+    ops.ROT_TWO: 0,
+    ops.ROT_THREE: 0,
+    ops.ROT_FOUR: 0,
+    ops.DUP_TOP: 1,
 
-    ops.UNARY_POSITIVE : 0,
-    ops.UNARY_NEGATIVE : 0,
-    ops.UNARY_NOT : 0,
-    ops.UNARY_CONVERT : 0,
-    ops.UNARY_INVERT : 0,
+    ops.UNARY_POSITIVE: 0,
+    ops.UNARY_NEGATIVE: 0,
+    ops.UNARY_NOT: 0,
+    ops.UNARY_CONVERT: 0,
+    ops.UNARY_INVERT: 0,
 
-    ops.LIST_APPEND : -1,
-    ops.SET_ADD : -1,
-    ops.MAP_ADD : -2,
-    ops.STORE_MAP : -2,
+    ops.LIST_APPEND: -1,
+    ops.SET_ADD: -1,
+    ops.MAP_ADD: -2,
+    ops.STORE_MAP: -2,
 
-    ops.BINARY_POWER : -1,
-    ops.BINARY_MULTIPLY : -1,
-    ops.BINARY_DIVIDE : -1,
-    ops.BINARY_MODULO : -1,
-    ops.BINARY_ADD : -1,
-    ops.BINARY_SUBTRACT : -1,
-    ops.BINARY_SUBSCR : -1,
-    ops.BINARY_FLOOR_DIVIDE : -1,
-    ops.BINARY_TRUE_DIVIDE : -1,
-    ops.BINARY_LSHIFT : -1,
-    ops.BINARY_RSHIFT : -1,
-    ops.BINARY_AND : -1,
-    ops.BINARY_OR : -1,
-    ops.BINARY_XOR : -1,
+    ops.BINARY_POWER: -1,
+    ops.BINARY_MULTIPLY: -1,
+    ops.BINARY_DIVIDE: -1,
+    ops.BINARY_MODULO: -1,
+    ops.BINARY_ADD: -1,
+    ops.BINARY_SUBTRACT: -1,
+    ops.BINARY_SUBSCR: -1,
+    ops.BINARY_FLOOR_DIVIDE: -1,
+    ops.BINARY_TRUE_DIVIDE: -1,
+    ops.BINARY_LSHIFT: -1,
+    ops.BINARY_RSHIFT: -1,
+    ops.BINARY_AND: -1,
+    ops.BINARY_OR: -1,
+    ops.BINARY_XOR: -1,
 
-    ops.INPLACE_FLOOR_DIVIDE : -1,
-    ops.INPLACE_TRUE_DIVIDE : -1,
-    ops.INPLACE_ADD : -1,
-    ops.INPLACE_SUBTRACT : -1,
-    ops.INPLACE_MULTIPLY : -1,
-    ops.INPLACE_DIVIDE : -1,
-    ops.INPLACE_MODULO : -1,
-    ops.INPLACE_POWER : -1,
-    ops.INPLACE_LSHIFT : -1,
-    ops.INPLACE_RSHIFT : -1,
-    ops.INPLACE_AND : -1,
-    ops.INPLACE_OR : -1,
-    ops.INPLACE_XOR : -1,
+    ops.INPLACE_FLOOR_DIVIDE: -1,
+    ops.INPLACE_TRUE_DIVIDE: -1,
+    ops.INPLACE_ADD: -1,
+    ops.INPLACE_SUBTRACT: -1,
+    ops.INPLACE_MULTIPLY: -1,
+    ops.INPLACE_DIVIDE: -1,
+    ops.INPLACE_MODULO: -1,
+    ops.INPLACE_POWER: -1,
+    ops.INPLACE_LSHIFT: -1,
+    ops.INPLACE_RSHIFT: -1,
+    ops.INPLACE_AND: -1,
+    ops.INPLACE_OR: -1,
+    ops.INPLACE_XOR: -1,
 
-    ops.SLICE+0 : 1,
-    ops.SLICE+1 : 0,
-    ops.SLICE+2 : 0,
-    ops.SLICE+3 : -1,
-    ops.STORE_SLICE+0 : -2,
-    ops.STORE_SLICE+1 : -3,
-    ops.STORE_SLICE+2 : -3,
-    ops.STORE_SLICE+3 : -4,
-    ops.DELETE_SLICE+0 : -1,
-    ops.DELETE_SLICE+1 : -2,
-    ops.DELETE_SLICE+2 : -2,
-    ops.DELETE_SLICE+3 : -3,
+    ops.SLICE+0: 1,
+    ops.SLICE+1: 0,
+    ops.SLICE+2: 0,
+    ops.SLICE+3: -1,
+    ops.STORE_SLICE+0: -2,
+    ops.STORE_SLICE+1: -3,
+    ops.STORE_SLICE+2: -3,
+    ops.STORE_SLICE+3: -4,
+    ops.DELETE_SLICE+0: -1,
+    ops.DELETE_SLICE+1: -2,
+    ops.DELETE_SLICE+2: -2,
+    ops.DELETE_SLICE+3: -3,
 
-    ops.STORE_SUBSCR : -2,
-    ops.DELETE_SUBSCR : -2,
+    ops.STORE_SUBSCR: -2,
+    ops.DELETE_SUBSCR: -2,
 
-    ops.GET_ITER : 0,
-    ops.FOR_ITER : 1,
-    ops.BREAK_LOOP : 0,
-    ops.CONTINUE_LOOP : 0,
-    ops.SETUP_LOOP : 0,
+    ops.GET_ITER: 0,
+    ops.FOR_ITER: 1,
+    ops.BREAK_LOOP: 0,
+    ops.CONTINUE_LOOP: 0,
+    ops.SETUP_LOOP: 0,
 
-    ops.PRINT_EXPR : -1,
-    ops.PRINT_ITEM : -1,
-    ops.PRINT_NEWLINE : 0,
-    ops.PRINT_ITEM_TO : -2,
-    ops.PRINT_NEWLINE_TO : -1,
+    ops.PRINT_EXPR: -1,
+    ops.PRINT_ITEM: -1,
+    ops.PRINT_NEWLINE: 0,
+    ops.PRINT_ITEM_TO: -2,
+    ops.PRINT_NEWLINE_TO: -1,
 
-    ops.WITH_CLEANUP : -1,
-    ops.POP_BLOCK : 0,
-    ops.END_FINALLY : -1,
-    ops.SETUP_WITH : 1,
-    ops.SETUP_FINALLY : 0,
-    ops.SETUP_EXCEPT : 0,
+    ops.WITH_CLEANUP: -1,
+    ops.POP_BLOCK: 0,
+    ops.END_FINALLY: -1,
+    ops.SETUP_WITH: 1,
+    ops.SETUP_FINALLY: 0,
+    ops.SETUP_EXCEPT: 0,
 
-    ops.LOAD_LOCALS : 1,
-    ops.RETURN_VALUE : -1,
-    ops.EXEC_STMT : -3,
-    ops.YIELD_VALUE : 0,
-    ops.BUILD_CLASS : -2,
-    ops.BUILD_MAP : 1,
-    ops.BUILD_SET : 1,
-    ops.COMPARE_OP : -1,
+    ops.LOAD_LOCALS: 1,
+    ops.RETURN_VALUE: -1,
+    ops.EXEC_STMT: -3,
+    ops.YIELD_VALUE: 0,
+    ops.BUILD_CLASS: -2,
+    ops.BUILD_MAP: 1,
+    ops.BUILD_SET: 1,
+    ops.COMPARE_OP: -1,
 
-    ops.LOOKUP_METHOD : 1,
+    ops.LOOKUP_METHOD: 1,
 
-    ops.LOAD_NAME : 1,
-    ops.STORE_NAME : -1,
-    ops.DELETE_NAME : 0,
+    ops.LOAD_NAME: 1,
+    ops.STORE_NAME: -1,
+    ops.DELETE_NAME: 0,
 
-    ops.LOAD_FAST : 1,
-    ops.STORE_FAST : -1,
-    ops.DELETE_FAST : 0,
+    ops.LOAD_FAST: 1,
+    ops.STORE_FAST: -1,
+    ops.DELETE_FAST: 0,
 
-    ops.LOAD_ATTR : 0,
-    ops.STORE_ATTR : -2,
-    ops.DELETE_ATTR : -1,
+    ops.LOAD_ATTR: 0,
+    ops.STORE_ATTR: -2,
+    ops.DELETE_ATTR: -1,
 
-    ops.LOAD_GLOBAL : 1,
-    ops.STORE_GLOBAL : -1,
-    ops.DELETE_GLOBAL : 0,
+    ops.LOAD_GLOBAL: 1,
+    ops.STORE_GLOBAL: -1,
+    ops.DELETE_GLOBAL: 0,
 
-    ops.LOAD_CLOSURE : 1,
-    ops.LOAD_DEREF : 1,
-    ops.STORE_DEREF : -1,
+    ops.LOAD_CLOSURE: 1,
+    ops.LOAD_DEREF: 1,
+    ops.STORE_DEREF: -1,
 
-    ops.LOAD_CONST : 1,
+    ops.LOAD_CONST: 1,
 
-    ops.IMPORT_STAR : -1,
-    ops.IMPORT_NAME : -1,
-    ops.IMPORT_FROM : 1,
+    ops.IMPORT_STAR: -1,
+    ops.IMPORT_NAME: -1,
+    ops.IMPORT_FROM: 1,
 
-    ops.JUMP_FORWARD : 0,
-    ops.JUMP_ABSOLUTE : 0,
-    ops.JUMP_IF_TRUE_OR_POP : 0,
-    ops.JUMP_IF_FALSE_OR_POP : 0,
-    ops.POP_JUMP_IF_TRUE : -1,
-    ops.POP_JUMP_IF_FALSE : -1,
-    ops.JUMP_IF_NOT_DEBUG : 0,
+    ops.JUMP_FORWARD: 0,
+    ops.JUMP_ABSOLUTE: 0,
+    ops.JUMP_IF_TRUE_OR_POP: 0,
+    ops.JUMP_IF_FALSE_OR_POP: 0,
+    ops.POP_JUMP_IF_TRUE: -1,
+    ops.POP_JUMP_IF_FALSE: -1,
+    ops.JUMP_IF_NOT_DEBUG: 0,
 
     ops.BUILD_LIST_FROM_ARG: 1,
 }

From noreply at buildbot.pypy.org  Wed Nov 26 10:11:30 2014
From: noreply at buildbot.pypy.org (fijal)
Date: Wed, 26 Nov 2014 10:11:30 +0100 (CET)
Subject: [pypy-commit] pypy optresult: finish porting test_runner (yay!)
Message-ID: <20141126091130.9779A1C0EF6@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: optresult
Changeset: r74723:94595cb2a9e7
Date: 2014-11-26 11:11 +0200
http://bitbucket.org/pypy/pypy/changeset/94595cb2a9e7/

Log:	finish porting test_runner (yay!)

diff --git a/rpython/jit/backend/llgraph/runner.py b/rpython/jit/backend/llgraph/runner.py
--- a/rpython/jit/backend/llgraph/runner.py
+++ b/rpython/jit/backend/llgraph/runner.py
@@ -949,7 +949,7 @@
     execute_call_may_force_f = _execute_call_may_force
     execute_call_may_force_i = _execute_call_may_force
 
-    def execute_call_release_gil(self, descr, func, *args):
+    def _execute_call_release_gil(self, descr, func, *args):
         if hasattr(descr, '_original_func_'):
             func = descr._original_func_     # see pyjitpl.py
             # we want to call the function that does the aroundstate
@@ -974,7 +974,12 @@
         del self.force_guard_op
         return support.cast_result(descr.RESULT, result)
 
-    def execute_call_assembler(self, descr, *args):
+    execute_call_release_gil_n = _execute_call_release_gil
+    execute_call_release_gil_i = _execute_call_release_gil
+    execute_call_release_gil_r = _execute_call_release_gil
+    execute_call_release_gil_f = _execute_call_release_gil
+
+    def _execute_call_assembler(self, descr, *args):
         # XXX simplify the following a bit
         #
         # pframe = CALL_ASSEMBLER(args..., descr=looptoken)
@@ -1027,6 +1032,11 @@
             result = support.cast_to_floatstorage(result)
         return result
 
+    execute_call_assembler_i = _execute_call_assembler
+    execute_call_assembler_r = _execute_call_assembler
+    execute_call_assembler_f = _execute_call_assembler
+    execute_call_assembler_n = _execute_call_assembler
+
     def execute_same_as_i(self, _, x):
         return x
     execute_same_as_f = execute_same_as_i
diff --git a/rpython/jit/backend/test/runner_test.py b/rpython/jit/backend/test/runner_test.py
--- a/rpython/jit/backend/test/runner_test.py
+++ b/rpython/jit/backend/test/runner_test.py
@@ -2423,22 +2423,18 @@
         assert c_tolower.call(argchain, rffi.INT) == ord('a')
 
         cpu = self.cpu
-        func_adr = llmemory.cast_ptr_to_adr(c_tolower.funcsym)
-        funcbox = ConstInt(heaptracker.adr2int(func_adr))
+        func_adr = c_tolower.funcsym
         calldescr = cpu._calldescr_dynamic_for_tests([types.uchar], types.sint)
-        i1 = InputArgInt()
-        i2 = InputArgInt()
-        tok = InputArgInt()
         faildescr = BasicFailDescr(1)
-        ops = [
-        ResOperation(rop.CALL_RELEASE_GIL, [funcbox, i1], i2,
-                     descr=calldescr),
-        ResOperation(rop.GUARD_NOT_FORCED, [], None, descr=faildescr),
-        ResOperation(rop.FINISH, [i2], None, descr=BasicFinalDescr(0))
-        ]
-        ops[1].setfailargs([i1, i2])
+        finaldescr = BasicFinalDescr(0)
+        loop = parse("""
+        [i1]
+        i2 = call_release_gil_i(ConstClass(func_adr), i1, descr=calldescr)
+        guard_not_forced(descr=faildescr) [i1, i2]
+        finish(i2, descr=finaldescr)
+        """, namespace=locals())
         looptoken = JitCellToken()
-        self.cpu.compile_loop([i1], ops, looptoken)
+        self.cpu.compile_loop(loop.inputargs, loop.operations, looptoken)
         deadframe = self.cpu.execute_token(looptoken, ord('G'))
         fail = self.cpu.get_latest_descr(deadframe)
         assert fail.identifier == 0
@@ -2477,26 +2473,20 @@
         del glob.lst[:]
 
         cpu = self.cpu
-        func_adr = llmemory.cast_ptr_to_adr(c_qsort.funcsym)
-        funcbox = ConstInt(heaptracker.adr2int(func_adr))
+        func_adr = c_qsort.funcsym
         calldescr = cpu._calldescr_dynamic_for_tests(
             [types.pointer, types_size_t, types_size_t, types.pointer],
             types.void)
-        i0 = InputArgInt()
-        i1 = InputArgInt()
-        i2 = InputArgInt()
-        i3 = InputArgInt()
-        tok = InputArgInt()
         faildescr = BasicFailDescr(1)
-        ops = [
-        ResOperation(rop.CALL_RELEASE_GIL, [funcbox, i0, i1, i2, i3], None,
-                     descr=calldescr),
-        ResOperation(rop.GUARD_NOT_FORCED, [], None, descr=faildescr),
-        ResOperation(rop.FINISH, [], None, descr=BasicFinalDescr(0))
-        ]
-        ops[1].setfailargs([])
+        finaldescr = BasicFinalDescr(0)
+        loop = parse("""
+        [i0, i1, i2, i3]
+        call_release_gil_n(ConstClass(func_adr), i0, i1, i2, i3, descr=calldescr)
+        guard_not_forced(descr=faildescr) []
+        finish(descr=finaldescr)
+        """, namespace=locals())
         looptoken = JitCellToken()
-        self.cpu.compile_loop([i0, i1, i2, i3], ops, looptoken)
+        self.cpu.compile_loop(loop.inputargs, loop.operations, looptoken)
         args = [rffi.cast(lltype.Signed, raw),
                 2,
                 4,
@@ -2609,18 +2599,16 @@
             faildescr = BasicFailDescr(1)
             kind = types.getkind(ffitype)
             if kind in 'uis':
-                b3 = InputArgInt()
+                opnum = rop.CALL_RELEASE_GIL_I
             elif kind in 'fUI':
-                b3 = BoxFloat()
+                opnum = rop.CALL_RELEASE_GIL_F
             else:
                 assert 0, kind
             #
-            ops = [
-                ResOperation(rop.CALL_RELEASE_GIL, [funcbox], b3,
-                             descr=calldescr),
-                ResOperation(rop.GUARD_NOT_FORCED, [], None, descr=faildescr),
-                ResOperation(rop.FINISH, [b3], None, descr=BasicFinalDescr(0))
-                ]
+            op0 = ResOperation(opnum, [funcbox], descr=calldescr)
+            op1 = ResOperation(rop.GUARD_NOT_FORCED, [], descr=faildescr)
+            op2 = ResOperation(rop.FINISH, [op0], BasicFinalDescr(0))
+            ops = [op0, op1, op2]
             ops[1].setfailargs([])
             looptoken = JitCellToken()
             self.cpu.compile_loop([], ops, looptoken)
@@ -2628,7 +2616,7 @@
             deadframe = self.cpu.execute_token(looptoken)
             fail = self.cpu.get_latest_descr(deadframe)
             assert fail.identifier == 0
-            if isinstance(b3, InputArgInt):
+            if kind in 'uis':
                 r = self.cpu.get_int_value(deadframe, 0)
                 if isinstance(result, r_singlefloat):
                     assert -sys.maxint-1 <= r <= 0xFFFFFFFF
@@ -2637,7 +2625,7 @@
                 else:
                     r = rffi.cast(TP, r)
                 assert r == result
-            elif isinstance(b3, BoxFloat):
+            elif kind in 'fUI':
                 r = self.cpu.get_float_value(deadframe, 0)
                 if isinstance(result, float):
                     r = longlong.getrealfloat(r)
@@ -2838,16 +2826,15 @@
 
     def test_guard_not_invalidated(self):
         cpu = self.cpu
-        i0 = InputArgInt()
-        i1 = InputArgInt()
         faildescr = BasicFailDescr(1)
-        ops = [
-            ResOperation(rop.GUARD_NOT_INVALIDATED, [], None, descr=faildescr),
-            ResOperation(rop.FINISH, [i0], None, descr=BasicFinalDescr(0))
-        ]
-        ops[0].setfailargs([i1])
+        finaldescr = BasicFinalDescr(0)
+        loop = parse("""
+        [i0, i1]
+        guard_not_invalidated(descr=faildescr) [i1]
+        finish(i0, descr=finaldescr)
+        """, namespace=locals())
         looptoken = JitCellToken()
-        self.cpu.compile_loop([i0, i1], ops, looptoken)
+        self.cpu.compile_loop(loop.inputargs, loop.operations, looptoken)
 
         deadframe = self.cpu.execute_token(looptoken, -42, 9)
         fail = self.cpu.get_latest_descr(deadframe)
@@ -2867,14 +2854,15 @@
         print '-'*79
 
         # attach a bridge
-        i2 = InputArgInt()
         faildescr2 = BasicFailDescr(2)
-        ops = [
-            ResOperation(rop.GUARD_NOT_INVALIDATED, [],None, descr=faildescr2),
-            ResOperation(rop.FINISH, [i2], None, descr=BasicFinalDescr(3))
-        ]
-        ops[0].setfailargs([])
-        self.cpu.compile_bridge(faildescr, [i2], ops, looptoken)
+        finaldescr2 = BasicFinalDescr(3)
+        bridge = parse("""
+        [i2]
+        guard_not_invalidated(descr=faildescr2) []
+        finish(i2, descr=finaldescr2)
+        """, namespace=locals())
+        self.cpu.compile_bridge(faildescr, bridge.inputargs,
+                                bridge.operations, looptoken)
 
         deadframe = self.cpu.execute_token(looptoken, -42, 9)
         fail = self.cpu.get_latest_descr(deadframe)
@@ -2897,23 +2885,23 @@
         # the label.  If it doesn't, then in this example after we invalidate
         # the guard, jumping to the label will hit the invalidation code too
         cpu = self.cpu
-        i0 = InputArgInt()
         faildescr = BasicFailDescr(1)
         labeldescr = TargetToken()
-        ops = [
-            ResOperation(rop.GUARD_NOT_INVALIDATED, [], None, descr=faildescr),
-            ResOperation(rop.LABEL, [i0], None, descr=labeldescr),
-            ResOperation(rop.FINISH, [i0], None, descr=BasicFinalDescr(3)),
-        ]
-        ops[0].setfailargs([])
+        finaldescr = BasicFinalDescr(3)
+        loop = parse("""
+        [i0]
+        guard_not_invalidated(descr=faildescr) []
+        label(i0, descr=labeldescr)
+        finish(i0, descr=finaldescr)
+        """, namespace=locals())
         looptoken = JitCellToken()
-        self.cpu.compile_loop([i0], ops, looptoken)
+        self.cpu.compile_loop(loop.inputargs, loop.operations, looptoken)
         # mark as failing
         self.cpu.invalidate_loop(looptoken)
         # attach a bridge
         i2 = InputArgInt()
         ops2 = [
-            ResOperation(rop.JUMP, [ConstInt(333)], None, descr=labeldescr),
+            ResOperation(rop.JUMP, [ConstInt(333)], descr=labeldescr),
         ]
         self.cpu.compile_bridge(faildescr, [], ops2, looptoken)
         # run: must not be caught in an infinite loop
@@ -3135,7 +3123,7 @@
         ops = '''
         [i0, i1, i2, i3, i4, i5, i6, i7, i8, i9]
         i10 = int_add(i0, 42)
-        i11 = call_assembler(i10, i1, i2, i3, i4, i5, i6, i7, i8, i9, descr=looptoken)
+        i11 = call_assembler_i(i10, i1, i2, i3, i4, i5, i6, i7, i8, i9, descr=looptoken)
         guard_not_forced()[]
         finish(i11)
         '''
@@ -3249,7 +3237,7 @@
         assert longlong.getrealfloat(x) == 1.2 + 2.3
         ops = '''
         [f4, f5]
-        f3 = call_assembler(f4, f5, descr=looptoken)
+        f3 = call_assembler_f(f4, f5, descr=looptoken)
         guard_not_forced()[]
         finish(f3)
         '''
@@ -3282,9 +3270,9 @@
         a[7] = -4242
         addr = llmemory.cast_ptr_to_adr(a)
         abox = InputArgInt(heaptracker.adr2int(addr))
-        r1 = self.execute_operation(rop.GETARRAYITEM_RAW, [abox, InputArgInt(7)],
+        r1 = self.execute_operation(rop.GETARRAYITEM_RAW_I, [abox, InputArgInt(7)],
                                     'int', descr=descr)
-        assert r1.getint() == -4242
+        assert r1 == -4242
         lltype.free(a, flavor='raw')
 
     def test_raw_malloced_setarrayitem(self):
@@ -3341,7 +3329,7 @@
 
         ops = '''
         [f4, f5]
-        f3 = call_assembler(f4, f5, descr=looptoken)
+        f3 = call_assembler_f(f4, f5, descr=looptoken)
         guard_not_forced()[]
         finish(f3)
         '''
@@ -3416,10 +3404,11 @@
             expected = rffi.cast(lltype.Signed, rffi.cast(RESTYPE, value))
             s.x = rffi.cast(RESTYPE, value)
             s_gcref = lltype.cast_opaque_ptr(llmemory.GCREF, s)
-            res = self.execute_operation(rop.GETFIELD_GC, [BoxPtr(s_gcref)],
+            res = self.execute_operation(rop.GETFIELD_GC_I,
+                                         [InputArgRef(s_gcref)],
                                          'int', descr=descrfld_x)
-            assert res.value == expected, (
-                "%r: got %r, expected %r" % (RESTYPE, res.value, expected))
+            assert res == expected, (
+                "%r: got %r, expected %r" % (RESTYPE, res, expected))
 
     def test_short_result_of_getarrayitem_direct(self):
         # Test that a getarrayitem that returns a CHAR, SHORT or INT, signed
@@ -3457,11 +3446,11 @@
             expected = rffi.cast(lltype.Signed, rffi.cast(RESTYPE, value))
             a[3] = rffi.cast(RESTYPE, value)
             a_gcref = lltype.cast_opaque_ptr(llmemory.GCREF, a)
-            res = self.execute_operation(rop.GETARRAYITEM_GC,
-                                         [BoxPtr(a_gcref), InputArgInt(3)],
+            res = self.execute_operation(rop.GETARRAYITEM_GC_I,
+                                         [InputArgRef(a_gcref), InputArgInt(3)],
                                          'int', descr=descrarray)
-            assert res.value == expected, (
-                "%r: got %r, expected %r" % (RESTYPE, res.value, expected))
+            assert res == expected, (
+                "%r: got %r, expected %r" % (RESTYPE, res, expected))
 
     def test_short_result_of_getarrayitem_raw_direct(self):
         # Test that a getarrayitem that returns a CHAR, SHORT or INT, signed
@@ -3500,11 +3489,11 @@
             expected = rffi.cast(lltype.Signed, rffi.cast(RESTYPE, value))
             a[3] = rffi.cast(RESTYPE, value)
             a_rawint = heaptracker.adr2int(llmemory.cast_ptr_to_adr(a))
-            res = self.execute_operation(rop.GETARRAYITEM_RAW,
+            res = self.execute_operation(rop.GETARRAYITEM_RAW_I,
                                          [InputArgInt(a_rawint), InputArgInt(3)],
                                          'int', descr=descrarray)
-            assert res.value == expected, (
-                "%r: got %r, expected %r" % (RESTYPE, res.value, expected))
+            assert res == expected, (
+                "%r: got %r, expected %r" % (RESTYPE, res, expected))
             lltype.free(a, flavor='raw')
 
     def test_short_result_of_call_direct(self):
@@ -3569,10 +3558,10 @@
             calldescr = self.cpu.calldescrof(FUNC, FUNC.ARGS, FUNC.RESULT,
                                              EffectInfo.MOST_GENERAL)
             funcbox = self.get_funcbox(self.cpu, f)
-            res = self.execute_operation(rop.CALL, [funcbox, InputArgInt(value)],
+            res = self.execute_operation(rop.CALL_I, [funcbox, InputArgInt(value)],
                                          'int', descr=calldescr)
-            assert res.value == expected, (
-                "%r: got %r, expected %r" % (RESTYPE, res.value, expected))
+            assert res == expected, (
+                "%r: got %r, expected %r" % (RESTYPE, res, expected))
 
     def test_supports_longlong(self):
         if IS_64_BIT:
@@ -3689,9 +3678,9 @@
         funcbox = self.get_funcbox(self.cpu, f)
         ivalue = longlong.singlefloat2int(value)
         iexpected = longlong.singlefloat2int(expected)
-        res = self.execute_operation(rop.CALL, [funcbox, InputArgInt(ivalue)],
+        res = self.execute_operation(rop.CALL_I, [funcbox, InputArgInt(ivalue)],
                                      'int', descr=calldescr)
-        assert res.value == iexpected
+        assert res == iexpected
 
     def test_free_loop_and_bridges(self):
         from rpython.jit.backend.llsupport.llmodel import AbstractLLCPU
@@ -3747,47 +3736,44 @@
         calldescr = self.cpu.calldescrof(FUNC, FUNC.ARGS, FUNC.RESULT, effectinfo)
         testcases = [(4.0, 2.0), (6.25, 2.5)]
         for arg, expected in testcases:
-            res = self.execute_operation(rop.CALL,
+            res = self.execute_operation(rop.CALL_F,
                         [funcbox, boxfloat(arg)],
                          'float', descr=calldescr)
-            assert res.getfloat() == expected
+            assert res == expected
 
     def test_compile_loop_with_target(self):
-        i0 = InputArgInt()
-        i1 = InputArgInt()
-        i2 = InputArgInt()
-        i3 = InputArgInt()
         looptoken = JitCellToken()
         targettoken1 = TargetToken()
         targettoken2 = TargetToken()
         faildescr = BasicFailDescr(2)
-        operations = [
-            ResOperation(rop.LABEL, [i0], None, descr=targettoken1),
-            ResOperation(rop.INT_ADD, [i0, ConstInt(1)], i1),
-            ResOperation(rop.INT_LE, [i1, ConstInt(9)], i2),
-            ResOperation(rop.GUARD_TRUE, [i2], None, descr=faildescr),
-            ResOperation(rop.LABEL, [i1], None, descr=targettoken2),
-            ResOperation(rop.INT_GE, [i1, ConstInt(0)], i3),
-            ResOperation(rop.GUARD_TRUE, [i3], None, descr=BasicFailDescr(3)),
-            ResOperation(rop.JUMP, [i1], None, descr=targettoken1),
-            ]
-        inputargs = [i0]
-        operations[3].setfailargs([i1])
-        operations[6].setfailargs([i1])
-
-        self.cpu.compile_loop(inputargs, operations, looptoken)
+        faildescr3 = BasicFailDescr(3)
+        loop = parse("""
+        [i0]
+        label(i0, descr=targettoken1)
+        i1 = int_add(i0, 1)
+        i2 = int_le(i1, 9)
+        guard_true(i2, descr=faildescr) [i1]
+        label(i1, descr=targettoken2)
+        i3 = int_ge(i1, 0)
+        guard_true(i3, descr=faildescr3) [i1]
+        jump(i1, descr=targettoken1)
+        """, namespace=locals())
+
+        self.cpu.compile_loop(loop.inputargs, loop.operations, looptoken)
         deadframe = self.cpu.execute_token(looptoken, 2)
         fail = self.cpu.get_latest_descr(deadframe)
         assert fail.identifier == 2
         res = self.cpu.get_int_value(deadframe, 0)
         assert res == 10
 
-        inputargs2 = [i0]
-        operations2 = [
-            ResOperation(rop.INT_SUB, [i0, ConstInt(20)], i2),
-            ResOperation(rop.JUMP, [i2], None, descr=targettoken2),
-            ]
-        self.cpu.compile_bridge(faildescr, inputargs2, operations2, looptoken)
+        bridge = parse("""
+        [i0]
+        i2 = int_sub(i0, 20)
+        jump(i2, descr=targettoken2)
+        """, namespace=locals())
+
+        self.cpu.compile_bridge(faildescr, bridge.inputargs,
+                                bridge.operations, looptoken)
 
         deadframe = self.cpu.execute_token(looptoken, 2)
         fail = self.cpu.get_latest_descr(deadframe)
@@ -3870,19 +3856,17 @@
         # It catches a case in which we underestimate the needed frame_depth across
         # the cross-loop JUMP, because we estimate it based on the frame_depth stored
         # in the original loop.
-        i0 = InputArgInt()
-        i1 = InputArgInt()
         looptoken1 = JitCellToken()
         targettoken1 = TargetToken()
         faildescr1 = BasicFailDescr(2)
-        inputargs = [i0]
-        operations = [
-            ResOperation(rop.INT_LE, [i0, ConstInt(1)], i1),
-            ResOperation(rop.GUARD_TRUE, [i1], None, descr=faildescr1),
-            ResOperation(rop.FINISH, [i0], None, descr=BasicFinalDescr(1234)),
-            ]
-        operations[1].setfailargs([i0])
-        self.cpu.compile_loop(inputargs, operations, looptoken1)
+        finaldescr1 = BasicFinalDescr(1234)
+        loop = parse("""
+        [i0]
+        i1 = int_le(i0, 1)
+        guard_true(i1, descr=faildescr1) [i0]
+        finish(i0, descr=finaldescr1)
+        """, namespace=locals())
+        self.cpu.compile_loop(loop.inputargs, loop.operations, looptoken1)
 
         def func(a, b, c, d, e, f, g, h, i):
             assert a + 2 == b
@@ -3898,50 +3882,42 @@
         cpu = self.cpu
         calldescr = cpu.calldescrof(deref(FPTR), (lltype.Signed,)*9, lltype.Void,
                                     EffectInfo.MOST_GENERAL)
-        funcbox = self.get_funcbox(cpu, func_ptr)
-
-        i0 = InputArgInt(); i1 = InputArgInt(); i2 = InputArgInt(); i3 = InputArgInt(); i4 = InputArgInt()
-        i5 = InputArgInt(); i6 = InputArgInt(); i7 = InputArgInt(); i8 = InputArgInt(); i9 = InputArgInt()
-        i10 = InputArgInt(); i11 = InputArgInt(); i12 = InputArgInt(); i13 = InputArgInt(); i14 = InputArgInt()
-        i15 = InputArgInt(); i16 = InputArgInt(); i17 = InputArgInt(); i18 = InputArgInt(); i19 = InputArgInt()
-        i20 = InputArgInt()
-        inputargs = [i0]
-        operations2 = [
-            ResOperation(rop.LABEL, [i0], None, descr=targettoken1),
-            ResOperation(rop.INT_ADD, [i0, ConstInt(1)], i1),
-            ResOperation(rop.INT_ADD, [i1, ConstInt(1)], i2),
-            ResOperation(rop.INT_ADD, [i2, ConstInt(1)], i3),
-            ResOperation(rop.INT_ADD, [i3, ConstInt(1)], i4),
-            ResOperation(rop.INT_ADD, [i4, ConstInt(1)], i5),
-            ResOperation(rop.INT_ADD, [i5, ConstInt(1)], i6),
-            ResOperation(rop.INT_ADD, [i6, ConstInt(1)], i7),
-            ResOperation(rop.INT_ADD, [i7, ConstInt(1)], i8),
-            ResOperation(rop.INT_ADD, [i8, ConstInt(1)], i9),
-            ResOperation(rop.INT_ADD, [i9, ConstInt(1)], i10),
-            ResOperation(rop.INT_ADD, [i10, ConstInt(1)], i11),
-            ResOperation(rop.INT_ADD, [i11, ConstInt(1)], i12),
-            ResOperation(rop.INT_ADD, [i12, ConstInt(1)], i13),
-            ResOperation(rop.INT_ADD, [i13, ConstInt(1)], i14),
-            ResOperation(rop.INT_ADD, [i14, ConstInt(1)], i15),
-            ResOperation(rop.INT_ADD, [i15, ConstInt(1)], i16),
-            ResOperation(rop.INT_ADD, [i16, ConstInt(1)], i17),
-            ResOperation(rop.INT_ADD, [i17, ConstInt(1)], i18),
-            ResOperation(rop.INT_ADD, [i18, ConstInt(1)], i19),
-            ResOperation(rop.CALL, [funcbox, i2, i4, i6, i8, i10, i12, i14, i16, i18],
-                         None, descr=calldescr),
-            ResOperation(rop.CALL, [funcbox, i2, i4, i6, i8, i10, i12, i14, i16, i18],
-                         None, descr=calldescr),
-            ResOperation(rop.INT_LT, [i19, ConstInt(100)], i20),
-            ResOperation(rop.GUARD_TRUE, [i20], None, descr=BasicFailDescr(42)),
-            ResOperation(rop.JUMP, [i19], None, descr=targettoken1),
-            ]
-        operations2[-2].setfailargs([])
-        self.cpu.compile_bridge(faildescr1, inputargs, operations2, looptoken1)
+        faildescr=BasicFailDescr(42)
+        loop = parse("""
+        [i0]
+        label(i0, descr=targettoken1)
+        i1 = int_add(i0, 1)
+        i2 = int_add(i1, 1)
+        i3 = int_add(i2, 1)
+        i4 = int_add(i3, 1)
+        i5 = int_add(i4, 1)
+        i6 = int_add(i5, 1)
+        i7 = int_add(i6, 1)
+        i8 = int_add(i7, 1)
+        i9 = int_add(i8, 1)
+        i10 = int_add(i9, 1)
+        i11 = int_add(i10, 1)
+        i12 = int_add(i11, 1)
+        i13 = int_add(i12, 1)
+        i14 = int_add(i13, 1)
+        i15 = int_add(i14, 1)
+        i16 = int_add(i15, 1)
+        i17 = int_add(i16, 1)
+        i18 = int_add(i17, 1)
+        i19 = int_add(i18, 1)
+        call_n(ConstClass(func_ptr), i2, i4, i6, i8, i10, i12, i14, i16, i18, descr=calldescr)
+        call_n(ConstClass(func_ptr), i2, i4, i6, i8, i10, i12, i14, i16, i18, descr=calldescr)
+        i20 = int_lt(i19, 100)
+        guard_true(i20, descr=faildescr) []
+        jump(i19, descr=targettoken1)
+        """, namespace=locals())
+        self.cpu.compile_bridge(faildescr1, loop.inputargs,
+                                loop.operations, looptoken1)
 
         looptoken2 = JitCellToken()
         inputargs = [InputArgInt()]
         operations3 = [
-            ResOperation(rop.JUMP, [ConstInt(0)], None, descr=targettoken1),
+            ResOperation(rop.JUMP, [ConstInt(0)], descr=targettoken1),
             ]
         self.cpu.compile_loop(inputargs, operations3, looptoken2)
 
@@ -3954,15 +3930,15 @@
         null_box = self.null_instance()
         faildescr = BasicFailDescr(42)
         operations = [
-            ResOperation(rop.GUARD_NONNULL_CLASS, [t_box, T_box], None,
+            ResOperation(rop.GUARD_NONNULL_CLASS, [t_box, T_box],
                                                         descr=faildescr),
-            ResOperation(rop.FINISH, [], None, descr=BasicFinalDescr(1))]
+            ResOperation(rop.FINISH, [], descr=BasicFinalDescr(1))]
         operations[0].setfailargs([])
         looptoken = JitCellToken()
         inputargs = [t_box]
         self.cpu.compile_loop(inputargs, operations, looptoken)
         operations = [
-            ResOperation(rop.FINISH, [], None, descr=BasicFinalDescr(99))
+            ResOperation(rop.FINISH, [], descr=BasicFinalDescr(99))
         ]
         self.cpu.compile_bridge(faildescr, [], operations, looptoken)
         deadframe = self.cpu.execute_token(looptoken, null_box.getref_base())
@@ -3977,7 +3953,7 @@
                   rffi.ULONG, rffi.LONG]:
             ops = """
             [i0, i1]
-            i2 = raw_load(i0, i1, descr=arraydescr)
+            i2 = raw_load_i(i0, i1, descr=arraydescr)
             finish(i2)
             """
             arraydescr = self.cpu.arraydescrof(rffi.CArray(T))
@@ -4006,7 +3982,7 @@
         for T in [rffi.DOUBLE]:
             ops = """
             [i0, i1]
-            f2 = raw_load(i0, i1, descr=arraydescr)
+            f2 = raw_load_f(i0, i1, descr=arraydescr)
             finish(f2)
             """
             arraydescr = self.cpu.arraydescrof(rffi.CArray(T))
@@ -4037,7 +4013,7 @@
         for T in [rffi.FLOAT]:
             ops = """
             [i0, i1]
-            i2 = raw_load(i0, i1, descr=arraydescr)
+            i2 = raw_load_i(i0, i1, descr=arraydescr)
             finish(i2)
             """
             arraydescr = self.cpu.arraydescrof(rffi.CArray(T))
@@ -4166,26 +4142,21 @@
             values.append(self.cpu.get_int_value(deadframe, 0))
             return 42
 
-        FUNC = self.FuncType([llmemory.GCREF, lltype.Signed], lltype.Signed)
+        FUNC = self.FuncType([lltype.Signed, lltype.Signed], lltype.Signed)
         func_ptr = llhelper(lltype.Ptr(FUNC), maybe_force)
-        funcbox = self.get_funcbox(self.cpu, func_ptr).constbox()
         calldescr = self.cpu.calldescrof(FUNC, FUNC.ARGS, FUNC.RESULT,
                                          EffectInfo.MOST_GENERAL)
-        i0 = InputArgInt()
-        i1 = InputArgInt()
-        i2 = InputArgInt()
-        tok = BoxPtr()
+        finaldescr=BasicFinalDescr(0)
         faildescr = BasicFailDescr(23)
-        ops = [
-        ResOperation(rop.FORCE_TOKEN, [], tok),
-        ResOperation(rop.CALL_MAY_FORCE, [funcbox, tok, i1], i2,
-                     descr=calldescr),
-        ResOperation(rop.GUARD_NOT_FORCED, [], None, descr=faildescr),
-        ResOperation(rop.FINISH, [i2], None, descr=BasicFinalDescr(0))
-        ]
-        ops[2].setfailargs([i2])
+        loop = parse("""
+        [i0, i1]
+        i2 = force_token()
+        i3 = call_may_force_i(ConstClass(func_ptr), i2, i1, descr=calldescr)
+        guard_not_forced(descr=faildescr) [i3]
+        finish(i3, descr=finaldescr)
+        """, namespace=locals())
         looptoken = JitCellToken()
-        self.cpu.compile_loop([i0, i1], ops, looptoken)
+        self.cpu.compile_loop(loop.inputargs, loop.operations, looptoken)
         deadframe = self.cpu.execute_token(looptoken, 20, 0)
         fail = self.cpu.get_latest_descr(deadframe)
         assert fail.identifier == 23
@@ -4194,7 +4165,7 @@
     def test_compile_bridge_while_running(self):
         def func():
             bridge = parse("""
-            [i1, i2, px]
+            [i1, i2, ix]
             i3 = int_add(i1, i2)
             i4 = int_add(i1, i3)
             i5 = int_add(i1, i4)
@@ -4211,8 +4182,8 @@
             force_spill(i7)
             force_spill(i8)
             force_spill(i9)
-            call(ConstClass(func2_ptr), 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, descr=calldescr2)
-            guard_true(i1, descr=guarddescr) [i1, i2, i3, i4, i5, i6, i7, i8, i9, px]
+            call_n(ConstClass(func2_ptr), 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, descr=calldescr2)
+            guard_true(i1, descr=guarddescr) [i1, i2, i3, i4, i5, i6, i7, i8, i9, ix]
             finish(i1, descr=finaldescr)
             """, namespace={'finaldescr': finaldescr, 'calldescr2': calldescr2,
                             'guarddescr': guarddescr, 'func2_ptr': func2_ptr})
@@ -4244,9 +4215,9 @@
         looptoken = JitCellToken()
         loop = parse("""
         [i0, i1, i2]
-        call(ConstClass(func_ptr), descr=calldescr)
-        px = force_token()
-        guard_true(i0, descr=faildescr) [i1, i2, px]
+        call_n(ConstClass(func_ptr), descr=calldescr)
+        ix = force_token()
+        guard_true(i0, descr=faildescr) [i1, i2, ix]
         finish(i2, descr=finaldescr2)
         """, namespace=locals())
         self.cpu.compile_loop(loop.inputargs, loop.operations, looptoken)
@@ -4311,7 +4282,7 @@
         looptoken = JitCellToken()
         loop = parse("""
         [i0, i1, i2]
-        call(ConstClass(raising_ptr), descr=calldescr)
+        call_n(ConstClass(raising_ptr), descr=calldescr)
         guard_no_exception(descr=faildescr) [i1, i2]
         finish(i2, descr=finaldescr2)
         """, namespace={'raising_ptr': raising_ptr,
@@ -4356,7 +4327,7 @@
             py.test.skip("requires singlefloats")
         res = self.execute_operation(rop.CAST_FLOAT_TO_SINGLEFLOAT,
                                    [boxfloat(12.5)], 'int')
-        assert res.getint() == struct.unpack("I", struct.pack("f", 12.5))[0]
+        assert res == struct.unpack("I", struct.pack("f", 12.5))[0]
 
     def test_zero_ptr_field(self):
         from rpython.jit.backend.llsupport.llmodel import AbstractLLCPU
diff --git a/rpython/jit/metainterp/resoperation.py b/rpython/jit/metainterp/resoperation.py
--- a/rpython/jit/metainterp/resoperation.py
+++ b/rpython/jit/metainterp/resoperation.py
@@ -321,7 +321,7 @@
 
     getvalue = getfloatstorage
     getfloat = getfloatstorage
-
+    
     def setfloatstorage(self, floatval):
         self._resfloat = floatval
 
@@ -580,7 +580,7 @@
     'FLOAT_ABS/1/f',
     'CAST_FLOAT_TO_INT/1/i',          # don't use for unsigned ints; we would
     'CAST_INT_TO_FLOAT/1/f',          # need some messy code in the backend
-    'CAST_FLOAT_TO_SINGLEFLOAT/1/f',
+    'CAST_FLOAT_TO_SINGLEFLOAT/1/i',
     'CAST_SINGLEFLOAT_TO_FLOAT/1/f',
     'CONVERT_FLOAT_BYTES_TO_LONGLONG/1/i',
     'CONVERT_LONGLONG_BYTES_TO_FLOAT/1/f',

From noreply at buildbot.pypy.org  Wed Nov 26 12:16:56 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Wed, 26 Nov 2014 12:16:56 +0100 (CET)
Subject: [pypy-commit] pypy portable-threadlocal: fixes
Message-ID: <20141126111656.86E511C3526@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: portable-threadlocal
Changeset: r74724:9e8ef9cecb2e
Date: 2014-11-26 12:16 +0100
http://bitbucket.org/pypy/pypy/changeset/9e8ef9cecb2e/

Log:	fixes

diff --git a/pypy/module/pypyjit/test_pypy_c/test_call.py b/pypy/module/pypyjit/test_pypy_c/test_call.py
--- a/pypy/module/pypyjit/test_pypy_c/test_call.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_call.py
@@ -82,7 +82,7 @@
         assert log.opnames(ops) == []
         #
         assert entry_bridge.match_by_id('call', """
-            p38 = call(ConstClass(_ll_0_threadlocalref_getter___), descr=)
+            p38 = call(ConstClass(_ll_1_threadlocalref_get__Ptr_GcStruct_objectLlT_Signed), #, descr=)
             p39 = getfield_gc(p38, descr=)
             i40 = force_token()
             p41 = getfield_gc_pure(p38, descr=)
@@ -444,7 +444,7 @@
             p26 = getfield_gc(p7, descr=)
             guard_value(p26, ConstPtr(ptr27), descr=...)
             guard_not_invalidated(descr=...)
-            p29 = call(ConstClass(_ll_0_threadlocalref_getter___), descr=)
+            p29 = call(ConstClass(_ll_1_threadlocalref_get__Ptr_GcStruct_objectLlT_Signed), #, descr=)
             p30 = getfield_gc(p29, descr=)
             p31 = force_token()
             p32 = getfield_gc_pure(p29, descr=)

From noreply at buildbot.pypy.org  Wed Nov 26 12:39:10 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Wed, 26 Nov 2014 12:39:10 +0100 (CET)
Subject: [pypy-commit] pypy portable-threadlocal: Update the test
Message-ID: <20141126113910.A320B1C0399@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: portable-threadlocal
Changeset: r74727:250984ff4950
Date: 2014-11-26 12:38 +0100
http://bitbucket.org/pypy/pypy/changeset/250984ff4950/

Log:	Update the test

diff --git a/pypy/module/pypyjit/test_pypy_c/test_ffi.py b/pypy/module/pypyjit/test_pypy_c/test_ffi.py
--- a/pypy/module/pypyjit/test_pypy_c/test_ffi.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_ffi.py
@@ -199,21 +199,16 @@
         ldexp_addr, res = log.result
         assert res == 8.0 * 300
         loop, = log.loops_by_filename(self.filepath)
-        if 'ConstClass(ldexp)' in repr(loop):   # e.g. OS/X
-            ldexp_addr = 'ConstClass(ldexp)'
         assert loop.match_by_id('cfficall', """
-            ...
-            f1 = call_release_gil(..., descr=)
-            ...
-        """)
-        ops = loop.ops_by_id('cfficall')
-        for name in ['raw_malloc', 'raw_free']:
-            assert name not in str(ops)
-        for name in ['raw_load', 'raw_store', 'getarrayitem_raw', 'setarrayitem_raw']:
-            assert name not in log.opnames(ops)
-        # so far just check that call_release_gil() is produced.
-        # later, also check that the arguments to call_release_gil()
-        # are constants
+            setarrayitem_raw(i69, 0, i95, descr=)    # write 'errno'
+            p96 = force_token()
+            setfield_gc(p0, p96, descr=)
+            f97 = call_release_gil(i59, 1.0, 3, descr=)
+            guard_not_forced(descr=...)
+            guard_no_exception(descr=...)
+            i98 = getarrayitem_raw(i69, 0, descr=)   # read 'errno'
+            setfield_gc(p65, i98, descr=)
+        """, ignore_ops=['guard_not_invalidated'])
 
     def test_cffi_call_guard_not_forced_fails(self):
         # this is the test_pypy_c equivalent of

From noreply at buildbot.pypy.org  Wed Nov 26 12:41:55 2014
From: noreply at buildbot.pypy.org (fijal)
Date: Wed, 26 Nov 2014 12:41:55 +0100 (CET)
Subject: [pypy-commit] pypy optresult: gah, that took forever to fix
Message-ID: <20141126114155.1175D1C0399@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: optresult
Changeset: r74728:279090680681
Date: 2014-11-26 13:06 +0200
http://bitbucket.org/pypy/pypy/changeset/279090680681/

Log:	gah, that took forever to fix

diff --git a/rpython/jit/metainterp/executor.py b/rpython/jit/metainterp/executor.py
--- a/rpython/jit/metainterp/executor.py
+++ b/rpython/jit/metainterp/executor.py
@@ -479,7 +479,7 @@
 def execute_nonspec_const(cpu, metainterp, opnum, argboxes, descr=None,
                           type='i'):
     return wrap_constant(_execute_nonspec(cpu, metainterp, opnum, argboxes,
-                                          descr, type))
+                                          descr))
 
 @specialize.arg(2)
 def _execute_nonspec(cpu, metainterp, opnum, argboxes, descr=None):
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py
@@ -2421,12 +2421,12 @@
             pfieldvar = match.group(3)
             pendingfields.append((pvar, pfieldname, pfieldvar))
         #
-        def _variables_equal(value, varname, strict):
+        def _variables_equal(box, varname, strict):
             if varname not in virtuals:
                 if strict:
                     assert box.same_box(oparse.getvar(varname))
                 else:
-                    assert value == oparse.getvar(varname).getvalue()
+                    assert box.getvalue() == oparse.getvar(varname).getvalue()
             else:
                 tag, resolved, fieldstext = virtuals[varname]
                 if tag[0] == 'virtual':
@@ -2438,7 +2438,7 @@
                 else:
                     assert 0
                 if resolved is not None:
-                    assert resolved.value == box.value
+                    assert resolved.getvalue() == box.getvalue()
                 else:
                     virtuals[varname] = tag, box, fieldstext
         #
@@ -2482,7 +2482,8 @@
                                                 resolved, ConstInt(index))
                 else:
                     assert 0
-                _variables_equal(fieldval, fieldvalue.strip(), strict=False)
+                _variables_equal(executor.wrap_constant(fieldval),
+                                 fieldvalue.strip(), strict=False)
                 index += 1
 
     def check_expanded_fail_descr(self, expectedtext, guard_opnum):

From noreply at buildbot.pypy.org  Wed Nov 26 12:41:56 2014
From: noreply at buildbot.pypy.org (fijal)
Date: Wed, 26 Nov 2014 12:41:56 +0100 (CET)
Subject: [pypy-commit] pypy optresult: port some more tests and fix some
	issues
Message-ID: <20141126114156.6C62A1C0399@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: optresult
Changeset: r74729:b066f43553d4
Date: 2014-11-26 13:41 +0200
http://bitbucket.org/pypy/pypy/changeset/b066f43553d4/

Log:	port some more tests and fix some issues

diff --git a/rpython/jit/metainterp/optimizeopt/earlyforce.py b/rpython/jit/metainterp/optimizeopt/earlyforce.py
--- a/rpython/jit/metainterp/optimizeopt/earlyforce.py
+++ b/rpython/jit/metainterp/optimizeopt/earlyforce.py
@@ -4,7 +4,7 @@
 
 
 def is_raw_free(op, opnum):
-    if not op.is_call():
+    if not op.is_real_call():
         return False
     einfo = op.getdescr().get_extra_info()
     return einfo.oopspecindex == EffectInfo.OS_RAW_FREE
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py
@@ -2452,11 +2452,13 @@
         for pvar, pfieldname, pfieldvar in pendingfields:
             box = oparse.getvar(pvar)
             fielddescr = self.namespace[pfieldname.strip()]
-            fieldbox = executor.execute(self.cpu, None,
-                                        rop.GETFIELD_GC,
+            opnum = OpHelpers.getfield_for_descr(fielddescr)
+            fieldval = executor.execute(self.cpu, None,
+                                        opnum,
                                         fielddescr,
                                         box)
-            _variables_equal(fieldbox, pfieldvar, strict=True)
+            _variables_equal(executor.wrap_constant(fieldval), pfieldvar,
+                             strict=True)
         #
         for match in parts:
             pvar = match.group(1)
@@ -2476,8 +2478,9 @@
                                                 resolved)
                 elif tag[0] == 'varray':
                     fieldvalue = fieldtext
-                    fieldbox = executor.execute(self.cpu, None,
-                                                rop.GETARRAYITEM_GC,
+                    #opnum = OpHelpers.getarrayitem_for_descr(fielddescr)
+                    fieldval = executor.execute(self.cpu, None,
+                                                rop.GETARRAYITEM_GC_I,
                                                 tag[1],
                                                 resolved, ConstInt(index))
                 else:
@@ -2637,7 +2640,7 @@
         setarrayitem_gc(p1, 1, i1, descr=arraydescr)
         setarrayitem_gc(p1, 0, 25, descr=arraydescr)
         guard_true(i1, descr=fdescr) [p1]
-        i2 = getarrayitem_gc(p1, 1, descr=arraydescr)
+        i2 = getarrayitem_gc_i(p1, 1, descr=arraydescr)
         jump(i2)
         """
         expected = """
@@ -2731,7 +2734,7 @@
         jump(p1, i2, i4)
         """
         self.optimize_loop(ops, expected)
-        self.loop.inputargs[0].value = self.nodebox.value
+        self.loop.inputargs[0].setref_base(self.nodeaddr)
         self.check_expanded_fail_descr('''
             p1.nextdescr = p2
             where p2 is a node_vtable, valuedescr=i2
@@ -2765,7 +2768,7 @@
         ops = """
         [p1, p2]
         i1 = getfield_gc_i(p1, descr=valuedescr)
-        i2 = call(i1, descr=nonwritedescr)
+        i2 = call_i(i1, descr=nonwritedescr)
         i3 = getfield_gc_i(p1, descr=valuedescr)
         escape(i1)
         escape(i3)
@@ -2774,7 +2777,7 @@
         expected = """
         [p1, p2]
         i1 = getfield_gc_i(p1, descr=valuedescr)
-        i2 = call(i1, descr=nonwritedescr)
+        i2 = call_i(i1, descr=nonwritedescr)
         escape(i1)
         escape(i1)
         jump(p1, p2)
@@ -2786,7 +2789,7 @@
         [p1, p2]
         i1 = getfield_gc_i(p1, descr=adescr)
         i2 = getfield_gc_i(p1, descr=bdescr)
-        i3 = call(i1, descr=writeadescr)
+        i3 = call_i(i1, descr=writeadescr)
         i4 = getfield_gc_i(p1, descr=adescr)
         i5 = getfield_gc_i(p1, descr=bdescr)
         escape(i1)
@@ -2799,7 +2802,7 @@
         [p1, p2]
         i1 = getfield_gc_i(p1, descr=adescr)
         i2 = getfield_gc_i(p1, descr=bdescr)
-        i3 = call(i1, descr=writeadescr)
+        i3 = call_i(i1, descr=writeadescr)
         i4 = getfield_gc_i(p1, descr=adescr)
         escape(i1)
         escape(i2)
@@ -2812,11 +2815,11 @@
     def test_residual_call_invalidate_arrays(self):
         ops = """
         [p1, p2, i1]
-        p3 = getarrayitem_gc(p1, 0, descr=arraydescr2)
-        p4 = getarrayitem_gc(p2, 1, descr=arraydescr2)
-        i3 = call(i1, descr=writeadescr)
-        p5 = getarrayitem_gc(p1, 0, descr=arraydescr2)
-        p6 = getarrayitem_gc(p2, 1, descr=arraydescr2)
+        p3 = getarrayitem_gc_r(p1, 0, descr=arraydescr2)
+        p4 = getarrayitem_gc_r(p2, 1, descr=arraydescr2)
+        i3 = call_i(i1, descr=writeadescr)
+        p5 = getarrayitem_gc_r(p1, 0, descr=arraydescr2)
+        p6 = getarrayitem_gc_r(p2, 1, descr=arraydescr2)
         escape(p3)
         escape(p4)
         escape(p5)
@@ -2825,9 +2828,9 @@
         """
         expected = """
         [p1, p2, i1]
-        p3 = getarrayitem_gc(p1, 0, descr=arraydescr2)
-        p4 = getarrayitem_gc(p2, 1, descr=arraydescr2)
-        i3 = call(i1, descr=writeadescr)
+        p3 = getarrayitem_gc_r(p1, 0, descr=arraydescr2)
+        p4 = getarrayitem_gc_r(p2, 1, descr=arraydescr2)
+        i3 = call_i(i1, descr=writeadescr)
         escape(p3)
         escape(p4)
         escape(p3)
@@ -2839,13 +2842,13 @@
     def test_residual_call_invalidate_some_arrays(self):
         ops = """
         [p1, p2, i1]
-        p3 = getarrayitem_gc(p2, 0, descr=arraydescr2)
-        p4 = getarrayitem_gc(p2, 1, descr=arraydescr2)
-        i2 = getarrayitem_gc(p1, 1, descr=arraydescr)
-        i3 = call(i1, descr=writearraydescr)
-        p5 = getarrayitem_gc(p2, 0, descr=arraydescr2)
-        p6 = getarrayitem_gc(p2, 1, descr=arraydescr2)
-        i4 = getarrayitem_gc(p1, 1, descr=arraydescr)
+        p3 = getarrayitem_gc_r(p2, 0, descr=arraydescr2)
+        p4 = getarrayitem_gc_r(p2, 1, descr=arraydescr2)
+        i2 = getarrayitem_gc_i(p1, 1, descr=arraydescr)
+        i3 = call_i(i1, descr=writearraydescr)
+        p5 = getarrayitem_gc_r(p2, 0, descr=arraydescr2)
+        p6 = getarrayitem_gc_r(p2, 1, descr=arraydescr2)
+        i4 = getarrayitem_gc_i(p1, 1, descr=arraydescr)
         escape(p3)
         escape(p4)
         escape(p5)
@@ -2856,11 +2859,11 @@
         """
         expected = """
         [p1, p2, i1]
-        p3 = getarrayitem_gc(p2, 0, descr=arraydescr2)
-        p4 = getarrayitem_gc(p2, 1, descr=arraydescr2)
-        i2 = getarrayitem_gc(p1, 1, descr=arraydescr)
-        i3 = call(i1, descr=writearraydescr)
-        i4 = getarrayitem_gc(p1, 1, descr=arraydescr)
+        p3 = getarrayitem_gc_r(p2, 0, descr=arraydescr2)
+        p4 = getarrayitem_gc_r(p2, 1, descr=arraydescr2)
+        i2 = getarrayitem_gc_i(p1, 1, descr=arraydescr)
+        i3 = call_i(i1, descr=writearraydescr)
+        i4 = getarrayitem_gc_i(p1, 1, descr=arraydescr)
         escape(p3)
         escape(p4)
         escape(p3)
@@ -2876,7 +2879,7 @@
         [p1, i1, p2, i2]
         setfield_gc(p1, i1, descr=valuedescr)
         setfield_gc(p2, i2, descr=adescr)
-        i3 = call(i1, descr=readadescr)
+        i3 = call_i(i1, descr=readadescr)
         setfield_gc(p1, i3, descr=valuedescr)
         setfield_gc(p2, i3, descr=adescr)
         jump(p1, i1, p2, i2)
@@ -2884,7 +2887,7 @@
         expected = """
         [p1, i1, p2, i2]
         setfield_gc(p2, i2, descr=adescr)
-        i3 = call(i1, descr=readadescr)
+        i3 = call_i(i1, descr=readadescr)
         setfield_gc(p1, i3, descr=valuedescr)
         setfield_gc(p2, i3, descr=adescr)
         jump(p1, i1, p2, i2)
@@ -2896,7 +2899,7 @@
         [p1, i1, p2, i2]
         setfield_gc(p1, i1, descr=valuedescr)
         setfield_gc(p2, i2, descr=adescr)
-        i3 = call(i1, descr=writeadescr)
+        i3 = call_i(i1, descr=writeadescr)
         setfield_gc(p1, i3, descr=valuedescr)
         setfield_gc(p2, i3, descr=adescr)
         jump(p1, i1, p2, i2)
@@ -2904,7 +2907,7 @@
         expected = """
         [p1, i1, p2, i2]
         setfield_gc(p2, i2, descr=adescr)
-        i3 = call(i1, descr=writeadescr)
+        i3 = call_i(i1, descr=writeadescr)
         setfield_gc(p1, i3, descr=valuedescr)
         setfield_gc(p2, i3, descr=adescr)
         jump(p1, i1, p2, i2)
@@ -2916,7 +2919,7 @@
         [p1, i1, p2, i2]
         setfield_gc(p1, i1, descr=valuedescr)
         setfield_gc(p2, i2, descr=adescr)
-        i3 = call(i1, descr=plaincalldescr)
+        i3 = call_i(i1, descr=plaincalldescr)
         setfield_gc(p1, i3, descr=valuedescr)
         setfield_gc(p2, i3, descr=adescr)
         jump(p1, i1, p2, i2)
@@ -2927,7 +2930,7 @@
         ops = '''
         [p1, i1]
         setfield_gc(p1, i1, descr=valuedescr)
-        i3 = call_assembler(i1, descr=asmdescr)
+        i3 = call_assembler_i(i1, descr=asmdescr)
         setfield_gc(p1, i3, descr=valuedescr)
         jump(p1, i3)
         '''
@@ -2938,14 +2941,14 @@
         ops = '''
         [p1, i1]
         setfield_gc(p1, i1, descr=valuedescr)
-        i3 = call_pure(p1, descr=plaincalldescr)
+        i3 = call_pure_i(p1, descr=plaincalldescr)
         setfield_gc(p1, i3, descr=valuedescr)
         jump(p1, i3)
         '''
         expected = '''
         [p1, i1]
         setfield_gc(p1, i1, descr=valuedescr)
-        i3 = call(p1, descr=plaincalldescr)
+        i3 = call_i(p1, descr=plaincalldescr)
         setfield_gc(p1, i3, descr=valuedescr)
         jump(p1, i3)
         '''
@@ -2963,15 +2966,15 @@
         [i0, i1, i2]
         escape(i1)
         escape(i2)
-        i3 = call_pure(123456, 4, 5, 6, descr=plaincalldescr)
-        i4 = call_pure(123456, 4, i0, 6, descr=plaincalldescr)
+        i3 = call_pure_i(123456, 4, 5, 6, descr=plaincalldescr)
+        i4 = call_pure_i(123456, 4, i0, 6, descr=plaincalldescr)
         jump(i0, i3, i4)
         '''
         expected = '''
         [i0, i1, i2]
         escape(i1)
         escape(i2)
-        i4 = call(123456, 4, i0, 6, descr=plaincalldescr)
+        i4 = call_i(123456, 4, i0, 6, descr=plaincalldescr)
         jump(i0, 42, i4)
         '''
         self.optimize_loop(ops, expected, call_pure_results)
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_util.py b/rpython/jit/metainterp/optimizeopt/test/test_util.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_util.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_util.py
@@ -17,7 +17,7 @@
 from rpython.jit.metainterp.jitprof import EmptyProfiler
 from rpython.jit.metainterp.counter import DeterministicJitCounter
 from rpython.config.translationoption import get_combined_translation_config
-from rpython.jit.metainterp.resoperation import rop, ResOperation
+from rpython.jit.metainterp.resoperation import rop, ResOperation, InputArgRef
 from rpython.jit.metainterp.optimizeopt.unroll import Inliner
 
 def test_sort_descrs():
@@ -98,12 +98,14 @@
                                      ('other', lltype.Ptr(NODE)))
     node = lltype.malloc(NODE)
     node.parent.typeptr = node_vtable
+    nodeaddr = lltype.cast_opaque_ptr(llmemory.GCREF, node)
+    #nodebox = InputArgRef(lltype.cast_opaque_ptr(llmemory.GCREF, node))
     node2 = lltype.malloc(NODE2)
     node2.parent.parent.typeptr = node_vtable2
     myptr = lltype.cast_opaque_ptr(llmemory.GCREF, node)
     myptr2 = lltype.cast_opaque_ptr(llmemory.GCREF, lltype.malloc(NODE))
     nullptr = lltype.nullptr(llmemory.GCREF.TO)
-    #nodebox2 = BoxPtr(lltype.cast_opaque_ptr(llmemory.GCREF, node2))
+    #nodebox2 = InputArgRef(lltype.cast_opaque_ptr(llmemory.GCREF, node2))
     nodesize = cpu.sizeof(NODE)
     nodesize2 = cpu.sizeof(NODE2)
     valuedescr = cpu.fielddescrof(NODE, 'value')
diff --git a/rpython/jit/metainterp/optimizeopt/virtualize.py b/rpython/jit/metainterp/optimizeopt/virtualize.py
--- a/rpython/jit/metainterp/optimizeopt/virtualize.py
+++ b/rpython/jit/metainterp/optimizeopt/virtualize.py
@@ -255,7 +255,7 @@
             else:
                 box = None
             itemboxes.append(box)
-        visitor.register_virtual_fields(self.keybox, itemboxes)
+        visitor.register_virtual_fields(self.source_op, itemboxes)
         for i in range(self.getlength()):
             itemvalue = self.get_item_value(i)
             if itemvalue is not None:
diff --git a/rpython/jit/metainterp/resoperation.py b/rpython/jit/metainterp/resoperation.py
--- a/rpython/jit/metainterp/resoperation.py
+++ b/rpython/jit/metainterp/resoperation.py
@@ -194,6 +194,13 @@
     def is_call(self):
         return rop._CALL_FIRST <= self.getopnum() <= rop._CALL_LAST
 
+    def is_real_call(self):
+        opnum = self.opnum
+        return (opnum == rop.CALL_I or
+                opnum == rop.CALL_R or
+                opnum == rop.CALL_F or
+                opnum == rop.CALL_N)
+
     def is_call_assembler(self):
         opnum = self.opnum
         return (opnum == rop.CALL_ASSEMBLER_I or
@@ -935,6 +942,14 @@
         return rop.GETARRAYITEM_GC_PURE_I
 
     @staticmethod
+    def getarrayitem_for_descr(descr):
+        if descr.is_array_of_pointers():
+            return rop.GETARRAYITEM_GC_R
+        elif descr.is_array_of_floats():
+            return rop.GETARRAYITEM_GC_F
+        return rop.GETARRAYITEM_GC_I
+
+    @staticmethod
     def same_as_for_type(tp):
         if tp == 'i':
             return rop.SAME_AS_I
diff --git a/rpython/jit/metainterp/resume.py b/rpython/jit/metainterp/resume.py
--- a/rpython/jit/metainterp/resume.py
+++ b/rpython/jit/metainterp/resume.py
@@ -348,8 +348,9 @@
         self._register_boxes(fieldboxes)
 
     def register_box(self, box):
-        if (not isinstance(box, Const) and box not in self.liveboxes_from_env
-                                 and box not in self.liveboxes):
+        if (box is not None and not isinstance(box, Const)
+            and box not in self.liveboxes_from_env
+            and box not in self.liveboxes):
             self.liveboxes[box] = UNASSIGNED
 
     def _register_boxes(self, boxes):

From noreply at buildbot.pypy.org  Wed Nov 26 12:48:43 2014
From: noreply at buildbot.pypy.org (rlamy)
Date: Wed, 26 Nov 2014 12:48:43 +0100 (CET)
Subject: [pypy-commit] pypy framestate: Use (bc_block,
	index-in-block) as position
Message-ID: <20141126114843.888E11C0EF6@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy 
Branch: framestate
Changeset: r74730:84a74ea27d86
Date: 2014-11-25 19:21 +0100
http://bitbucket.org/pypy/pypy/changeset/84a74ea27d86/

Log:	Use (bc_block, index-in-block) as position

diff --git a/rpython/flowspace/bytecode.py b/rpython/flowspace/bytecode.py
--- a/rpython/flowspace/bytecode.py
+++ b/rpython/flowspace/bytecode.py
@@ -245,10 +245,11 @@
             return block.operations[i].offset
 
     def get_position(self, offset):
-        return offset
+        return self.pos_index[offset]
 
     def get_offset(self, position):
-        return position
+        block, i = position
+        return block[i].offset
 
     def dump(self):
         all_blocks = set(x[0] for x in self.pos_index.values())
diff --git a/rpython/flowspace/pygraph.py b/rpython/flowspace/pygraph.py
--- a/rpython/flowspace/pygraph.py
+++ b/rpython/flowspace/pygraph.py
@@ -1,7 +1,7 @@
 """
 Implements flow graphs for Python callables
 """
-from rpython.flowspace.model import FunctionGraph, Constant, Variable
+from rpython.flowspace.model import FunctionGraph, Variable
 from rpython.flowspace.framestate import FrameState
 
 class PyGraph(FunctionGraph):
@@ -14,7 +14,9 @@
         locals = [None] * code.co_nlocals
         for i in range(code.formalargcount):
             locals[i] = Variable(code.co_varnames[i])
-        state = FrameState(locals, [], None, [], 0)
+        bc_graph = code.graph
+        start_pos = bc_graph.entry._exits[0], 0
+        state = FrameState(locals, [], None, [], start_pos)
         initialblock = SpamBlock(state)
         super(PyGraph, self).__init__(self._sanitize_funcname(func), initialblock)
         self.func = func

From noreply at buildbot.pypy.org  Wed Nov 26 12:48:44 2014
From: noreply at buildbot.pypy.org (rlamy)
Date: Wed, 26 Nov 2014 12:48:44 +0100 (CET)
Subject: [pypy-commit] pypy framestate: create bc_graph.iter_instr()
Message-ID: <20141126114844.B5A601C0EF6@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy 
Branch: framestate
Changeset: r74731:dc288d5b57c8
Date: 2014-11-26 00:57 +0100
http://bitbucket.org/pypy/pypy/changeset/dc288d5b57c8/

Log:	create bc_graph.iter_instr()

diff --git a/rpython/flowspace/bytecode.py b/rpython/flowspace/bytecode.py
--- a/rpython/flowspace/bytecode.py
+++ b/rpython/flowspace/bytecode.py
@@ -251,6 +251,12 @@
         block, i = position
         return block[i].offset
 
+    def iter_instr(self):
+        while True:
+            offset = self.get_offset(self.curr_position)
+            instr = self.read(offset)
+            yield instr
+
     def dump(self):
         all_blocks = set(x[0] for x in self.pos_index.values())
         blocks = sorted(all_blocks, key=lambda b: b.startpos)
diff --git a/rpython/flowspace/flowcontext.py b/rpython/flowspace/flowcontext.py
--- a/rpython/flowspace/flowcontext.py
+++ b/rpython/flowspace/flowcontext.py
@@ -350,13 +350,13 @@
         self.setstate(block.framestate)
         self.recorder = block.make_recorder()
         bc_graph = self.pycode.graph
-        next_offset = bc_graph.get_offset(block.framestate.position)
+        bc_graph.curr_position = block.framestate.position
         try:
-            while True:
-                instr = bc_graph.read(next_offset)
+            for instr in bc_graph.iter_instr():
                 self.last_offset = instr.offset
                 next_offset = self.handle_bytecode(instr)
                 position = bc_graph.get_position(next_offset)
+                bc_graph.curr_position = position
                 self.recorder.final_state = self.getstate(position)
 
         except RaiseImplicit as e:

From noreply at buildbot.pypy.org  Wed Nov 26 14:26:41 2014
From: noreply at buildbot.pypy.org (rlamy)
Date: Wed, 26 Nov 2014 14:26:41 +0100 (CET)
Subject: [pypy-commit] pypy framestate: move FOR_ITER
Message-ID: <20141126132641.ECA731C07EE@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy 
Branch: framestate
Changeset: r74732:b8afe00c2ad2
Date: 2014-11-26 14:03 +0100
http://bitbucket.org/pypy/pypy/changeset/b8afe00c2ad2/

Log:	move FOR_ITER

diff --git a/rpython/flowspace/bytecode.py b/rpython/flowspace/bytecode.py
--- a/rpython/flowspace/bytecode.py
+++ b/rpython/flowspace/bytecode.py
@@ -476,6 +476,22 @@
     reader.end_block()
 
 @bc_reader.register_opcode
+class FOR_ITER(BCInstruction):
+    def eval(self, ctx):
+        from rpython.flowspace.flowcontext import Raise
+        w_iterator = ctx.peekvalue()
+        try:
+            w_nextitem = op.next(w_iterator).eval(ctx)
+            ctx.pushvalue(w_nextitem)
+        except Raise as e:
+            if ctx.exception_match(e.w_exc.w_type, const(StopIteration)):
+                ctx.popvalue()
+                return self.arg
+            else:
+                raise
+
+
+ at bc_reader.register_opcode
 class SETUP_EXCEPT(BCInstruction):
     def eval(self, ctx):
         from rpython.flowspace.flowcontext import ExceptBlock
diff --git a/rpython/flowspace/flowcontext.py b/rpython/flowspace/flowcontext.py
--- a/rpython/flowspace/flowcontext.py
+++ b/rpython/flowspace/flowcontext.py
@@ -708,18 +708,6 @@
         w_iterator = op.iter(w_iterable).eval(self)
         self.pushvalue(w_iterator)
 
-    def FOR_ITER(self, target):
-        w_iterator = self.peekvalue()
-        try:
-            w_nextitem = op.next(w_iterator).eval(self)
-            self.pushvalue(w_nextitem)
-        except Raise as e:
-            if self.exception_match(e.w_exc.w_type, const(StopIteration)):
-                self.popvalue()
-                return target
-            else:
-                raise
-
     def SETUP_LOOP(self, target):
         block = LoopBlock(self.stackdepth, target)
         self.blockstack.append(block)

From noreply at buildbot.pypy.org  Wed Nov 26 14:26:43 2014
From: noreply at buildbot.pypy.org (rlamy)
Date: Wed, 26 Nov 2014 14:26:43 +0100 (CET)
Subject: [pypy-commit] pypy framestate: return blocks, not offsets,
	from FOR_ITER.eval()
Message-ID: <20141126132643.24DB81C07EE@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy 
Branch: framestate
Changeset: r74733:be6505ef80a3
Date: 2014-11-26 14:11 +0100
http://bitbucket.org/pypy/pypy/changeset/be6505ef80a3/

Log:	return blocks, not offsets, from FOR_ITER.eval()

diff --git a/rpython/flowspace/bytecode.py b/rpython/flowspace/bytecode.py
--- a/rpython/flowspace/bytecode.py
+++ b/rpython/flowspace/bytecode.py
@@ -477,16 +477,25 @@
 
 @bc_reader.register_opcode
 class FOR_ITER(BCInstruction):
+    def bc_flow(self, reader):
+        block = reader.curr_block
+        block.operations.append(self)
+        self.exit = reader.get_block_at(self.arg)
+        self.body = reader.get_next_block()
+        block.set_exits([self.body, self.exit])
+        reader.end_block()
+
     def eval(self, ctx):
         from rpython.flowspace.flowcontext import Raise
         w_iterator = ctx.peekvalue()
         try:
             w_nextitem = op.next(w_iterator).eval(ctx)
             ctx.pushvalue(w_nextitem)
+            return self.body
         except Raise as e:
             if ctx.exception_match(e.w_exc.w_type, const(StopIteration)):
                 ctx.popvalue()
-                return self.arg
+                return self.exit
             else:
                 raise
 

From noreply at buildbot.pypy.org  Wed Nov 26 14:26:44 2014
From: noreply at buildbot.pypy.org (rlamy)
Date: Wed, 26 Nov 2014 14:26:44 +0100 (CET)
Subject: [pypy-commit] pypy framestate: return positions from
	handle_bytecode()
Message-ID: <20141126132644.618451C07EE@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy 
Branch: framestate
Changeset: r74734:d5cab6a66583
Date: 2014-11-26 14:25 +0100
http://bitbucket.org/pypy/pypy/changeset/d5cab6a66583/

Log:	return positions from handle_bytecode()

diff --git a/rpython/flowspace/bytecode.py b/rpython/flowspace/bytecode.py
--- a/rpython/flowspace/bytecode.py
+++ b/rpython/flowspace/bytecode.py
@@ -234,15 +234,15 @@
         bc_block, i = self.pos_index[pos]
         return bc_block[i]
 
-    def next_pos(self, instr):
-        block, i = self.pos_index[instr.offset]
+    def next_pos(self):
+        block, i = self.curr_position
         i = i + 1
         if i >= len(block.operations):
             assert len(block._exits) == 1
             assert block._exits[0] is not block
-            return block._exits[0].startpos
+            return (block._exits[0], 0)
         else:
-            return block.operations[i].offset
+            return block, i
 
     def get_position(self, offset):
         return self.pos_index[offset]
diff --git a/rpython/flowspace/flowcontext.py b/rpython/flowspace/flowcontext.py
--- a/rpython/flowspace/flowcontext.py
+++ b/rpython/flowspace/flowcontext.py
@@ -354,8 +354,7 @@
         try:
             for instr in bc_graph.iter_instr():
                 self.last_offset = instr.offset
-                next_offset = self.handle_bytecode(instr)
-                position = bc_graph.get_position(next_offset)
+                position = self.handle_bytecode(instr)
                 bc_graph.curr_position = position
                 self.recorder.final_state = self.getstate(position)
 
@@ -459,11 +458,12 @@
         try:
             next_offset = instr.eval(self)
         except FlowSignal as signal:
-            return self.unroll(signal)
+            return bc_graph.get_position(self.unroll(signal))
         if next_offset is None:
-            next_offset = bc_graph.next_pos(instr)
-        elif isinstance(next_offset, BytecodeBlock):
-            next_offset = next_offset.startpos
+            next_offset = bc_graph.next_pos()
+        else:
+            assert isinstance(next_offset, BytecodeBlock)
+            next_offset = next_offset, 0
         return next_offset
 
     def unroll(self, signal):

From noreply at buildbot.pypy.org  Wed Nov 26 15:00:28 2014
From: noreply at buildbot.pypy.org (rlamy)
Date: Wed, 26 Nov 2014 15:00:28 +0100 (CET)
Subject: [pypy-commit] pypy framestate: inline handle_bytecode()
Message-ID: <20141126140028.C6F031D2961@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy 
Branch: framestate
Changeset: r74735:d1f5bf302728
Date: 2014-11-26 14:59 +0100
http://bitbucket.org/pypy/pypy/changeset/d1f5bf302728/

Log:	inline handle_bytecode()

diff --git a/rpython/flowspace/flowcontext.py b/rpython/flowspace/flowcontext.py
--- a/rpython/flowspace/flowcontext.py
+++ b/rpython/flowspace/flowcontext.py
@@ -15,8 +15,7 @@
 from rpython.flowspace.specialcase import (rpython_print_item,
     rpython_print_newline)
 from rpython.flowspace.operation import op
-from rpython.flowspace.bytecode import (
-    BytecodeBlock, BytecodeCorruption, bc_reader)
+from rpython.flowspace.bytecode import BytecodeCorruption, bc_reader
 from rpython.flowspace.pygraph import PyGraph
 
 w_None = const(None)
@@ -354,9 +353,17 @@
         try:
             for instr in bc_graph.iter_instr():
                 self.last_offset = instr.offset
-                position = self.handle_bytecode(instr)
-                bc_graph.curr_position = position
-                self.recorder.final_state = self.getstate(position)
+                try:
+                    next_block = instr.eval(self)
+                except FlowSignal as signal:
+                    next_position = bc_graph.get_position(self.unroll(signal))
+                else:
+                    if next_block is None:
+                        next_position = bc_graph.next_pos()
+                    else:
+                        next_position = next_block, 0
+                bc_graph.curr_position = next_position
+                self.recorder.final_state = self.getstate(next_position)
 
         except RaiseImplicit as e:
             w_exc = e.w_exc
@@ -453,19 +460,6 @@
                     stack_items_w[i] = w_new
                     break
 
-    def handle_bytecode(self, instr):
-        bc_graph = self.pycode.graph
-        try:
-            next_offset = instr.eval(self)
-        except FlowSignal as signal:
-            return bc_graph.get_position(self.unroll(signal))
-        if next_offset is None:
-            next_offset = bc_graph.next_pos()
-        else:
-            assert isinstance(next_offset, BytecodeBlock)
-            next_offset = next_offset, 0
-        return next_offset
-
     def unroll(self, signal):
         while self.blockstack:
             block = self.blockstack.pop()

From noreply at buildbot.pypy.org  Wed Nov 26 15:44:24 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Wed, 26 Nov 2014 15:44:24 +0100 (CET)
Subject: [pypy-commit] pypy portable-threadlocal: test and fix for the
	no-__thread case
Message-ID: <20141126144424.1D6EA1C3526@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: portable-threadlocal
Changeset: r74736:31fa1fdc28a7
Date: 2014-11-26 15:44 +0100
http://bitbucket.org/pypy/pypy/changeset/31fa1fdc28a7/

Log:	test and fix for the no-__thread case

diff --git a/rpython/translator/c/src/threadlocal.c b/rpython/translator/c/src/threadlocal.c
--- a/rpython/translator/c/src/threadlocal.c
+++ b/rpython/translator/c/src/threadlocal.c
@@ -69,6 +69,9 @@
    explicitly, with malloc()/free(), and attached to (a single) thread-
    local key using the API of Windows or pthread. */
 
+pthread_key_t pypy_threadlocal_key;
+
+
 void RPython_ThreadLocals_ProgramInit(void)
 {
 #ifdef _WIN32
diff --git a/rpython/translator/c/src/threadlocal.h b/rpython/translator/c/src/threadlocal.h
--- a/rpython/translator/c/src/threadlocal.h
+++ b/rpython/translator/c/src/threadlocal.h
@@ -55,15 +55,16 @@
 #ifdef _WIN32
 #  include 
 #  include 
-#  define _RPy_ThreadLocals_Get  TlsGetValue
-#  define _RPy_ThreadLocals_Set  TlsSetValue
-RPY_EXTERN DWORD pypy_threadlocal_key;
+#  define _RPy_ThreadLocals_Get()   TlsGetValue(pypy_threadlocal_key)
+#  define _RPy_ThreadLocals_Set(x)  TlsSetValue(pypy_threadlocal_key, x)
+typedef DWORD pthread_key_t;
 #else
 #  include 
-#  define _RPy_ThreadLocals_Get  pthread_getspecific
-#  define _RPy_ThreadLocals_Set  pthread_setspecific
+#  define _RPy_ThreadLocals_Get()   pthread_getspecific(pypy_threadlocal_key)
+#  define _RPy_ThreadLocals_Set(x)  pthread_setspecific(pypy_threadlocal_key, x)
+#endif
+
 RPY_EXTERN pthread_key_t pypy_threadlocal_key;
-#endif
 
 
 #define OP_THREADLOCALREF_ADDR(r)               \
diff --git a/rpython/translator/c/test/test_standalone.py b/rpython/translator/c/test/test_standalone.py
--- a/rpython/translator/c/test/test_standalone.py
+++ b/rpython/translator/c/test/test_standalone.py
@@ -2,6 +2,7 @@
 import sys, os, re
 
 from rpython.config.translationoption import get_combined_translation_config
+from rpython.config.translationoption import SUPPORT__THREAD
 from rpython.rlib.objectmodel import keepalive_until_here
 from rpython.rlib.rarithmetic import r_longlong
 from rpython.rlib.debug import ll_assert, have_debug_prints, debug_flush
@@ -1026,11 +1027,12 @@
     gcrootfinder = 'shadowstack'
     config = None
 
-    def compile(self, entry_point):
+    def compile(self, entry_point, no__thread=True):
         t = TranslationContext(self.config)
         t.config.translation.gc = "semispace"
         t.config.translation.gcrootfinder = self.gcrootfinder
         t.config.translation.thread = True
+        t.config.translation.no__thread = no__thread
         t.buildannotator().build_types(entry_point, [s_list_of_strings])
         t.buildrtyper().specialize()
         #
@@ -1142,7 +1144,7 @@
 
     def test_thread_and_gc(self):
         import time, gc
-        from rpython.rlib import rthread
+        from rpython.rlib import rthread, rposix
         from rpython.rtyper.lltypesystem import lltype
         from rpython.rlib.objectmodel import invoke_around_extcall
 
@@ -1163,14 +1165,22 @@
                 self.head = head
                 self.tail = tail
 
+        def check_errno(value):
+            rposix.set_errno(value)
+            for i in range(10000000):
+                pass
+            assert rposix.get_errno() == value
+
         def bootstrap():
             rthread.gc_thread_start()
+            check_errno(42)
             state.xlist.append(Cons(123, Cons(456, None)))
             gc.collect()
             rthread.gc_thread_die()
 
         def new_thread():
             ident = rthread.start_new_thread(bootstrap, ())
+            check_errno(41)
             time.sleep(0.5)    # enough time to start, hopefully
             return ident
 
@@ -1209,14 +1219,19 @@
                 os.write(1, "%d ok\n" % (i+1))
             return 0
 
-        t, cbuilder = self.compile(entry_point)
-        data = cbuilder.cmdexec('')
-        assert data.splitlines() == ['hello world',
-                                     '1 ok',
-                                     '2 ok',
-                                     '3 ok',
-                                     '4 ok',
-                                     '5 ok']
+        def runme(no__thread):
+            t, cbuilder = self.compile(entry_point, no__thread=no__thread)
+            data = cbuilder.cmdexec('')
+            assert data.splitlines() == ['hello world',
+                                         '1 ok',
+                                         '2 ok',
+                                         '3 ok',
+                                         '4 ok',
+                                         '5 ok']
+
+        if SUPPORT__THREAD:
+            runme(no__thread=False)
+        runme(no__thread=True)
 
 
     def test_gc_with_fork_without_threads(self):

From noreply at buildbot.pypy.org  Wed Nov 26 15:51:21 2014
From: noreply at buildbot.pypy.org (fijal)
Date: Wed, 26 Nov 2014 15:51:21 +0100 (CET)
Subject: [pypy-commit] pypy optresult: FORCE_TOKEN returns a GCREF.
 otherwise just whack whack until more tests pass
Message-ID: <20141126145121.27BFC1C3526@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: optresult
Changeset: r74737:4518e77297b1
Date: 2014-11-26 16:32 +0200
http://bitbucket.org/pypy/pypy/changeset/4518e77297b1/

Log:	FORCE_TOKEN returns a GCREF. otherwise just whack whack until more
	tests pass

diff --git a/rpython/jit/backend/llgraph/runner.py b/rpython/jit/backend/llgraph/runner.py
--- a/rpython/jit/backend/llgraph/runner.py
+++ b/rpython/jit/backend/llgraph/runner.py
@@ -679,7 +679,7 @@
 
 
 class LLFrame(object):
-    _TYPE = lltype.Signed
+    _TYPE = llmemory.GCREF
 
     forced_deadframe = None
     overflow_flag = False
diff --git a/rpython/jit/backend/test/runner_test.py b/rpython/jit/backend/test/runner_test.py
--- a/rpython/jit/backend/test/runner_test.py
+++ b/rpython/jit/backend/test/runner_test.py
@@ -2272,7 +2272,7 @@
                 values.append(self.cpu.get_int_value(deadframe, 1))
                 self.cpu.set_savedata_ref(deadframe, random_gcref)
 
-        FUNC = self.FuncType([lltype.Signed, lltype.Signed], lltype.Void)
+        FUNC = self.FuncType([llmemory.GCREF, lltype.Signed], lltype.Void)
         func_ptr = llhelper(lltype.Ptr(FUNC), maybe_force)
         calldescr = self.cpu.calldescrof(FUNC, FUNC.ARGS, FUNC.RESULT,
                                          EffectInfo.MOST_GENERAL)
@@ -2281,8 +2281,8 @@
         finaldescr = BasicFinalDescr(0)
         loop = parse("""
         [i0, i1]
-        i2 = force_token()
-        call_may_force_n(ConstClass(func_ptr), i2, i1, descr=calldescr)
+        p2 = force_token()
+        call_may_force_n(ConstClass(func_ptr), p2, i1, descr=calldescr)
         guard_not_forced(descr=faildescr) [i1, i0]
         finish(i0, descr=finaldescr)
         """, namespace=locals())
@@ -2313,7 +2313,7 @@
                 self.cpu.set_savedata_ref(deadframe, random_gcref)
             return 42
 
-        FUNC = self.FuncType([lltype.Signed, lltype.Signed], lltype.Signed)
+        FUNC = self.FuncType([llmemory.GCREF, lltype.Signed], lltype.Signed)
         func_ptr = llhelper(lltype.Ptr(FUNC), maybe_force)
         calldescr = self.cpu.calldescrof(FUNC, FUNC.ARGS, FUNC.RESULT,
                                          EffectInfo.MOST_GENERAL)
@@ -2322,8 +2322,8 @@
         finaldescr = BasicFinalDescr(0)
         loop = parse("""
         [i0, i1]
-        i3 = force_token()
-        i2 = call_may_force_i(ConstClass(func_ptr), i3, i1, descr=calldescr)
+        p3 = force_token()
+        i2 = call_may_force_i(ConstClass(func_ptr), p3, i1, descr=calldescr)
         guard_not_forced(descr=faildescr) [i1, i2, i0]
         finish(i2, descr=finaldescr)
         """, namespace=locals())
@@ -2356,7 +2356,7 @@
                 self.cpu.set_savedata_ref(deadframe, random_gcref)
             return 42.5
 
-        FUNC = self.FuncType([lltype.Signed, lltype.Signed], lltype.Float)
+        FUNC = self.FuncType([llmemory.GCREF, lltype.Signed], lltype.Float)
         func_ptr = llhelper(lltype.Ptr(FUNC), maybe_force)
         funcbox = self.get_funcbox(self.cpu, func_ptr).constbox()
         calldescr = self.cpu.calldescrof(FUNC, FUNC.ARGS, FUNC.RESULT,
@@ -2366,8 +2366,8 @@
         finaldescr = BasicFinalDescr(0)
         loop = parse("""
         [i0, i1]
-        i3 = force_token()
-        f2 = call_may_force_f(ConstClass(func_ptr), i3, i1, descr=calldescr)
+        p3 = force_token()
+        f2 = call_may_force_f(ConstClass(func_ptr), p3, i1, descr=calldescr)
         guard_not_forced(descr=faildescr) [i1, f2, i0]
         finish(f2, descr=finaldescr)
         """, namespace=locals())
@@ -2397,16 +2397,16 @@
         loop = parse("""
         [i0]
         i1 = int_add(i0, 10)
-        i2 = force_token()
+        p2 = force_token()
         guard_not_forced_2(descr=faildescr) [i1]
-        finish(i2, descr=finaldescr)
+        finish(p2, descr=finaldescr)
         """, namespace=locals())
         looptoken = JitCellToken()
         self.cpu.compile_loop(loop.inputargs, loop.operations, looptoken)
         deadframe = self.cpu.execute_token(looptoken, 20)
         fail = self.cpu.get_latest_descr(deadframe)
         assert fail.identifier == 0
-        frame = self.cpu.get_int_value(deadframe, 0)
+        frame = self.cpu.get_ref_value(deadframe, 0)
         # actually, we should get the same pointer in 'frame' and 'deadframe'
         # but it is not the case on LLGraph
         if not getattr(self.cpu, 'is_llgraph', False):
@@ -4142,7 +4142,7 @@
             values.append(self.cpu.get_int_value(deadframe, 0))
             return 42
 
-        FUNC = self.FuncType([lltype.Signed, lltype.Signed], lltype.Signed)
+        FUNC = self.FuncType([llmemory.GCREF, lltype.Signed], lltype.Signed)
         func_ptr = llhelper(lltype.Ptr(FUNC), maybe_force)
         calldescr = self.cpu.calldescrof(FUNC, FUNC.ARGS, FUNC.RESULT,
                                          EffectInfo.MOST_GENERAL)
@@ -4150,8 +4150,8 @@
         faildescr = BasicFailDescr(23)
         loop = parse("""
         [i0, i1]
-        i2 = force_token()
-        i3 = call_may_force_i(ConstClass(func_ptr), i2, i1, descr=calldescr)
+        p2 = force_token()
+        i3 = call_may_force_i(ConstClass(func_ptr), p2, i1, descr=calldescr)
         guard_not_forced(descr=faildescr) [i3]
         finish(i3, descr=finaldescr)
         """, namespace=locals())
@@ -4165,7 +4165,7 @@
     def test_compile_bridge_while_running(self):
         def func():
             bridge = parse("""
-            [i1, i2, ix]
+            [i1, i2, px]
             i3 = int_add(i1, i2)
             i4 = int_add(i1, i3)
             i5 = int_add(i1, i4)
@@ -4183,7 +4183,7 @@
             force_spill(i8)
             force_spill(i9)
             call_n(ConstClass(func2_ptr), 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, descr=calldescr2)
-            guard_true(i1, descr=guarddescr) [i1, i2, i3, i4, i5, i6, i7, i8, i9, ix]
+            guard_true(i1, descr=guarddescr) [i1, i2, i3, i4, i5, i6, i7, i8, i9, px]
             finish(i1, descr=finaldescr)
             """, namespace={'finaldescr': finaldescr, 'calldescr2': calldescr2,
                             'guarddescr': guarddescr, 'func2_ptr': func2_ptr})
@@ -4216,8 +4216,8 @@
         loop = parse("""
         [i0, i1, i2]
         call_n(ConstClass(func_ptr), descr=calldescr)
-        ix = force_token()
-        guard_true(i0, descr=faildescr) [i1, i2, ix]
+        px = force_token()
+        guard_true(i0, descr=faildescr) [i1, i2, px]
         finish(i2, descr=finaldescr2)
         """, namespace=locals())
         self.cpu.compile_loop(loop.inputargs, loop.operations, looptoken)
diff --git a/rpython/jit/metainterp/optimizeopt/rewrite.py b/rpython/jit/metainterp/optimizeopt/rewrite.py
--- a/rpython/jit/metainterp/optimizeopt/rewrite.py
+++ b/rpython/jit/metainterp/optimizeopt/rewrite.py
@@ -516,18 +516,13 @@
                     assert isinstance(source_value, VArrayValue)
                     val = source_value.getitem(index + source_start)
                 else:
-                    if arraydescr.is_array_of_pointers():
-                        resbox = BoxPtr()
-                    elif arraydescr.is_array_of_floats():
-                        resbox = BoxFloat()
-                    else:
-                        resbox = BoxInt()
-                    newop = ResOperation(rop.GETARRAYITEM_GC,
+                    opnum = OpHelpers.getarrayitem_for_descr(arraydescr)
+                    newop = ResOperation(opnum,
                                       [op.getarg(1),
-                                       ConstInt(index + source_start)], resbox,
+                                       ConstInt(index + source_start)],
                                        descr=arraydescr)
                     self.optimizer.send_extra_operation(newop)
-                    val = self.getvalue(resbox)
+                    val = self.getvalue(newop)
                 if val is None:
                     continue
                 if dest_value.is_virtual():
@@ -536,7 +531,7 @@
                     newop = ResOperation(rop.SETARRAYITEM_GC,
                                          [op.getarg(2),
                                           ConstInt(index + dest_start),
-                                          val.get_key_box()], None,
+                                          val.get_key_box()],
                                          descr=arraydescr)
                     self.emit_operation(newop)
             return True
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py
@@ -2448,7 +2448,7 @@
             varnames = []
         assert len(boxes) == len(varnames)
         for box, varname in zip(boxes, varnames):
-            _variables_equal(box, varname, strict=True)
+            _variables_equal(box, varname, strict=False)
         for pvar, pfieldname, pfieldvar in pendingfields:
             box = oparse.getvar(pvar)
             fielddescr = self.namespace[pfieldname.strip()]
@@ -3027,7 +3027,7 @@
         #
         p2 = virtual_ref(p1, 3)
         setfield_gc(p0, p2, descr=nextdescr)
-        call_may_force(i1, descr=mayforcevirtdescr)
+        call_may_force_n(i1, descr=mayforcevirtdescr)
         guard_not_forced() [i1]
         virtual_ref_finish(p2, p1)
         setfield_gc(p0, NULL, descr=nextdescr)
@@ -3042,7 +3042,7 @@
         setfield_gc(p2, p3, descr=virtualtokendescr)
         setfield_gc(p0, p2, descr=nextdescr)
         #
-        call_may_force(i1, descr=mayforcevirtdescr)
+        call_may_force_n(i1, descr=mayforcevirtdescr)
         guard_not_forced() [i1]
         #
         setfield_gc(p0, NULL, descr=nextdescr)
@@ -3067,7 +3067,7 @@
         #
         p2 = virtual_ref(p1, 2)
         setfield_gc(p0, p2, descr=nextdescr)
-        call_may_force(i1, descr=mayforcevirtdescr)
+        call_may_force_n(i1, descr=mayforcevirtdescr)
         guard_not_forced(descr=fdescr) [p2, p1]
         virtual_ref_finish(p2, p1)
         setfield_gc(p0, NULL, descr=nextdescr)
@@ -3082,7 +3082,7 @@
         setfield_gc(p2, p3, descr=virtualtokendescr)
         setfield_gc(p0, p2, descr=nextdescr)
         #
-        call_may_force(i1, descr=mayforcevirtdescr)
+        call_may_force_n(i1, descr=mayforcevirtdescr)
         guard_not_forced(descr=fdescr) [p2, i1]
         #
         setfield_gc(p0, NULL, descr=nextdescr)
@@ -3113,7 +3113,7 @@
         #
         p2 = virtual_ref(p1, 2)
         setfield_gc(p0, p2, descr=refdescr)
-        call(i1, descr=nonwritedescr)
+        call_n(i1, descr=nonwritedescr)
         guard_no_exception(descr=fdescr) [p2, p1]
         virtual_ref_finish(p2, p1)
         setfield_gc(p0, NULL, descr=refdescr)
@@ -3122,7 +3122,7 @@
         expected = """
         [p0, i1]
         p3 = force_token()
-        call(i1, descr=nonwritedescr)
+        call_n(i1, descr=nonwritedescr)
         guard_no_exception(descr=fdescr) [p3, i1, p0]
         setfield_gc(p0, NULL, descr=refdescr)
         jump(p0, i1)
@@ -3132,7 +3132,8 @@
         #  - p3 is from the virtual expansion of p2
         #  - i1 is from the virtual expansion of p1
         #  - p0 is from the extra pendingfields
-        self.loop.inputargs[0].value = self.nodeobjvalue
+        self.loop.inputargs[0].setref_base(self.nodeobjvalue)
+        py.test.skip("XXX")
         self.check_expanded_fail_descr('''p2, p1
             p0.refdescr = p2
             where p2 is a jit_virtual_ref_vtable, virtualtokendescr=p3
@@ -3147,7 +3148,7 @@
         p2 = virtual_ref(p1, 7)
         escape(p2)
         virtual_ref_finish(p2, p1)
-        call_may_force(i1, descr=mayforcevirtdescr)
+        call_may_force_n(i1, descr=mayforcevirtdescr)
         guard_not_forced() []
         jump(i1)
         """
@@ -3161,7 +3162,7 @@
         p1 = new_with_vtable(ConstClass(node_vtable))
         setfield_gc(p2, p1, descr=virtualforceddescr)
         setfield_gc(p2, NULL, descr=virtualtokendescr)
-        call_may_force(i1, descr=mayforcevirtdescr)
+        call_may_force_n(i1, descr=mayforcevirtdescr)
         guard_not_forced() []
         jump(i1)
         """
@@ -3173,7 +3174,7 @@
         p2 = virtual_ref(p1, 23)
         escape(p2)
         virtual_ref_finish(p2, p1)
-        call_may_force(i1, descr=mayforcevirtdescr)
+        call_may_force_n(i1, descr=mayforcevirtdescr)
         guard_not_forced() [i1]
         jump(i1, p1)
         """
@@ -3186,7 +3187,7 @@
         escape(p2)
         setfield_gc(p2, p1, descr=virtualforceddescr)
         setfield_gc(p2, NULL, descr=virtualtokendescr)
-        call_may_force(i1, descr=mayforcevirtdescr)
+        call_may_force_n(i1, descr=mayforcevirtdescr)
         guard_not_forced() [i1]
         jump(i1, p1)
         """
@@ -3199,8 +3200,8 @@
         setarrayitem_gc(p1, 1, 1, descr=arraydescr)
         p2 = new_array(3, descr=arraydescr)
         setarrayitem_gc(p2, 1, 3, descr=arraydescr)
-        call(0, p1, p2, 1, 1, 2, descr=arraycopydescr)
-        i2 = getarrayitem_gc(p2, 1, descr=arraydescr)
+        call_n(0, p1, p2, 1, 1, 2, descr=arraycopydescr)
+        i2 = getarrayitem_gc_i(p2, 1, descr=arraydescr)
         jump(i2)
         '''
         expected = '''
@@ -3216,8 +3217,8 @@
         p2 = new_array(3, descr=arraydescr)
         setarrayitem_gc(p1, 0, i0, descr=arraydescr)
         setarrayitem_gc(p2, 0, 3, descr=arraydescr)
-        call(0, p1, p2, 1, 1, 2, descr=arraycopydescr)
-        i2 = getarrayitem_gc(p2, 0, descr=arraydescr)
+        call_n(0, p1, p2, 1, 1, 2, descr=arraycopydescr)
+        i2 = getarrayitem_gc_i(p2, 0, descr=arraydescr)
         jump(i2)
         '''
         expected = '''
@@ -3233,7 +3234,7 @@
         p2 = new_array(3, descr=arraydescr)
         setarrayitem_gc(p1, 2, 10, descr=arraydescr)
         setarrayitem_gc(p2, 2, 13, descr=arraydescr)
-        call(0, p1, p2, 0, 0, 3, descr=arraycopydescr)
+        call_n(0, p1, p2, 0, 0, 3, descr=arraycopydescr)
         jump(p2)
         '''
         expected = '''
@@ -3248,15 +3249,15 @@
         ops = '''
         [p0]
         p1 = new_array(3, descr=arraydescr)
-        call(0, p0, p1, 0, 0, 3, descr=arraycopydescr)
-        i0 = getarrayitem_gc(p1, 0, descr=arraydescr)
+        call_n(0, p0, p1, 0, 0, 3, descr=arraycopydescr)
+        i0 = getarrayitem_gc_i(p1, 0, descr=arraydescr)
         jump(i0)
         '''
         expected = '''
         [p0]
-        i0 = getarrayitem_gc(p0, 0, descr=arraydescr)
-        i1 = getarrayitem_gc(p0, 1, descr=arraydescr) # removed by the backend
-        i2 = getarrayitem_gc(p0, 2, descr=arraydescr) # removed by the backend
+        i0 = getarrayitem_gc_i(p0, 0, descr=arraydescr)
+        i1 = getarrayitem_gc_i(p0, 1, descr=arraydescr) # removed by the backend
+        i2 = getarrayitem_gc_i(p0, 2, descr=arraydescr) # removed by the backend
         jump(i0)
         '''
         self.optimize_loop(ops, expected)
@@ -3264,15 +3265,15 @@
     def test_arraycopy_not_virtual_3(self):
         ops = '''
         [p0, p1]
-        call(0, p0, p1, 0, 0, 3, descr=arraycopydescr)
-        i0 = getarrayitem_gc(p1, 0, descr=arraydescr)
+        call_n(0, p0, p1, 0, 0, 3, descr=arraycopydescr)
+        i0 = getarrayitem_gc_i(p1, 0, descr=arraydescr)
         jump(i0)
         '''
         expected = '''
         [p0, p1]
-        i0 = getarrayitem_gc(p0, 0, descr=arraydescr)
-        i1 = getarrayitem_gc(p0, 1, descr=arraydescr)
-        i2 = getarrayitem_gc(p0, 2, descr=arraydescr)
+        i0 = getarrayitem_gc_i(p0, 0, descr=arraydescr)
+        i1 = getarrayitem_gc_i(p0, 1, descr=arraydescr)
+        i2 = getarrayitem_gc_i(p0, 2, descr=arraydescr)
         setarrayitem_gc(p1, 0, i0, descr=arraydescr)
         setarrayitem_gc(p1, 1, i1, descr=arraydescr)
         setarrayitem_gc(p1, 2, i2, descr=arraydescr)
@@ -3286,7 +3287,7 @@
         ops = '''
         [p1]
         p0 = new_array(0, descr=arraydescr)
-        call(0, p0, p1, 0, 0, 0, descr=arraycopydescr)
+        call_n(0, p0, p1, 0, 0, 0, descr=arraycopydescr)
         jump(p1)
         '''
         expected = '''
@@ -3617,7 +3618,7 @@
         i1 = arraylen_gc(p1)
         i2 = int_gt(i1, -1)
         guard_true(i2) []
-        setarrayitem_gc(p0, 0, p1)
+        setarrayitem_gc(p0, 0, p1, descr=arraydescr)
         jump(i0, p0)
         """
         # The dead arraylen_gc will be eliminated by the backend.
@@ -3625,7 +3626,7 @@
         [i0, p0]
         p1 = new_array(i0, descr=arraydescr)
         i1 = arraylen_gc(p1)
-        setarrayitem_gc(p0, 0, p1)
+        setarrayitem_gc(p0, 0, p1, descr=arraydescr)
         jump(i0, p0)
         """
         self.optimize_loop(ops, expected)
@@ -5211,7 +5212,7 @@
         strsetitem(p0, 0, 97)
         strsetitem(p0, 1, 98)
         strsetitem(p0, 2, 99)
-        i0 = call_pure(123, p0, descr=nonwritedescr)
+        i0 = call_pure_i(123, p0, descr=nonwritedescr)
         finish(i0)
         """
         expected = """
@@ -5228,8 +5229,8 @@
         []
         quasiimmut_field(ConstPtr(quasiptr), descr=quasiimmutdescr)
         guard_not_invalidated() []
-        i0 = getfield_gc_pure(ConstPtr(quasiptr), descr=quasifielddescr)
-        i1 = call_pure(123, i0, descr=nonwritedescr)
+        i0 = getfield_gc_pure_i(ConstPtr(quasiptr), descr=quasifielddescr)
+        i1 = call_pure_i(123, i0, descr=nonwritedescr)
         finish(i1)
         """
         expected = """
@@ -5293,14 +5294,14 @@
     def test_getfieldraw_cmp_outside_bounds(self):
         ops = """
         [p0]
-        i0 = getfield_raw(p0, descr=chardescr)
+        i0 = getfield_raw_i(p0, descr=chardescr)
         i1 = int_gt(i0, -1)
         guard_true(i1) []
         """
 
         expected = """
         [p0]
-        i0 = getfield_raw(p0, descr=chardescr)
+        i0 = getfield_raw_i(p0, descr=chardescr)
         """
         self.optimize_loop(ops, expected)
 
@@ -5308,51 +5309,51 @@
     def test_rawarray_cmp_outside_intbounds(self):
         ops = """
         [i0]
-        i1 = getarrayitem_raw(i0, 0, descr=rawarraydescr_char)
+        i1 = getarrayitem_raw_i(i0, 0, descr=rawarraydescr_char)
         i2 = int_lt(i1, 256)
         guard_true(i2) []
         """
 
         expected = """
         [i0]
-        i1 = getarrayitem_raw(i0, 0, descr=rawarraydescr_char)
+        i1 = getarrayitem_raw_i(i0, 0, descr=rawarraydescr_char)
         """
         self.optimize_loop(ops, expected)
 
     def test_gcarray_outside_intbounds(self):
         ops = """
         [p0]
-        i0 = getarrayitem_gc(p0, 0, descr=chararraydescr)
+        i0 = getarrayitem_gc_i(p0, 0, descr=chararraydescr)
         i1 = int_lt(i0, 256)
         guard_true(i1) []
         """
 
         expected = """
         [p0]
-        i0 = getarrayitem_gc(p0, 0, descr=chararraydescr)
+        i0 = getarrayitem_gc_i(p0, 0, descr=chararraydescr)
         """
         self.optimize_loop(ops, expected)
 
     def test_getinterior_outside_intbounds(self):
         ops = """
         [p0]
-        f0 = getinteriorfield_gc(p0, 0, descr=fc_array_floatdescr)
-        i0 = getinteriorfield_gc(p0, 0, descr=fc_array_chardescr)
+        f0 = getinteriorfield_gc_f(p0, 0, descr=fc_array_floatdescr)
+        i0 = getinteriorfield_gc_i(p0, 0, descr=fc_array_chardescr)
         i1 = int_lt(i0, 256)
         guard_true(i1) []
         """
 
         expected = """
         [p0]
-        f0 = getinteriorfield_gc(p0, 0, descr=fc_array_floatdescr)
-        i0 = getinteriorfield_gc(p0, 0, descr=fc_array_chardescr)
+        f0 = getinteriorfield_gc_f(p0, 0, descr=fc_array_floatdescr)
+        i0 = getinteriorfield_gc_i(p0, 0, descr=fc_array_chardescr)
         """
         self.optimize_loop(ops, expected)
 
     def test_intand_1mask_covering_bitrange(self):
         ops = """
         [p0]
-        i0 = getarrayitem_gc(p0, 0, descr=chararraydescr)
+        i0 = getarrayitem_gc_i(p0, 0, descr=chararraydescr)
         i1 = int_and(i0, 255)
         i2 = int_and(i1, -1)
         i3 = int_and(511, i2)
@@ -5361,7 +5362,7 @@
 
         expected = """
         [p0]
-        i0 = getarrayitem_gc(p0, 0, descr=chararraydescr)
+        i0 = getarrayitem_gc_i(p0, 0, descr=chararraydescr)
         jump(i0)
         """
         self.optimize_loop(ops, expected)
@@ -5369,9 +5370,9 @@
     def test_intand_maskwith0_in_bitrange(self):
         ops = """
         [p0]
-        i0 = getarrayitem_gc(p0, 0, descr=chararraydescr)
+        i0 = getarrayitem_gc_i(p0, 0, descr=chararraydescr)
         i1 = int_and(i0, 257)
-        i2 = getarrayitem_gc(p0, 1, descr=chararraydescr)
+        i2 = getarrayitem_gc_i(p0, 1, descr=chararraydescr)
         i3 = int_and(259, i2)
         jump(i1, i3)
         """
@@ -5380,8 +5381,8 @@
     def test_int_and_cmp_above_bounds(self):
         ops = """
         [p0,p1]
-        i0 = getarrayitem_gc(p0, 0, descr=chararraydescr)
-        i1 = getarrayitem_gc(p1, 0, descr=u2arraydescr)
+        i0 = getarrayitem_gc_i(p0, 0, descr=chararraydescr)
+        i1 = getarrayitem_gc_i(p1, 0, descr=u2arraydescr)
         i2 = int_and(i0, i1)
         i3 = int_le(i2, 255)
         guard_true(i3) []
@@ -5390,8 +5391,8 @@
 
         expected = """
         [p0,p1]
-        i0 = getarrayitem_gc(p0, 0, descr=chararraydescr)
-        i1 = getarrayitem_gc(p1, 0, descr=u2arraydescr)
+        i0 = getarrayitem_gc_i(p0, 0, descr=chararraydescr)
+        i1 = getarrayitem_gc_i(p1, 0, descr=u2arraydescr)
         i2 = int_and(i0, i1)
         jump(i2)
         """
@@ -5400,8 +5401,8 @@
     def test_int_and_cmp_below_bounds(self):
         ops = """
         [p0,p1]
-        i0 = getarrayitem_gc(p0, 0, descr=chararraydescr)
-        i1 = getarrayitem_gc(p1, 0, descr=u2arraydescr)
+        i0 = getarrayitem_gc_i(p0, 0, descr=chararraydescr)
+        i1 = getarrayitem_gc_i(p1, 0, descr=u2arraydescr)
         i2 = int_and(i0, i1)
         i3 = int_lt(i2, 255)
         guard_true(i3) []
@@ -5412,8 +5413,8 @@
     def test_int_or_cmp_above_bounds(self):
         ops = """
         [p0,p1]
-        i0 = getarrayitem_gc(p0, 0, descr=chararraydescr)
-        i1 = getarrayitem_gc(p1, 0, descr=u2arraydescr)
+        i0 = getarrayitem_gc_i(p0, 0, descr=chararraydescr)
+        i1 = getarrayitem_gc_i(p1, 0, descr=u2arraydescr)
         i2 = int_or(i0, i1)
         i3 = int_le(i2, 65535)
         guard_true(i3) []
@@ -5422,8 +5423,8 @@
 
         expected = """
         [p0,p1]
-        i0 = getarrayitem_gc(p0, 0, descr=chararraydescr)
-        i1 = getarrayitem_gc(p1, 0, descr=u2arraydescr)
+        i0 = getarrayitem_gc_i(p0, 0, descr=chararraydescr)
+        i1 = getarrayitem_gc_i(p1, 0, descr=u2arraydescr)
         i2 = int_or(i0, i1)
         jump(i2)
         """
@@ -5432,8 +5433,8 @@
     def test_int_or_cmp_below_bounds(self):
         ops = """
         [p0,p1]
-        i0 = getarrayitem_gc(p0, 0, descr=chararraydescr)
-        i1 = getarrayitem_gc(p1, 0, descr=u2arraydescr)
+        i0 = getarrayitem_gc_i(p0, 0, descr=chararraydescr)
+        i1 = getarrayitem_gc_i(p1, 0, descr=u2arraydescr)
         i2 = int_or(i0, i1)
         i3 = int_lt(i2, 65535)
         guard_true(i3) []
@@ -5444,8 +5445,8 @@
     def test_int_xor_cmp_above_bounds(self):
         ops = """
         [p0,p1]
-        i0 = getarrayitem_gc(p0, 0, descr=chararraydescr)
-        i1 = getarrayitem_gc(p1, 0, descr=u2arraydescr)
+        i0 = getarrayitem_gc_i(p0, 0, descr=chararraydescr)
+        i1 = getarrayitem_gc_i(p1, 0, descr=u2arraydescr)
         i2 = int_xor(i0, i1)
         i3 = int_le(i2, 65535)
         guard_true(i3) []
@@ -5454,8 +5455,8 @@
 
         expected = """
         [p0,p1]
-        i0 = getarrayitem_gc(p0, 0, descr=chararraydescr)
-        i1 = getarrayitem_gc(p1, 0, descr=u2arraydescr)
+        i0 = getarrayitem_gc_i(p0, 0, descr=chararraydescr)
+        i1 = getarrayitem_gc_i(p1, 0, descr=u2arraydescr)
         i2 = int_xor(i0, i1)
         jump(i2)
         """
@@ -5464,8 +5465,8 @@
     def test_int_xor_cmp_below_bounds(self):
         ops = """
         [p0,p1]
-        i0 = getarrayitem_gc(p0, 0, descr=chararraydescr)
-        i1 = getarrayitem_gc(p1, 0, descr=u2arraydescr)
+        i0 = getarrayitem_gc_i(p0, 0, descr=chararraydescr)
+        i1 = getarrayitem_gc_i(p1, 0, descr=u2arraydescr)
         i2 = int_xor(i0, i1)
         i3 = int_lt(i2, 65535)
         guard_true(i3) []
@@ -5489,13 +5490,13 @@
         py.test.skip("we want this to pass")
         ops = """
         [p0, i0]
-        i1 = getinteriorfield_gc(p0, i0, descr=valuedescr)
-        i2 = getinteriorfield_gc(p0, i0, descr=valuedescr)
+        i1 = getinteriorfield_gc_i(p0, i0, descr=valuedescr)
+        i2 = getinteriorfield_gc_i(p0, i0, descr=valuedescr)
         jump(i1, i2)
         """
         expected = """
         [p0, i0]
-        i1 = getinteriorfield_gc(p0, i0, descr=valuedescr)
+        i1 = getinteriorfield_gc_i(p0, i0, descr=valuedescr)
         jump(i1, i1)
         """
         self.optimize_loop(ops, expected)
diff --git a/rpython/jit/metainterp/optimizeopt/unroll.py b/rpython/jit/metainterp/optimizeopt/unroll.py
--- a/rpython/jit/metainterp/optimizeopt/unroll.py
+++ b/rpython/jit/metainterp/optimizeopt/unroll.py
@@ -35,7 +35,7 @@
 
     def emit_operation(self, op):
         if op.returns_bool_result():
-            self.bool_boxes[self.getvalue(op.result)] = None
+            self.bool_boxes[self.getvalue(op)] = None
         if self.emitting_dissabled:
             return
         if op.is_guard():
@@ -102,7 +102,7 @@
 
         cell_token = jumpop.getdescr()
         assert isinstance(cell_token, JitCellToken)
-        stop_label = ResOperation(rop.LABEL, jumpop.getarglist(), None, TargetToken(cell_token))
+        stop_label = ResOperation(rop.LABEL, jumpop.getarglist(), TargetToken(cell_token))
 
         if jumpop.getopnum() == rop.JUMP:
             if self.jump_to_already_compiled_trace(jumpop, patchguardop):
diff --git a/rpython/jit/metainterp/optimizeopt/virtualize.py b/rpython/jit/metainterp/optimizeopt/virtualize.py
--- a/rpython/jit/metainterp/optimizeopt/virtualize.py
+++ b/rpython/jit/metainterp/optimizeopt/virtualize.py
@@ -605,11 +605,13 @@
         # Replace the VIRTUAL_REF operation with a virtual structure of type
         # 'jit_virtual_ref'.  The jit_virtual_ref structure may be forced soon,
         # but the point is that doing so does not force the original structure.
-        op = ResOperation(rop.NEW_WITH_VTABLE, [c_cls], op)
-        vrefvalue = self.make_virtual(c_cls, op)
-        tokenbox = BoxPtr()
-        self.emit_operation(ResOperation(rop.FORCE_TOKEN, [], tokenbox))
-        vrefvalue.setfield(descr_virtual_token, self.getvalue(tokenbox))
+        newop = ResOperation(rop.NEW_WITH_VTABLE, [c_cls])
+        vrefvalue = self.make_virtual(c_cls, newop)
+        assert op not in self.optimizer.values
+        self.optimizer.values[op] = vrefvalue
+        token = ResOperation(rop.FORCE_TOKEN, [])
+        self.emit_operation(token)
+        vrefvalue.setfield(descr_virtual_token, self.getvalue(token))
         vrefvalue.setfield(descr_forced, self.optimizer.cpu.ts.CVAL_NULLREF)
 
     def optimize_VIRTUAL_REF_FINISH(self, op):
@@ -632,12 +634,12 @@
         # - set 'forced' to point to the real object
         objbox = op.getarg(1)
         if not CONST_NULL.same_constant(objbox):
-            seo(ResOperation(rop.SETFIELD_GC, op.getarglist(), None,
+            seo(ResOperation(rop.SETFIELD_GC, op.getarglist(),
                              descr=vrefinfo.descr_forced))
 
         # - set 'virtual_token' to TOKEN_NONE (== NULL)
         args = [op.getarg(0), CONST_NULL]
-        seo(ResOperation(rop.SETFIELD_GC, args, None,
+        seo(ResOperation(rop.SETFIELD_GC, args,
                          descr=vrefinfo.descr_virtual_token))
         # Note that in some cases the virtual in op.getarg(1) has been forced
         # already.  This is fine.  In that case, and *if* a residual
diff --git a/rpython/jit/metainterp/optimizeopt/virtualstate.py b/rpython/jit/metainterp/optimizeopt/virtualstate.py
--- a/rpython/jit/metainterp/optimizeopt/virtualstate.py
+++ b/rpython/jit/metainterp/optimizeopt/virtualstate.py
@@ -686,18 +686,18 @@
             raise BoxNotProducable
 
     def add_potential(self, op, synthetic=False):
-        if op.result and op.result in self.optimizer.values:
-            value = self.optimizer.values[op.result]
+        if op in self.optimizer.values:
+            value = self.optimizer.values[op]
             if value in self.optimizer.opaque_pointers:
                 classbox = value.get_constant_class(self.optimizer.cpu)
                 if classbox:
-                    self.assumed_classes[op.result] = classbox
-        if op.result not in self.potential_ops:
-            self.potential_ops[op.result] = op
+                    self.assumed_classes[op] = classbox
+        if op not in self.potential_ops:
+            self.potential_ops[op] = op
         else:
-            if op.result not in self.alternatives:
-                self.alternatives[op.result] = [self.potential_ops[op.result]]
-            self.alternatives[op.result].append(op)
+            if op not in self.alternatives:
+                self.alternatives[op] = [self.potential_ops[op]]
+            self.alternatives[op].append(op)
         if synthetic:
             self.synthetic[op] = True
 
diff --git a/rpython/jit/metainterp/optimizeopt/vstring.py b/rpython/jit/metainterp/optimizeopt/vstring.py
--- a/rpython/jit/metainterp/optimizeopt/vstring.py
+++ b/rpython/jit/metainterp/optimizeopt/vstring.py
@@ -48,7 +48,7 @@
 class __extend__(optimizer.OptValue):
     """New methods added to the base class OptValue for this file."""
 
-    def getstrlen(self, string_optimizer, mode, lengthbox):
+    def getstrlen(self, string_optimizer, mode, lengthop):
         if mode is mode_string:
             s = self.get_constant_string_spec(mode_string)
             if s is not None:
@@ -61,10 +61,13 @@
             return None
         self.ensure_nonnull()
         box = self.force_box(string_optimizer)
-        if lengthbox is None:
-            lengthbox = BoxInt()
-        string_optimizer.emit_operation(ResOperation(mode.STRLEN, [box], lengthbox))
-        return lengthbox
+        if lengthop is None:
+            xxx
+        else:
+            lengthop = string_optimizer.optimizer.replace_op_with(lengthop,
+                mode.STRLEN, [box])
+        string_optimizer.emit_operation(lengthop)
+        return lengthop
 
     @specialize.arg(1)
     def get_constant_string_spec(self, mode):
@@ -278,6 +281,7 @@
         self.vlength = vlength
 
     def getstrlen(self, optforce, mode, lengthbox):
+        xxx
         return self.vlength.force_box(optforce)
 
     @specialize.arg(1)
@@ -485,11 +489,11 @@
 
     def _optimize_STRLEN(self, op, mode):
         value = self.getvalue(op.getarg(0))
-        lengthbox = value.getstrlen(self, mode, op.result)
-        if op.result in self.optimizer.values:
-            assert self.getvalue(op.result) is self.getvalue(lengthbox)
-        elif op.result is not lengthbox:
-            self.make_equal_to(op.result, self.getvalue(lengthbox))
+        lengthbox = value.getstrlen(self, mode, op)
+        if op in self.optimizer.values:
+            assert self.getvalue(op) is self.getvalue(lengthbox)
+        elif op is not lengthbox:
+            self.make_equal_to(op, self.getvalue(lengthbox))
 
     def optimize_COPYSTRCONTENT(self, op):
         self._optimize_COPYSTRCONTENT(op, mode_string)
diff --git a/rpython/jit/metainterp/resoperation.py b/rpython/jit/metainterp/resoperation.py
--- a/rpython/jit/metainterp/resoperation.py
+++ b/rpython/jit/metainterp/resoperation.py
@@ -650,7 +650,7 @@
     'NEWSTR/1/r',         #-> STR, the hash field is zeroed
     'NEWUNICODE/1/r',     #-> UNICODE, the hash field is zeroed
     '_MALLOC_LAST',
-    'FORCE_TOKEN/0/i',
+    'FORCE_TOKEN/0/r',
     'VIRTUAL_REF/2/r',    # removed before it's passed to the backend
     'MARK_OPAQUE_PTR/1b/n',
     # this one has no *visible* side effect, since the virtualizable

From noreply at buildbot.pypy.org  Wed Nov 26 18:16:52 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Wed, 26 Nov 2014 18:16:52 +0100 (CET)
Subject: [pypy-commit] pypy default: Add 'int_signext' to replace the three
 instructions which have all very
Message-ID: <20141126171652.454961C0399@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r74738:68c97410a30e
Date: 2014-11-26 18:16 +0100
http://bitbucket.org/pypy/pypy/changeset/68c97410a30e/

Log:	Add 'int_signext' to replace the three instructions which have all
	very large constants when used to sign-extend a 32-bit integer into
	a full 64-bit word.

diff --git a/rpython/jit/backend/arm/codebuilder.py b/rpython/jit/backend/arm/codebuilder.py
--- a/rpython/jit/backend/arm/codebuilder.py
+++ b/rpython/jit/backend/arm/codebuilder.py
@@ -318,6 +318,18 @@
                     | (rd & 0xF) << 12
                     | imm16 & 0xFFF)
 
+    def SXTB_rr(self, rd, rm, c=cond.AL):
+        self.write32(c << 28
+                    | 0x06AF0070
+                    | (rd & 0xF) << 12
+                    | (rm & 0xF))
+
+    def SXTB16_rr(self, rd, rm, c=cond.AL):
+        self.write32(c << 28
+                    | 0x068F0070
+                    | (rd & 0xF) << 12
+                    | (rm & 0xF))
+
     def LDREX(self, rt, rn, c=cond.AL):
         self.write32(c << 28
                     | 0x01900f9f
diff --git a/rpython/jit/backend/arm/opassembler.py b/rpython/jit/backend/arm/opassembler.py
--- a/rpython/jit/backend/arm/opassembler.py
+++ b/rpython/jit/backend/arm/opassembler.py
@@ -102,6 +102,17 @@
         self.mc.MOV_rr(res.value, arg.value, cond=c.GE)
         return fcond
 
+    def emit_op_int_signext(self, op, arglocs, regalloc, fcond):
+        arg, numbytes, res = arglocs
+        assert numbytes.is_imm()
+        if numbytes.value == 1:
+            self.mc.SXTB_rr(res.value, arg.value)
+        elif numbytes.value == 2:
+            self.mc.SXTB16_rr(res.value, arg.value)
+        else:
+            raise AssertionError("bad number of bytes")
+        return fcond
+
     #ref: http://blogs.arm.com/software-enablement/detecting-overflow-from-mul/
     def emit_guard_int_mul_ovf(self, op, guard, arglocs, regalloc, fcond):
         reg1 = arglocs[0]
diff --git a/rpython/jit/backend/arm/regalloc.py b/rpython/jit/backend/arm/regalloc.py
--- a/rpython/jit/backend/arm/regalloc.py
+++ b/rpython/jit/backend/arm/regalloc.py
@@ -458,6 +458,12 @@
         resloc = self.force_allocate_reg(op.result, [op.getarg(0)])
         return [argloc, resloc]
 
+    def prepare_op_int_signext(self, op, fcond):
+        argloc = self.make_sure_var_in_reg(op.getarg(0))
+        numbytes = op.getarg(1).getint()
+        resloc = self.force_allocate_reg(op.result)
+        return [argloc, imm(numbytes), resloc]
+
     def prepare_guard_int_mul_ovf(self, op, guard, fcond):
         boxes = op.getarglist()
         reg1 = self.make_sure_var_in_reg(boxes[0], forbidden_vars=boxes)
diff --git a/rpython/jit/backend/test/runner_test.py b/rpython/jit/backend/test/runner_test.py
--- a/rpython/jit/backend/test/runner_test.py
+++ b/rpython/jit/backend/test/runner_test.py
@@ -3890,6 +3890,26 @@
             deadframe = self.cpu.execute_token(looptoken, inp)
             assert outp == self.cpu.get_int_value(deadframe, 0)
 
+    def test_int_signext(self):
+        numbytes_cases = [1, 2] if IS_32_BIT else [1, 2, 4]
+        for numbytes in numbytes_cases:
+            ops = """
+            [i0]
+            i1 = int_signext(i0, %d)
+            finish(i1, descr=descr)
+            """ % numbytes
+            descr = BasicFinalDescr()
+            loop = parse(ops, self.cpu, namespace=locals())
+            looptoken = JitCellToken()
+            self.cpu.compile_loop(loop.inputargs, loop.operations, looptoken)
+            test_cases = [random.randrange(-sys.maxint-1, sys.maxint+1)
+                          for _ in range(100)]
+            for test_case in test_cases:
+                deadframe = self.cpu.execute_token(looptoken, test_case)
+                got = self.cpu.get_int_value(deadframe, 0)
+                expected = heaptracker.int_signext(test_case, numbytes)
+                assert got == expected
+
     def test_compile_asmlen(self):
         from rpython.jit.backend.llsupport.llmodel import AbstractLLCPU
         if not isinstance(self.cpu, AbstractLLCPU):
diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py
--- a/rpython/jit/backend/x86/assembler.py
+++ b/rpython/jit/backend/x86/assembler.py
@@ -1143,6 +1143,18 @@
     def genop_math_sqrt(self, op, arglocs, resloc):
         self.mc.SQRTSD(arglocs[0], resloc)
 
+    def genop_int_signext(self, op, arglocs, resloc):
+        argloc, numbytesloc = arglocs
+        assert isinstance(numbytesloc, ImmedLoc)
+        if numbytesloc.value == 1:
+            self.mc.MOVSX8(resloc, argloc)
+        elif numbytesloc.value == 2:
+            self.mc.MOVSX16(resloc, argloc)
+        elif IS_X86_64 and numbytesloc.value == 4:
+            self.mc.MOVSX32(resloc, argloc)
+        else:
+            raise AssertionError("bad number of bytes")
+
     def genop_guard_float_ne(self, op, guard_op, guard_token, arglocs, result_loc):
         guard_opnum = guard_op.getopnum()
         if isinstance(arglocs[0], RegLoc):
diff --git a/rpython/jit/backend/x86/regalloc.py b/rpython/jit/backend/x86/regalloc.py
--- a/rpython/jit/backend/x86/regalloc.py
+++ b/rpython/jit/backend/x86/regalloc.py
@@ -474,6 +474,12 @@
 
     consider_int_invert = consider_int_neg
 
+    def consider_int_signext(self, op):
+        argloc = self.loc(op.getarg(0))
+        numbytesloc = self.loc(op.getarg(1))
+        resloc = self.force_allocate_reg(op.result)
+        self.perform(op, [argloc, numbytesloc], resloc)
+
     def consider_int_lshift(self, op):
         if isinstance(op.getarg(1), Const):
             loc2 = self.rm.convert_to_imm(op.getarg(1))
diff --git a/rpython/jit/codewriter/heaptracker.py b/rpython/jit/codewriter/heaptracker.py
--- a/rpython/jit/codewriter/heaptracker.py
+++ b/rpython/jit/codewriter/heaptracker.py
@@ -1,6 +1,7 @@
 from rpython.rtyper.lltypesystem import lltype, llmemory
 from rpython.rtyper import rclass
 from rpython.rlib.objectmodel import we_are_translated
+from rpython.rlib.rarithmetic import r_uint, intmask
 
 
 def adr2int(addr):
@@ -11,6 +12,14 @@
 def int2adr(int):
     return llmemory.cast_int_to_adr(int)
 
+def int_signext(value, numbytes):
+    b8 = numbytes * 8
+    a = r_uint(value)
+    a += r_uint(1 << (b8 - 1))     # a += 128
+    a &= r_uint((1 << b8) - 1)     # a &= 255
+    a -= r_uint(1 << (b8 - 1))     # a -= 128
+    return intmask(a)
+
 def count_fields_if_immutable(STRUCT):
     assert isinstance(STRUCT, lltype.GcStruct)
     if STRUCT._hints.get('immutable', False):
diff --git a/rpython/jit/codewriter/jtransform.py b/rpython/jit/codewriter/jtransform.py
--- a/rpython/jit/codewriter/jtransform.py
+++ b/rpython/jit/codewriter/jtransform.py
@@ -1267,19 +1267,12 @@
 
         result = []
         if min2:
-            c_min2 = Constant(min2, lltype.Signed)
-            v2 = varoftype(lltype.Signed)
-            result.append(SpaceOperation('int_sub', [v_arg, c_min2], v2))
+            c_bytes = Constant(size2, lltype.Signed)
+            result.append(SpaceOperation('int_signext', [v_arg, c_bytes],
+                                         v_result))
         else:
-            v2 = v_arg
-        c_mask = Constant(int((1 << (8 * size2)) - 1), lltype.Signed)
-        if min2:
-            v3 = varoftype(lltype.Signed)
-        else:
-            v3 = v_result
-        result.append(SpaceOperation('int_and', [v2, c_mask], v3))
-        if min2:
-            result.append(SpaceOperation('int_add', [v3, c_min2], v_result))
+            c_mask = Constant(int((1 << (8 * size2)) - 1), lltype.Signed)
+            result.append(SpaceOperation('int_and', [v_arg, c_mask], v_result))
         return result
 
     def _float_to_float_cast(self, v_arg, v_result):
diff --git a/rpython/jit/codewriter/test/test_flatten.py b/rpython/jit/codewriter/test/test_flatten.py
--- a/rpython/jit/codewriter/test/test_flatten.py
+++ b/rpython/jit/codewriter/test/test_flatten.py
@@ -1,5 +1,6 @@
 import py, sys
 from rpython.jit.codewriter import support
+from rpython.jit.codewriter.heaptracker import int_signext
 from rpython.jit.codewriter.flatten import flatten_graph, reorder_renaming_list
 from rpython.jit.codewriter.flatten import GraphFlattener, ListOfKind, Register
 from rpython.jit.codewriter.format import assert_format
@@ -780,53 +781,37 @@
             (rffi.SIGNEDCHAR, rffi.LONG, ""),
             (rffi.SIGNEDCHAR, rffi.ULONG, ""),
 
-            (rffi.UCHAR, rffi.SIGNEDCHAR, """int_sub %i0, $-128 -> %i1
-                                             int_and %i1, $255 -> %i2
-                                             int_add %i2, $-128 -> %i3"""),
+            (rffi.UCHAR, rffi.SIGNEDCHAR, "int_signext %i0, $1 -> %i1"),
             (rffi.UCHAR, rffi.UCHAR, ""),
             (rffi.UCHAR, rffi.SHORT, ""),
             (rffi.UCHAR, rffi.USHORT, ""),
             (rffi.UCHAR, rffi.LONG, ""),
             (rffi.UCHAR, rffi.ULONG, ""),
 
-            (rffi.SHORT, rffi.SIGNEDCHAR, """int_sub %i0, $-128 -> %i1
-                                             int_and %i1, $255 -> %i2
-                                             int_add %i2, $-128 -> %i3"""),
+            (rffi.SHORT, rffi.SIGNEDCHAR, "int_signext %i0, $1 -> %i1"),
             (rffi.SHORT, rffi.UCHAR, "int_and %i0, $255 -> %i1"),
             (rffi.SHORT, rffi.SHORT, ""),
             (rffi.SHORT, rffi.USHORT, "int_and %i0, $65535 -> %i1"),
             (rffi.SHORT, rffi.LONG, ""),
             (rffi.SHORT, rffi.ULONG, ""),
 
-            (rffi.USHORT, rffi.SIGNEDCHAR, """int_sub %i0, $-128 -> %i1
-                                              int_and %i1, $255 -> %i2
-                                              int_add %i2, $-128 -> %i3"""),
+            (rffi.USHORT, rffi.SIGNEDCHAR, "int_signext %i0, $1 -> %i1"),
             (rffi.USHORT, rffi.UCHAR, "int_and %i0, $255 -> %i1"),
-            (rffi.USHORT, rffi.SHORT, """int_sub %i0, $-32768 -> %i1
-                                         int_and %i1, $65535 -> %i2
-                                         int_add %i2, $-32768 -> %i3"""),
+            (rffi.USHORT, rffi.SHORT, "int_signext %i0, $2 -> %i1"),
             (rffi.USHORT, rffi.USHORT, ""),
             (rffi.USHORT, rffi.LONG, ""),
             (rffi.USHORT, rffi.ULONG, ""),
 
-            (rffi.LONG, rffi.SIGNEDCHAR, """int_sub %i0, $-128 -> %i1
-                                            int_and %i1, $255 -> %i2
-                                            int_add %i2, $-128 -> %i3"""),
+            (rffi.LONG, rffi.SIGNEDCHAR, "int_signext %i0, $1 -> %i1"),
             (rffi.LONG, rffi.UCHAR, "int_and %i0, $255 -> %i1"),
-            (rffi.LONG, rffi.SHORT, """int_sub %i0, $-32768 -> %i1
-                                       int_and %i1, $65535 -> %i2
-                                       int_add %i2, $-32768 -> %i3"""),
+            (rffi.LONG, rffi.SHORT, "int_signext %i0, $2 -> %i1"),
             (rffi.LONG, rffi.USHORT, "int_and %i0, $65535 -> %i1"),
             (rffi.LONG, rffi.LONG, ""),
             (rffi.LONG, rffi.ULONG, ""),
 
-            (rffi.ULONG, rffi.SIGNEDCHAR, """int_sub %i0, $-128 -> %i1
-                                             int_and %i1, $255 -> %i2
-                                             int_add %i2, $-128 -> %i3"""),
+            (rffi.ULONG, rffi.SIGNEDCHAR, "int_signext %i0, $1 -> %i1"),
             (rffi.ULONG, rffi.UCHAR, "int_and %i0, $255 -> %i1"),
-            (rffi.ULONG, rffi.SHORT, """int_sub %i0, $-32768 -> %i1
-                                        int_and %i1, $65535 -> %i2
-                                        int_add %i2, $-32768 -> %i3"""),
+            (rffi.ULONG, rffi.SHORT, "int_signext %i0, $2 -> %i1"),
             (rffi.ULONG, rffi.USHORT, "int_and %i0, $65535 -> %i1"),
             (rffi.ULONG, rffi.LONG, ""),
             (rffi.ULONG, rffi.ULONG, ""),
@@ -910,18 +895,14 @@
             return rffi.cast(rffi.SIGNEDCHAR, n)
         self.encoding_test(f, [12.456], """
             cast_float_to_int %f0 -> %i0
-            int_sub %i0, $-128 -> %i1
-            int_and %i1, $255 -> %i2
-            int_add %i2, $-128 -> %i3
-            int_return %i3
+            int_signext %i0, $1 -> %i1
+            int_return %i1
         """, transform=True)
         self.encoding_test(f, [rffi.cast(lltype.SingleFloat, 12.456)], """
             cast_singlefloat_to_float %i0 -> %f0
             cast_float_to_int %f0 -> %i1
-            int_sub %i1, $-128 -> %i2
-            int_and %i2, $255 -> %i3
-            int_add %i3, $-128 -> %i4
-            int_return %i4
+            int_signext %i1, $1 -> %i2
+            int_return %i2
         """, transform=True)
 
         def f(dbl):
@@ -1068,9 +1049,12 @@
         match = r.match(op)
         assert match, "line %r does not match regexp" % (op,)
         opname = match.group(1)
-        if   opname == 'int_add': value += int(match.group(2))
-        elif opname == 'int_sub': value -= int(match.group(2))
-        elif opname == 'int_and': value &= int(match.group(2))
-        else: assert 0, opname
+        if opname == 'int_and':
+            value &= int(match.group(2))
+        elif opname == 'int_signext':
+            numbytes = int(match.group(2))
+            value = int_signext(value, numbytes)
+        else:
+            assert 0, opname
     #
     assert rffi.cast(lltype.Signed, value) == expected_value
diff --git a/rpython/jit/metainterp/blackhole.py b/rpython/jit/metainterp/blackhole.py
--- a/rpython/jit/metainterp/blackhole.py
+++ b/rpython/jit/metainterp/blackhole.py
@@ -489,6 +489,9 @@
         if i < 0:
             return 0
         return i
+    @arguments("i", "i", returns="i")
+    def bhimpl_int_signext(a, b):
+        return heaptracker.int_signext(a, b)
 
     @arguments("i", "i", returns="i")
     def bhimpl_uint_lt(a, b):
diff --git a/rpython/jit/metainterp/pyjitpl.py b/rpython/jit/metainterp/pyjitpl.py
--- a/rpython/jit/metainterp/pyjitpl.py
+++ b/rpython/jit/metainterp/pyjitpl.py
@@ -197,7 +197,7 @@
     # ------------------------------
 
     for _opimpl in ['int_add', 'int_sub', 'int_mul', 'int_floordiv', 'int_mod',
-                    'int_and', 'int_or', 'int_xor',
+                    'int_and', 'int_or', 'int_xor', 'int_signext',
                     'int_rshift', 'int_lshift', 'uint_rshift',
                     'uint_lt', 'uint_le', 'uint_gt', 'uint_ge',
                     'uint_floordiv',
diff --git a/rpython/jit/metainterp/resoperation.py b/rpython/jit/metainterp/resoperation.py
--- a/rpython/jit/metainterp/resoperation.py
+++ b/rpython/jit/metainterp/resoperation.py
@@ -419,6 +419,7 @@
     'INT_RSHIFT/2',
     'INT_LSHIFT/2',
     'UINT_RSHIFT/2',
+    'INT_SIGNEXT/2',
     'FLOAT_ADD/2',
     'FLOAT_SUB/2',
     'FLOAT_MUL/2',
diff --git a/rpython/jit/metainterp/test/test_ajit.py b/rpython/jit/metainterp/test/test_ajit.py
--- a/rpython/jit/metainterp/test/test_ajit.py
+++ b/rpython/jit/metainterp/test/test_ajit.py
@@ -3048,6 +3048,16 @@
         res = self.meta_interp(f, [32])
         assert res == f(32)
 
+    def test_int_signext(self):
+        def f(n):
+            return rffi.cast(rffi.SIGNEDCHAR, n)
+        res = self.interp_operations(f, [128])
+        assert res == -128
+        res = self.interp_operations(f, [-35 + 256 * 29])
+        assert res == -35
+        res = self.interp_operations(f, [127 - 256 * 29])
+        assert res == 127
+
 class BaseLLtypeTests(BasicTests):
 
     def test_identityhash(self):

From noreply at buildbot.pypy.org  Wed Nov 26 18:49:03 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Wed, 26 Nov 2014 18:49:03 +0100 (CET)
Subject: [pypy-commit] pypy default: Change cffi's primitive integer type to
 use int_signext (via type casts)
Message-ID: <20141126174903.C5A291D26BD@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r74739:4af7cfa6d69c
Date: 2014-11-26 18:48 +0100
http://bitbucket.org/pypy/pypy/changeset/4af7cfa6d69c/

Log:	Change cffi's primitive integer type to use int_signext (via type
	casts)

diff --git a/pypy/module/_cffi_backend/ctypeprim.py b/pypy/module/_cffi_backend/ctypeprim.py
--- a/pypy/module/_cffi_backend/ctypeprim.py
+++ b/pypy/module/_cffi_backend/ctypeprim.py
@@ -158,21 +158,14 @@
 
 
 class W_CTypePrimitiveSigned(W_CTypePrimitive):
-    _attrs_            = ['value_fits_long', 'vmin', 'vrangemax']
-    _immutable_fields_ = ['value_fits_long', 'vmin', 'vrangemax']
+    _attrs_            = ['value_fits_long', 'value_smaller_than_long']
+    _immutable_fields_ = ['value_fits_long', 'value_smaller_than_long']
     is_primitive_integer = True
 
     def __init__(self, *args):
         W_CTypePrimitive.__init__(self, *args)
         self.value_fits_long = self.size <= rffi.sizeof(lltype.Signed)
-        if self.size < rffi.sizeof(lltype.Signed):
-            assert self.value_fits_long
-            sh = self.size * 8
-            self.vmin = r_uint(-1) << (sh - 1)
-            self.vrangemax = (r_uint(1) << sh) - 1
-        else:
-            self.vmin = r_uint(0)
-            self.vrangemax = r_uint(-1)
+        self.value_smaller_than_long = self.size < rffi.sizeof(lltype.Signed)
 
     def cast_to_int(self, cdata):
         return self.convert_to_object(cdata)
@@ -192,8 +185,17 @@
     def convert_from_object(self, cdata, w_ob):
         if self.value_fits_long:
             value = misc.as_long(self.space, w_ob)
-            if self.size < rffi.sizeof(lltype.Signed):
-                if r_uint(value) - self.vmin > self.vrangemax:
+            if self.value_smaller_than_long:
+                size = self.size
+                if size == 1:
+                    signextended = misc.signext(value, 1)
+                elif size == 2:
+                    signextended = misc.signext(value, 2)
+                elif size == 4:
+                    signextended = misc.signext(value, 4)
+                else:
+                    raise AssertionError("unsupported size")
+                if value != signextended:
                     self._overflow(w_ob)
             misc.write_raw_signed_data(cdata, value, self.size)
         else:
@@ -221,7 +223,7 @@
             length = w_cdata.get_array_length()
             populate_list_from_raw_array(res, buf, length)
             return res
-        elif self.value_fits_long:
+        elif self.value_smaller_than_long:
             res = [0] * w_cdata.get_array_length()
             misc.unpack_list_from_raw_array(res, w_cdata._cdata, self.size)
             return res
@@ -235,8 +237,8 @@
                 cdata = rffi.cast(rffi.LONGP, cdata)
                 copy_list_to_raw_array(int_list, cdata)
             else:
-                overflowed = misc.pack_list_to_raw_array_bounds(
-                    int_list, cdata, self.size, self.vmin, self.vrangemax)
+                overflowed = misc.pack_list_to_raw_array_bounds_signed(
+                    int_list, cdata, self.size)
                 if overflowed != 0:
                     self._overflow(self.space.wrap(overflowed))
             return True
@@ -314,8 +316,8 @@
     def pack_list_of_items(self, cdata, w_ob):
         int_list = self.space.listview_int(w_ob)
         if int_list is not None:
-            overflowed = misc.pack_list_to_raw_array_bounds(
-                int_list, cdata, self.size, r_uint(0), self.vrangemax)
+            overflowed = misc.pack_list_to_raw_array_bounds_unsigned(
+                int_list, cdata, self.size, self.vrangemax)
             if overflowed != 0:
                 self._overflow(self.space.wrap(overflowed))
             return True
diff --git a/pypy/module/_cffi_backend/misc.py b/pypy/module/_cffi_backend/misc.py
--- a/pypy/module/_cffi_backend/misc.py
+++ b/pypy/module/_cffi_backend/misc.py
@@ -216,6 +216,19 @@
 neg_msg = "can't convert negative number to unsigned"
 ovf_msg = "long too big to convert"
 
+ at specialize.arg(1)
+def signext(value, size):
+    # 'value' is sign-extended from 'size' bytes to a full integer.
+    # 'size' should be a constant smaller than a full integer size.
+    if size == rffi.sizeof(rffi.SIGNEDCHAR):
+        return rffi.cast(lltype.Signed, rffi.cast(rffi.SIGNEDCHAR, value))
+    elif size == rffi.sizeof(rffi.SHORT):
+        return rffi.cast(lltype.Signed, rffi.cast(rffi.SHORT, value))
+    elif size == rffi.sizeof(rffi.INT):
+        return rffi.cast(lltype.Signed, rffi.cast(rffi.INT, value))
+    else:
+        raise AssertionError("unsupported size")
+
 # ____________________________________________________________
 
 class _NotStandardObject(Exception):
@@ -334,13 +347,26 @@
 
 # ____________________________________________________________
 
-def pack_list_to_raw_array_bounds(int_list, target, size, vmin, vrangemax):
+def pack_list_to_raw_array_bounds_signed(int_list, target, size):
     for TP, TPP in _prim_signed_types:
         if size == rffi.sizeof(TP):
             ptr = rffi.cast(TPP, target)
             for i in range(len(int_list)):
                 x = int_list[i]
-                if r_uint(x) - vmin > vrangemax:
+                y = rffi.cast(TP, x)
+                if x != rffi.cast(lltype.Signed, y):
+                    return x      # overflow
+                ptr[i] = y
+            return 0
+    raise NotImplementedError("bad integer size")
+
+def pack_list_to_raw_array_bounds_unsigned(int_list, target, size, vrangemax):
+    for TP, TPP in _prim_signed_types:
+        if size == rffi.sizeof(TP):
+            ptr = rffi.cast(TPP, target)
+            for i in range(len(int_list)):
+                x = int_list[i]
+                if r_uint(x) > vrangemax:
                     return x      # overflow
                 ptr[i] = rffi.cast(TP, x)
             return 0
diff --git a/pypy/module/pypyjit/test_pypy_c/test_ffi.py b/pypy/module/pypyjit/test_pypy_c/test_ffi.py
--- a/pypy/module/pypyjit/test_pypy_c/test_ffi.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_ffi.py
@@ -340,13 +340,11 @@
         guard_value(p166, ConstPtr(ptr72), descr=...)
         p167 = call(ConstClass(_ll_0_alloc_with_del___), descr=)
         guard_no_exception(descr=...)
-        i112 = int_sub(i160, -32768)
+        i112 = int_signext(i160, 2)
         setfield_gc(p167, ConstPtr(null), descr=)
         setfield_gc(p167, ConstPtr(ptr85), descr=)
-        i114 = uint_gt(i112, 65535)
+        i114 = int_ne(i112, i160)
         guard_false(i114, descr=...)
-        i115 = int_and(i112, 65535)
-        i116 = int_add(i115, -32768)
         --TICK--
         i119 = call(ConstClass(_ll_1_raw_malloc_varsize__Signed), 6, descr=)
         raw_store(i119, 0, i116, descr=)

From noreply at buildbot.pypy.org  Wed Nov 26 20:13:27 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Wed, 26 Nov 2014 20:13:27 +0100 (CET)
Subject: [pypy-commit] pypy portable-threadlocal: fix test
Message-ID: <20141126191327.F16901C0EF6@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: portable-threadlocal
Changeset: r74740:cd80e75755dc
Date: 2014-11-26 20:13 +0100
http://bitbucket.org/pypy/pypy/changeset/cd80e75755dc/

Log:	fix test

diff --git a/pypy/module/pypyjit/test_pypy_c/model.py b/pypy/module/pypyjit/test_pypy_c/model.py
--- a/pypy/module/pypyjit/test_pypy_c/model.py
+++ b/pypy/module/pypyjit/test_pypy_c/model.py
@@ -184,10 +184,10 @@
         matcher = OpMatcher(ops)
         return matcher.match(expected_src, **kwds)
 
-    def match_by_id(self, id, expected_src, **kwds):
+    def match_by_id(self, id, expected_src, ignore_ops=[], **kwds):
         ops = list(self.ops_by_id(id, **kwds))
         matcher = OpMatcher(ops, id)
-        return matcher.match(expected_src)
+        return matcher.match(expected_src, ignore_ops=ignore_ops)
 
 class PartialTraceWithIds(TraceWithIds):
     def __init__(self, trace, is_entry_bridge=False):

From noreply at buildbot.pypy.org  Wed Nov 26 22:27:22 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Wed, 26 Nov 2014 22:27:22 +0100 (CET)
Subject: [pypy-commit] pypy default: Wrong instruction.
Message-ID: <20141126212722.17CE21C34C4@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r74741:af4cfbb2548c
Date: 2014-11-26 22:27 +0100
http://bitbucket.org/pypy/pypy/changeset/af4cfbb2548c/

Log:	Wrong instruction.

diff --git a/rpython/jit/backend/arm/codebuilder.py b/rpython/jit/backend/arm/codebuilder.py
--- a/rpython/jit/backend/arm/codebuilder.py
+++ b/rpython/jit/backend/arm/codebuilder.py
@@ -324,9 +324,9 @@
                     | (rd & 0xF) << 12
                     | (rm & 0xF))
 
-    def SXTB16_rr(self, rd, rm, c=cond.AL):
+    def SXTH_rr(self, rd, rm, c=cond.AL):
         self.write32(c << 28
-                    | 0x068F0070
+                    | 0x06BF0070
                     | (rd & 0xF) << 12
                     | (rm & 0xF))
 
diff --git a/rpython/jit/backend/arm/opassembler.py b/rpython/jit/backend/arm/opassembler.py
--- a/rpython/jit/backend/arm/opassembler.py
+++ b/rpython/jit/backend/arm/opassembler.py
@@ -108,7 +108,7 @@
         if numbytes.value == 1:
             self.mc.SXTB_rr(res.value, arg.value)
         elif numbytes.value == 2:
-            self.mc.SXTB16_rr(res.value, arg.value)
+            self.mc.SXTH_rr(res.value, arg.value)
         else:
             raise AssertionError("bad number of bytes")
         return fcond

From noreply at buildbot.pypy.org  Thu Nov 27 00:17:20 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 27 Nov 2014 00:17:20 +0100 (CET)
Subject: [pypy-commit] pypy portable-threadlocal: document as_key() and add
 one for RawSPStackLocation, needed now
Message-ID: <20141126231720.CC4FF1C0399@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: portable-threadlocal
Changeset: r74742:a80d4cea2d6e
Date: 2014-11-27 00:17 +0100
http://bitbucket.org/pypy/pypy/changeset/a80d4cea2d6e/

Log:	document as_key() and add one for RawSPStackLocation, needed now

diff --git a/rpython/jit/backend/arm/locations.py b/rpython/jit/backend/arm/locations.py
--- a/rpython/jit/backend/arm/locations.py
+++ b/rpython/jit/backend/arm/locations.py
@@ -46,7 +46,7 @@
     def is_core_reg(self):
         return True
 
-    def as_key(self):
+    def as_key(self):       # 0 <= as_key <= 15
         return self.value
 
 
@@ -64,7 +64,7 @@
     def is_vfp_reg(self):
         return True
 
-    def as_key(self):
+    def as_key(self):            # 20 <= as_key <= 35
         return self.value + 20
 
     def is_float(self):
@@ -115,8 +115,8 @@
     def is_imm_float(self):
         return True
 
-    def as_key(self):
-        return self.value
+    def as_key(self):          # a real address + 1
+        return self.value | 1
 
     def is_float(self):
         return True
@@ -148,7 +148,7 @@
     def is_stack(self):
         return True
 
-    def as_key(self):
+    def as_key(self):                # an aligned word + 10000
         return self.position + 10000
 
     def is_float(self):
@@ -174,6 +174,9 @@
     def is_float(self):
         return self.type == FLOAT
 
+    def as_key(self):            # a word >= 1000, and < 1000 + size of SP frame
+        return self.value + 1000
+
 
 def imm(i):
     return ImmLocation(i)

From noreply at buildbot.pypy.org  Thu Nov 27 02:57:39 2014
From: noreply at buildbot.pypy.org (stefanor)
Date: Thu, 27 Nov 2014 02:57:39 +0100 (CET)
Subject: [pypy-commit] pypy default: Debian gcc 4.9.2 seems to be emmiting
 some nb variant instructions
Message-ID: <20141127015739.CE7281C0EF6@cobra.cs.uni-duesseldorf.de>

Author: Stefano Rivera 
Branch: 
Changeset: r74743:c1abec418acf
Date: 2014-11-26 17:57 -0800
http://bitbucket.org/pypy/pypy/changeset/c1abec418acf/

Log:	Debian gcc 4.9.2 seems to be emmiting some nb variant instructions

diff --git a/rpython/translator/c/gcc/trackgcroot.py b/rpython/translator/c/gcc/trackgcroot.py
--- a/rpython/translator/c/gcc/trackgcroot.py
+++ b/rpython/translator/c/gcc/trackgcroot.py
@@ -590,7 +590,7 @@
 
     # The various cmov* operations
     for name in '''
-        e ne g ge l le a ae b be p np s ns o no
+        e ne g ge l le a ae b be nb p np s ns o no
         '''.split():
         locals()['visit_cmov' + name] = binary_insn
         locals()['visit_cmov' + name + 'l'] = binary_insn
@@ -837,6 +837,7 @@
     visit_jb = conditional_jump
     visit_jbe = conditional_jump
     visit_jp = conditional_jump
+    visit_jnb = conditional_jump
     visit_jnp = conditional_jump
     visit_js = conditional_jump
     visit_jns = conditional_jump

From noreply at buildbot.pypy.org  Thu Nov 27 09:09:47 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 27 Nov 2014 09:09:47 +0100 (CET)
Subject: [pypy-commit] pypy portable-threadlocal: next fix
Message-ID: <20141127080947.2AAA41C0399@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: portable-threadlocal
Changeset: r74744:834b260fe0df
Date: 2014-11-27 09:09 +0100
http://bitbucket.org/pypy/pypy/changeset/834b260fe0df/

Log:	next fix

diff --git a/rpython/jit/backend/arm/assembler.py b/rpython/jit/backend/arm/assembler.py
--- a/rpython/jit/backend/arm/assembler.py
+++ b/rpython/jit/backend/arm/assembler.py
@@ -1152,6 +1152,14 @@
         else:
             assert 0, 'unsupported case'
 
+    def _mov_raw_sp_to_loc(self, prev_loc, loc, cond=c.AL):
+        if loc.is_core_reg():
+            # load a value from 'SP + n'
+            assert prev_loc.value <= 0xFFF     # not too far
+            self.load_reg(self.mc, loc.value, r.sp, prev_loc.value, cond=cond)
+        else:
+            assert 0, 'unsupported case'
+
     def regalloc_mov(self, prev_loc, loc, cond=c.AL):
         """Moves a value from a previous location to some other location"""
         if prev_loc.is_imm():
@@ -1165,7 +1173,7 @@
         elif prev_loc.is_vfp_reg():
             self._mov_vfp_reg_to_loc(prev_loc, loc, cond)
         elif prev_loc.is_raw_sp():
-            assert 0, 'raw sp locs are not supported as source loc'
+            self._mov_raw_sp_to_loc(prev_loc, loc, cond)
         else:
             assert 0, 'unsupported case'
     mov_loc_loc = regalloc_mov

From noreply at buildbot.pypy.org  Thu Nov 27 09:16:15 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 27 Nov 2014 09:16:15 +0100 (CET)
Subject: [pypy-commit] pypy portable-threadlocal: fix
Message-ID: <20141127081615.969AA1C3145@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: portable-threadlocal
Changeset: r74745:4c60d918693c
Date: 2014-11-27 09:15 +0100
http://bitbucket.org/pypy/pypy/changeset/4c60d918693c/

Log:	fix

diff --git a/rpython/jit/backend/arm/assembler.py b/rpython/jit/backend/arm/assembler.py
--- a/rpython/jit/backend/arm/assembler.py
+++ b/rpython/jit/backend/arm/assembler.py
@@ -1156,7 +1156,7 @@
         if loc.is_core_reg():
             # load a value from 'SP + n'
             assert prev_loc.value <= 0xFFF     # not too far
-            self.load_reg(self.mc, loc.value, r.sp, prev_loc.value, cond=cond)
+            self.load_reg(self.mc, loc, r.sp, prev_loc.value, cond=cond)
         else:
             assert 0, 'unsupported case'
 

From noreply at buildbot.pypy.org  Thu Nov 27 10:56:14 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 27 Nov 2014 10:56:14 +0100 (CET)
Subject: [pypy-commit] pypy portable-threadlocal: ready for merge. (Windows
	status unknown.)
Message-ID: <20141127095614.504331C1366@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: portable-threadlocal
Changeset: r74746:b75c43d35041
Date: 2014-11-27 10:55 +0100
http://bitbucket.org/pypy/pypy/changeset/b75c43d35041/

Log:	ready for merge. (Windows status unknown.)


From noreply at buildbot.pypy.org  Thu Nov 27 10:56:16 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 27 Nov 2014 10:56:16 +0100 (CET)
Subject: [pypy-commit] pypy default: hg merge portable-threadlocal
Message-ID: <20141127095616.9904A1C1366@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r74747:adc6ab4ae74d
Date: 2014-11-27 10:56 +0100
http://bitbucket.org/pypy/pypy/changeset/adc6ab4ae74d/

Log:	hg merge portable-threadlocal

	Change the way thread-locals are read from the JIT: found a way to
	do it portably, by passing around the pointer to a thread-local
	structure from outside the JIT and all the way inside.

diff too long, truncating to 2000 out of 2070 lines

diff --git a/pypy/module/_ssl/thread_lock.py b/pypy/module/_ssl/thread_lock.py
--- a/pypy/module/_ssl/thread_lock.py
+++ b/pypy/module/_ssl/thread_lock.py
@@ -24,12 +24,19 @@
 
 separate_module_source = """
 #include 
+#ifndef _WIN32
+# include 
+#endif
 
 static unsigned int _ssl_locks_count = 0;
 static struct RPyOpaque_ThreadLock *_ssl_locks;
 
 static unsigned long _ssl_thread_id_function(void) {
-    return RPyThreadGetIdent();
+#ifdef _WIN32
+    return (unsigned long)GetCurrentThreadId();
+#else
+    return (unsigned long)pthread_self();
+#endif
 }
 
 static void _ssl_thread_locking_function(int mode, int n, const char *file,
diff --git a/pypy/module/cpyext/src/pythread.c b/pypy/module/cpyext/src/pythread.c
--- a/pypy/module/cpyext/src/pythread.c
+++ b/pypy/module/cpyext/src/pythread.c
@@ -1,11 +1,18 @@
 #include 
+#ifndef _WIN32
+# include 
+#endif
 #include "pythread.h"
 #include "src/thread.h"
 
 long
 PyThread_get_thread_ident(void)
 {
-    return RPyThreadGetIdent();
+#ifdef _WIN32
+    return (long)GetCurrentThreadId();
+#else
+    return (long)pthread_self();
+#endif
 }
 
 PyThread_type_lock
diff --git a/pypy/module/pypyjit/test_pypy_c/model.py b/pypy/module/pypyjit/test_pypy_c/model.py
--- a/pypy/module/pypyjit/test_pypy_c/model.py
+++ b/pypy/module/pypyjit/test_pypy_c/model.py
@@ -184,10 +184,10 @@
         matcher = OpMatcher(ops)
         return matcher.match(expected_src, **kwds)
 
-    def match_by_id(self, id, expected_src, **kwds):
+    def match_by_id(self, id, expected_src, ignore_ops=[], **kwds):
         ops = list(self.ops_by_id(id, **kwds))
         matcher = OpMatcher(ops, id)
-        return matcher.match(expected_src)
+        return matcher.match(expected_src, ignore_ops=ignore_ops)
 
 class PartialTraceWithIds(TraceWithIds):
     def __init__(self, trace, is_entry_bridge=False):
diff --git a/pypy/module/pypyjit/test_pypy_c/test_call.py b/pypy/module/pypyjit/test_pypy_c/test_call.py
--- a/pypy/module/pypyjit/test_pypy_c/test_call.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_call.py
@@ -82,7 +82,7 @@
         assert log.opnames(ops) == []
         #
         assert entry_bridge.match_by_id('call', """
-            p38 = call(ConstClass(_ll_0_threadlocalref_getter___), descr=)
+            p38 = call(ConstClass(_ll_1_threadlocalref_get__Ptr_GcStruct_objectLlT_Signed), #, descr=)
             p39 = getfield_gc(p38, descr=)
             i40 = force_token()
             p41 = getfield_gc_pure(p38, descr=)
@@ -444,7 +444,7 @@
             p26 = getfield_gc(p7, descr=)
             guard_value(p26, ConstPtr(ptr27), descr=...)
             guard_not_invalidated(descr=...)
-            p29 = call(ConstClass(_ll_0_threadlocalref_getter___), descr=)
+            p29 = call(ConstClass(_ll_1_threadlocalref_get__Ptr_GcStruct_objectLlT_Signed), #, descr=)
             p30 = getfield_gc(p29, descr=)
             p31 = force_token()
             p32 = getfield_gc_pure(p29, descr=)
diff --git a/pypy/module/pypyjit/test_pypy_c/test_ffi.py b/pypy/module/pypyjit/test_pypy_c/test_ffi.py
--- a/pypy/module/pypyjit/test_pypy_c/test_ffi.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_ffi.py
@@ -199,21 +199,16 @@
         ldexp_addr, res = log.result
         assert res == 8.0 * 300
         loop, = log.loops_by_filename(self.filepath)
-        if 'ConstClass(ldexp)' in repr(loop):   # e.g. OS/X
-            ldexp_addr = 'ConstClass(ldexp)'
         assert loop.match_by_id('cfficall', """
-            ...
-            f1 = call_release_gil(..., descr=)
-            ...
-        """)
-        ops = loop.ops_by_id('cfficall')
-        for name in ['raw_malloc', 'raw_free']:
-            assert name not in str(ops)
-        for name in ['raw_load', 'raw_store', 'getarrayitem_raw', 'setarrayitem_raw']:
-            assert name not in log.opnames(ops)
-        # so far just check that call_release_gil() is produced.
-        # later, also check that the arguments to call_release_gil()
-        # are constants
+            setarrayitem_raw(i69, 0, i95, descr=)    # write 'errno'
+            p96 = force_token()
+            setfield_gc(p0, p96, descr=)
+            f97 = call_release_gil(i59, 1.0, 3, descr=)
+            guard_not_forced(descr=...)
+            guard_no_exception(descr=...)
+            i98 = getarrayitem_raw(i69, 0, descr=)   # read 'errno'
+            setfield_gc(p65, i98, descr=)
+        """, ignore_ops=['guard_not_invalidated'])
 
     def test_cffi_call_guard_not_forced_fails(self):
         # this is the test_pypy_c equivalent of
diff --git a/rpython/annotator/bookkeeper.py b/rpython/annotator/bookkeeper.py
--- a/rpython/annotator/bookkeeper.py
+++ b/rpython/annotator/bookkeeper.py
@@ -65,6 +65,7 @@
         self.external_class_cache = {}      # cache of ExternalType classes
 
         self.needs_generic_instantiate = {}
+        self.thread_local_fields = set()
 
         delayed_imports()
 
diff --git a/rpython/jit/backend/arm/assembler.py b/rpython/jit/backend/arm/assembler.py
--- a/rpython/jit/backend/arm/assembler.py
+++ b/rpython/jit/backend/arm/assembler.py
@@ -497,9 +497,11 @@
         if self.cpu.supports_floats:
             mc.VPOP([reg.value for reg in r.callee_saved_vfp_registers],
                                                                     cond=cond)
-        # pop all callee saved registers and IP to keep the alignment
+        # pop all callee saved registers.  This pops 'pc' last.
+        # It also pops the threadlocal_addr back into 'r1', but it
+        # is not needed any more and will be discarded.
         mc.POP([reg.value for reg in r.callee_restored_registers] +
-                                                       [r.ip.value], cond=cond)
+                                                       [r.r1.value], cond=cond)
         mc.BKPT()
 
     def gen_func_prolog(self):
@@ -508,11 +510,16 @@
         if self.cpu.supports_floats:
             stack_size += len(r.callee_saved_vfp_registers) * 2 * WORD
 
-        # push all callee saved registers and IP to keep the alignment
+        # push all callee saved registers including lr; and push r1 as
+        # well, which contains the threadlocal_addr argument.  Note that
+        # we're pushing a total of 10 words, which keeps the stack aligned.
         self.mc.PUSH([reg.value for reg in r.callee_saved_registers] +
-                                                        [r.ip.value])
+                                                        [r.r1.value])
+        self.saved_threadlocal_addr = 0   # at offset 0 from location 'sp'
         if self.cpu.supports_floats:
             self.mc.VPUSH([reg.value for reg in r.callee_saved_vfp_registers])
+            self.saved_threadlocal_addr += (
+                len(r.callee_saved_vfp_registers) * 2 * WORD)
         assert stack_size % 8 == 0 # ensure we keep alignment
 
         # set fp to point to the JITFRAME
@@ -952,16 +959,11 @@
             regalloc._check_invariants()
         self.mc.mark_op(None)  # end of the loop
 
-    def regalloc_emit_llong(self, op, arglocs, fcond, regalloc):
+    def regalloc_emit_extra(self, op, arglocs, fcond, regalloc):
+        # for calls to a function with a specifically-supported OS_xxx
         effectinfo = op.getdescr().get_extra_info()
         oopspecindex = effectinfo.oopspecindex
-        asm_llong_operations[oopspecindex](self, op, arglocs, regalloc, fcond)
-        return fcond
-
-    def regalloc_emit_math(self, op, arglocs, fcond, regalloc):
-        effectinfo = op.getdescr().get_extra_info()
-        oopspecindex = effectinfo.oopspecindex
-        asm_math_operations[oopspecindex](self, op, arglocs, regalloc, fcond)
+        asm_extra_operations[oopspecindex](self, op, arglocs, regalloc, fcond)
         return fcond
 
     def patch_trace(self, faildescr, looptoken, bridge_addr, regalloc):
@@ -1150,6 +1152,14 @@
         else:
             assert 0, 'unsupported case'
 
+    def _mov_raw_sp_to_loc(self, prev_loc, loc, cond=c.AL):
+        if loc.is_core_reg():
+            # load a value from 'SP + n'
+            assert prev_loc.value <= 0xFFF     # not too far
+            self.load_reg(self.mc, loc, r.sp, prev_loc.value, cond=cond)
+        else:
+            assert 0, 'unsupported case'
+
     def regalloc_mov(self, prev_loc, loc, cond=c.AL):
         """Moves a value from a previous location to some other location"""
         if prev_loc.is_imm():
@@ -1163,7 +1173,7 @@
         elif prev_loc.is_vfp_reg():
             self._mov_vfp_reg_to_loc(prev_loc, loc, cond)
         elif prev_loc.is_raw_sp():
-            assert 0, 'raw sp locs are not supported as source loc'
+            self._mov_raw_sp_to_loc(prev_loc, loc, cond)
         else:
             assert 0, 'unsupported case'
     mov_loc_loc = regalloc_mov
@@ -1509,22 +1519,17 @@
 
 asm_operations = [notimplemented_op] * (rop._LAST + 1)
 asm_operations_with_guard = [notimplemented_op_with_guard] * (rop._LAST + 1)
-asm_llong_operations = {}
-asm_math_operations = {}
+asm_extra_operations = {}
 
 for name, value in ResOpAssembler.__dict__.iteritems():
     if name.startswith('emit_guard_'):
         opname = name[len('emit_guard_'):]
         num = getattr(rop, opname.upper())
         asm_operations_with_guard[num] = value
-    elif name.startswith('emit_op_llong_'):
-        opname = name[len('emit_op_llong_'):]
-        num = getattr(EffectInfo, 'OS_LLONG_' + opname.upper())
-        asm_llong_operations[num] = value
-    elif name.startswith('emit_op_math_'):
-        opname = name[len('emit_op_math_'):]
-        num = getattr(EffectInfo, 'OS_MATH_' + opname.upper())
-        asm_math_operations[num] = value
+    elif name.startswith('emit_opx_'):
+        opname = name[len('emit_opx_'):]
+        num = getattr(EffectInfo, 'OS_' + opname.upper())
+        asm_extra_operations[num] = value
     elif name.startswith('emit_op_'):
         opname = name[len('emit_op_'):]
         num = getattr(rop, opname.upper())
diff --git a/rpython/jit/backend/arm/locations.py b/rpython/jit/backend/arm/locations.py
--- a/rpython/jit/backend/arm/locations.py
+++ b/rpython/jit/backend/arm/locations.py
@@ -46,7 +46,7 @@
     def is_core_reg(self):
         return True
 
-    def as_key(self):
+    def as_key(self):       # 0 <= as_key <= 15
         return self.value
 
 
@@ -64,7 +64,7 @@
     def is_vfp_reg(self):
         return True
 
-    def as_key(self):
+    def as_key(self):            # 20 <= as_key <= 35
         return self.value + 20
 
     def is_float(self):
@@ -115,8 +115,8 @@
     def is_imm_float(self):
         return True
 
-    def as_key(self):
-        return self.value
+    def as_key(self):          # a real address + 1
+        return self.value | 1
 
     def is_float(self):
         return True
@@ -148,7 +148,7 @@
     def is_stack(self):
         return True
 
-    def as_key(self):
+    def as_key(self):                # an aligned word + 10000
         return self.position + 10000
 
     def is_float(self):
@@ -174,6 +174,9 @@
     def is_float(self):
         return self.type == FLOAT
 
+    def as_key(self):            # a word >= 1000, and < 1000 + size of SP frame
+        return self.value + 1000
+
 
 def imm(i):
     return ImmLocation(i)
diff --git a/rpython/jit/backend/arm/opassembler.py b/rpython/jit/backend/arm/opassembler.py
--- a/rpython/jit/backend/arm/opassembler.py
+++ b/rpython/jit/backend/arm/opassembler.py
@@ -19,7 +19,7 @@
 from rpython.jit.backend.arm.codebuilder import InstrBuilder, OverwritingBuilder
 from rpython.jit.backend.arm.jump import remap_frame_layout
 from rpython.jit.backend.arm.regalloc import TempBox
-from rpython.jit.backend.arm.locations import imm
+from rpython.jit.backend.arm.locations import imm, RawSPStackLocation
 from rpython.jit.backend.llsupport import symbolic
 from rpython.jit.backend.llsupport.gcmap import allocate_gcmap
 from rpython.jit.backend.llsupport.descr import InteriorFieldDescr
@@ -982,7 +982,9 @@
         return fcond
 
     def _call_assembler_emit_call(self, addr, argloc, resloc):
-        self.simple_call(addr, [argloc], result_loc=resloc)
+        ofs = self.saved_threadlocal_addr
+        threadlocal_loc = RawSPStackLocation(ofs, INT)
+        self.simple_call(addr, [argloc, threadlocal_loc], result_loc=resloc)
 
     def _call_assembler_emit_helper_call(self, addr, arglocs, resloc):
         self.simple_call(addr, arglocs, result_loc=resloc)
@@ -1108,7 +1110,7 @@
 
     emit_op_float_neg = gen_emit_unary_float_op('float_neg', 'VNEG')
     emit_op_float_abs = gen_emit_unary_float_op('float_abs', 'VABS')
-    emit_op_math_sqrt = gen_emit_unary_float_op('math_sqrt', 'VSQRT')
+    emit_opx_math_sqrt = gen_emit_unary_float_op('math_sqrt', 'VSQRT')
 
     emit_op_float_lt = gen_emit_float_cmp_op('float_lt', c.VFP_LT)
     emit_op_float_le = gen_emit_float_cmp_op('float_le', c.VFP_LE)
@@ -1142,13 +1144,13 @@
 
     # the following five instructions are only ARMv7;
     # regalloc.py won't call them at all on ARMv6
-    emit_op_llong_add = gen_emit_float_op('llong_add', 'VADD_i64')
-    emit_op_llong_sub = gen_emit_float_op('llong_sub', 'VSUB_i64')
-    emit_op_llong_and = gen_emit_float_op('llong_and', 'VAND_i64')
-    emit_op_llong_or = gen_emit_float_op('llong_or', 'VORR_i64')
-    emit_op_llong_xor = gen_emit_float_op('llong_xor', 'VEOR_i64')
+    emit_opx_llong_add = gen_emit_float_op('llong_add', 'VADD_i64')
+    emit_opx_llong_sub = gen_emit_float_op('llong_sub', 'VSUB_i64')
+    emit_opx_llong_and = gen_emit_float_op('llong_and', 'VAND_i64')
+    emit_opx_llong_or = gen_emit_float_op('llong_or', 'VORR_i64')
+    emit_opx_llong_xor = gen_emit_float_op('llong_xor', 'VEOR_i64')
 
-    def emit_op_llong_to_int(self, op, arglocs, regalloc, fcond):
+    def emit_opx_llong_to_int(self, op, arglocs, regalloc, fcond):
         loc = arglocs[0]
         res = arglocs[1]
         assert loc.is_vfp_reg()
@@ -1282,3 +1284,11 @@
             regalloc.rm.possibly_free_var(length_box)
         regalloc.rm.possibly_free_var(dstaddr_box)
         return fcond
+
+    def emit_opx_threadlocalref_get(self, op, arglocs, regalloc, fcond):
+        ofs0, res = arglocs
+        assert ofs0.is_imm()
+        ofs = self.saved_threadlocal_addr
+        self.load_reg(self.mc, res, r.sp, ofs)
+        self.load_reg(self.mc, res, res, ofs0.value)
+        return fcond
diff --git a/rpython/jit/backend/arm/regalloc.py b/rpython/jit/backend/arm/regalloc.py
--- a/rpython/jit/backend/arm/regalloc.py
+++ b/rpython/jit/backend/arm/regalloc.py
@@ -373,11 +373,8 @@
         return gcmap
 
     # ------------------------------------------------------------
-    def perform_llong(self, op, args, fcond):
-        return self.assembler.regalloc_emit_llong(op, args, fcond, self)
-
-    def perform_math(self, op, args, fcond):
-        return self.assembler.regalloc_emit_math(op, args, self, fcond)
+    def perform_extra(self, op, args, fcond):
+        return self.assembler.regalloc_emit_extra(op, args, fcond, self)
 
     def force_spill_var(self, var):
         if var.type == FLOAT:
@@ -558,15 +555,19 @@
                             EffectInfo.OS_LLONG_XOR):
                 if self.cpu.cpuinfo.arch_version >= 7:
                     args = self._prepare_llong_binop_xx(op, fcond)
-                    self.perform_llong(op, args, fcond)
+                    self.perform_extra(op, args, fcond)
                     return
             elif oopspecindex == EffectInfo.OS_LLONG_TO_INT:
                 args = self._prepare_llong_to_int(op, fcond)
-                self.perform_llong(op, args, fcond)
+                self.perform_extra(op, args, fcond)
                 return
             elif oopspecindex == EffectInfo.OS_MATH_SQRT:
-                args = self.prepare_op_math_sqrt(op, fcond)
-                self.perform_math(op, args, fcond)
+                args = self._prepare_op_math_sqrt(op, fcond)
+                self.perform_extra(op, args, fcond)
+                return
+            elif oopspecindex == EffectInfo.OS_THREADLOCALREF_GET:
+                args = self._prepare_threadlocalref_get(op, fcond)
+                self.perform_extra(op, args, fcond)
                 return
             #elif oopspecindex == EffectInfo.OS_MATH_READ_TIMESTAMP:
             #    ...
@@ -624,6 +625,11 @@
         res = self.force_allocate_reg(op.result)
         return [loc0, res]
 
+    def _prepare_threadlocalref_get(self, op, fcond):
+        ofs0 = imm(op.getarg(1).getint())
+        res = self.force_allocate_reg(op.result)
+        return [ofs0, res]
+
     def _prepare_guard(self, op, args=None):
         if args is None:
             args = []
@@ -1284,7 +1290,7 @@
     prepare_guard_float_ge = prepare_float_op(guard=True,
                             float_result=False, name='prepare_guard_float_ge')
 
-    def prepare_op_math_sqrt(self, op, fcond):
+    def _prepare_op_math_sqrt(self, op, fcond):
         loc = self.make_sure_var_in_reg(op.getarg(1))
         self.possibly_free_vars_for_op(op)
         self.free_temp_vars()
diff --git a/rpython/jit/backend/llsupport/llmodel.py b/rpython/jit/backend/llsupport/llmodel.py
--- a/rpython/jit/backend/llsupport/llmodel.py
+++ b/rpython/jit/backend/llsupport/llmodel.py
@@ -217,7 +217,13 @@
         return lltype.cast_opaque_ptr(llmemory.GCREF, frame)
 
     def make_execute_token(self, *ARGS):
-        FUNCPTR = lltype.Ptr(lltype.FuncType([llmemory.GCREF],
+        # The JIT backend must generate functions with the following
+        # signature: it takes the jitframe and the threadlocal_addr
+        # as arguments, and it returns the (possibly reallocated) jitframe.
+        # The backend can optimize OS_THREADLOCALREF_GET calls to return a
+        # field of this threadlocal_addr, but only if 'translate_support_code':
+        # in untranslated tests, threadlocal_addr is a dummy NULL.
+        FUNCPTR = lltype.Ptr(lltype.FuncType([llmemory.GCREF, llmemory.Address],
                                              llmemory.GCREF))
 
         lst = [(i, history.getkind(ARG)[0]) for i, ARG in enumerate(ARGS)]
@@ -249,8 +255,13 @@
                     else:
                         assert kind == history.REF
                         self.set_ref_value(ll_frame, num, arg)
+                if self.translate_support_code:
+                    ll_threadlocal_addr = llop.threadlocalref_addr(
+                        llmemory.Address)
+                else:
+                    ll_threadlocal_addr = llmemory.NULL
                 llop.gc_writebarrier(lltype.Void, ll_frame)
-                ll_frame = func(ll_frame)
+                ll_frame = func(ll_frame, ll_threadlocal_addr)
             finally:
                 if not self.translate_support_code:
                     LLInterpreter.current_interpreter = prev_interpreter
diff --git a/rpython/jit/backend/llsupport/test/ztranslation_test.py b/rpython/jit/backend/llsupport/test/ztranslation_test.py
--- a/rpython/jit/backend/llsupport/test/ztranslation_test.py
+++ b/rpython/jit/backend/llsupport/test/ztranslation_test.py
@@ -26,8 +26,6 @@
         # - profiler
         # - full optimizer
         # - floats neg and abs
-        # - threadlocalref_get
-        # - get_errno, set_errno
         # - llexternal with macro=True
 
         class Frame(object):
@@ -36,10 +34,6 @@
             def __init__(self, i):
                 self.i = i
 
-        class Foo(object):
-            pass
-        t = ThreadLocalReference(Foo)
-
         eci = ExternalCompilationInfo(post_include_bits=['''
 #define pypy_my_fabs(x)  fabs(x)
 '''])
@@ -74,9 +68,6 @@
                 k = myabs1(myabs2(j))
                 if k - abs(j):  raise ValueError
                 if k - abs(-j): raise ValueError
-                if t.get().nine != 9: raise ValueError
-                rposix.set_errno(total)
-                if rposix.get_errno() != total: raise ValueError
             return chr(total % 253)
         #
         class Virt2(object):
@@ -104,12 +95,8 @@
             return res
         #
         def main(i, j):
-            foo = Foo()
-            foo.nine = -(i + j)
-            t.set(foo)
             a_char = f(i, j)
             a_float = libffi_stuff(i, j)
-            keepalive_until_here(foo)
             return ord(a_char) * 10 + int(a_float)
         expected = main(40, -49)
         res = self.meta_interp(main, [40, -49])
@@ -121,6 +108,7 @@
 
     def test_direct_assembler_call_translates(self):
         """Test CALL_ASSEMBLER and the recursion limit"""
+        # - also tests threadlocalref_get
         from rpython.rlib.rstackovf import StackOverflow
 
         class Thing(object):
@@ -138,6 +126,10 @@
 
         somewhere_else = SomewhereElse()
 
+        class Foo(object):
+            pass
+        t = ThreadLocalReference(Foo)
+
         def change(newthing):
             somewhere_else.frame.thing = newthing
 
@@ -163,6 +155,7 @@
                     nextval = 13
                 frame.thing = Thing(nextval + 1)
                 i += 1
+                if t.get().nine != 9: raise ValueError
             return frame.thing.val
 
         driver2 = JitDriver(greens = [], reds = ['n'])
@@ -184,13 +177,24 @@
                 n = portal2(n)
         assert portal2(10) == -9
 
+        def setup(value):
+            foo = Foo()
+            foo.nine = value
+            t.set(foo)
+            return foo
+
         def mainall(codeno, bound):
-            return main(codeno) + main2(bound)
+            foo = setup(bound + 8)
+            result = main(codeno) + main2(bound)
+            keepalive_until_here(foo)
+            return result
 
+        tmp_obj = setup(9)
+        expected_1 = main(0)
         res = self.meta_interp(mainall, [0, 1], inline=True,
                                policy=StopAtXPolicy(change))
         print hex(res)
-        assert res & 255 == main(0)
+        assert res & 255 == expected_1
         bound = res & ~255
         assert 1024 <= bound <= 131072
         assert bound & (bound-1) == 0       # a power of two
diff --git a/rpython/jit/backend/x86/arch.py b/rpython/jit/backend/x86/arch.py
--- a/rpython/jit/backend/x86/arch.py
+++ b/rpython/jit/backend/x86/arch.py
@@ -34,10 +34,16 @@
     FRAME_FIXED_SIZE = 19
     PASS_ON_MY_FRAME = 15
     JITFRAME_FIXED_SIZE = 6 + 8 * 2 # 6 GPR + 8 XMM * 2 WORDS/float
+    # 'threadlocal_addr' is passed as 2nd argument on the stack,
+    # and it can be left here for when it is needed
+    THREADLOCAL_OFS = (FRAME_FIXED_SIZE + 2) * WORD
 else:
-    # rbp + rbx + r12 + r13 + r14 + r15 + 13 extra words = 19
+    # rbp + rbx + r12 + r13 + r14 + r15 + threadlocal + 12 extra words = 19
     FRAME_FIXED_SIZE = 19
-    PASS_ON_MY_FRAME = 13
+    PASS_ON_MY_FRAME = 12
     JITFRAME_FIXED_SIZE = 28 # 13 GPR + 15 XMM
+    # 'threadlocal_addr' is passed as 2nd argument in %esi,
+    # and is moved into this frame location
+    THREADLOCAL_OFS = (FRAME_FIXED_SIZE - 1) * WORD
 
 assert PASS_ON_MY_FRAME >= 12       # asmgcc needs at least JIT_USE_WORDS + 3
diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py
--- a/rpython/jit/backend/x86/assembler.py
+++ b/rpython/jit/backend/x86/assembler.py
@@ -18,7 +18,7 @@
 from rpython.jit.backend.llsupport.regalloc import (get_scale, valid_addressing_size)
 from rpython.jit.backend.x86.arch import (FRAME_FIXED_SIZE, WORD, IS_X86_64,
                                        JITFRAME_FIXED_SIZE, IS_X86_32,
-                                       PASS_ON_MY_FRAME)
+                                       PASS_ON_MY_FRAME, THREADLOCAL_OFS)
 from rpython.jit.backend.x86.regloc import (eax, ecx, edx, ebx, esp, ebp, esi,
     xmm0, xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7, r8, r9, r10, r11, edi,
     r12, r13, r14, r15, X86_64_SCRATCH_REG, X86_64_XMM_SCRATCH_REG,
@@ -730,6 +730,7 @@
         self.mc.SUB_ri(esp.value, FRAME_FIXED_SIZE * WORD)
         self.mc.MOV_sr(PASS_ON_MY_FRAME * WORD, ebp.value)
         if IS_X86_64:
+            self.mc.MOV_sr(THREADLOCAL_OFS, esi.value)
             self.mc.MOV_rr(ebp.value, edi.value)
         else:
             self.mc.MOV_rs(ebp.value, (FRAME_FIXED_SIZE + 1) * WORD)
@@ -1969,7 +1970,8 @@
         self._emit_guard_not_forced(guard_token)
 
     def _call_assembler_emit_call(self, addr, argloc, _):
-        self.simple_call(addr, [argloc])
+        threadlocal_loc = RawEspLoc(THREADLOCAL_OFS, INT)
+        self.simple_call(addr, [argloc, threadlocal_loc])
 
     def _call_assembler_emit_helper_call(self, addr, arglocs, result_loc):
         self.simple_call(addr, arglocs, result_loc)
@@ -2334,48 +2336,16 @@
         assert isinstance(reg, RegLoc)
         self.mc.MOV_rr(reg.value, ebp.value)
 
-    def threadlocalref_get(self, op, resloc):
-        # this function is only called on Linux
-        from rpython.jit.codewriter.jitcode import ThreadLocalRefDescr
-        from rpython.jit.backend.x86 import stmtlocal
+    def threadlocalref_get(self, offset, resloc):
+        # This loads the stack location THREADLOCAL_OFS into a
+        # register, and then read the word at the given offset.
+        # It is only supported if 'translate_support_code' is
+        # true; otherwise, the original call to the piece of assembler
+        # was done with a dummy NULL value.
+        assert self.cpu.translate_support_code
         assert isinstance(resloc, RegLoc)
-        effectinfo = op.getdescr().get_extra_info()
-        assert effectinfo.extradescrs is not None
-        ed = effectinfo.extradescrs[0]
-        assert isinstance(ed, ThreadLocalRefDescr)
-        addr1 = rffi.cast(lltype.Signed, ed.get_tlref_addr())
-        # 'addr1' is the address is the current thread, but we assume that
-        # it is a thread-local at a constant offset from %fs/%gs.
-        addr0 = stmtlocal.threadlocal_base()
-        addr = addr1 - addr0
-        assert rx86.fits_in_32bits(addr)
-        mc = self.mc
-        mc.writechar(stmtlocal.SEGMENT_TL)     # prefix: %fs or %gs
-        mc.MOV_rj(resloc.value, addr)          # memory read
-
-    def get_set_errno(self, op, loc, issue_a_write):
-        # this function is only called on Linux
-        from rpython.jit.backend.x86 import stmtlocal
-        addr = stmtlocal.get_errno_tl()
-        assert rx86.fits_in_32bits(addr)
-        mc = self.mc
-        mc.writechar(stmtlocal.SEGMENT_TL)     # prefix: %fs or %gs
-        # !!important: the *next* instruction must be the one using 'addr'!!
-        if issue_a_write:
-            if isinstance(loc, RegLoc):
-                mc.MOV32_jr(addr, loc.value)       # memory write from reg
-            else:
-                assert isinstance(loc, ImmedLoc)
-                newvalue = loc.value
-                newvalue = rffi.cast(rffi.INT, newvalue)
-                newvalue = rffi.cast(lltype.Signed, newvalue)
-                mc.MOV32_ji(addr, newvalue)        # memory write immediate
-        else:
-            assert isinstance(loc, RegLoc)
-            if IS_X86_32:
-                mc.MOV_rj(loc.value, addr)         # memory read
-            elif IS_X86_64:
-                mc.MOVSX32_rj(loc.value, addr)     # memory read, sign-extend
+        self.mc.MOV_rs(resloc.value, THREADLOCAL_OFS)
+        self.mc.MOV_rm(resloc.value, (resloc.value, offset))
 
     def genop_discard_zero_array(self, op, arglocs):
         (base_loc, startindex_loc, bytes_loc,
diff --git a/rpython/jit/backend/x86/regalloc.py b/rpython/jit/backend/x86/regalloc.py
--- a/rpython/jit/backend/x86/regalloc.py
+++ b/rpython/jit/backend/x86/regalloc.py
@@ -699,29 +699,11 @@
         loc0 = self.xrm.force_result_in_reg(op.result, op.getarg(1))
         self.perform_math(op, [loc0], loc0)
 
-    TLREF_SUPPORT = sys.platform.startswith('linux')
-    ERRNO_SUPPORT = sys.platform.startswith('linux')
-
     def _consider_threadlocalref_get(self, op):
-        if self.TLREF_SUPPORT:
+        if self.translate_support_code:
+            offset = op.getarg(1).getint()   # getarg(0) == 'threadlocalref_get'
             resloc = self.force_allocate_reg(op.result)
-            self.assembler.threadlocalref_get(op, resloc)
-        else:
-            self._consider_call(op)
-
-    def _consider_get_errno(self, op):
-        if self.ERRNO_SUPPORT:
-            resloc = self.force_allocate_reg(op.result)
-            self.assembler.get_set_errno(op, resloc, issue_a_write=False)
-        else:
-            self._consider_call(op)
-
-    def _consider_set_errno(self, op):
-        if self.ERRNO_SUPPORT:
-            # op.getarg(0) is the function set_errno; op.getarg(1) is
-            # the new errno value
-            loc0 = self.rm.make_sure_var_in_reg(op.getarg(1))
-            self.assembler.get_set_errno(op, loc0, issue_a_write=True)
+            self.assembler.threadlocalref_get(offset, resloc)
         else:
             self._consider_call(op)
 
@@ -804,10 +786,6 @@
                 return self._consider_math_sqrt(op)
             if oopspecindex == EffectInfo.OS_THREADLOCALREF_GET:
                 return self._consider_threadlocalref_get(op)
-            if oopspecindex == EffectInfo.OS_GET_ERRNO:
-                return self._consider_get_errno(op)
-            if oopspecindex == EffectInfo.OS_SET_ERRNO:
-                return self._consider_set_errno(op)
             if oopspecindex == EffectInfo.OS_MATH_READ_TIMESTAMP:
                 return self._consider_math_read_timestamp(op)
         self._consider_call(op)
diff --git a/rpython/jit/backend/x86/stmtlocal.py b/rpython/jit/backend/x86/stmtlocal.py
deleted file mode 100644
--- a/rpython/jit/backend/x86/stmtlocal.py
+++ /dev/null
@@ -1,43 +0,0 @@
-from rpython.rtyper.lltypesystem import lltype, rffi
-from rpython.translator.tool.cbuild import ExternalCompilationInfo
-from rpython.jit.backend.x86.arch import WORD
-
-SEGMENT_FS = '\x64'
-SEGMENT_GS = '\x65'
-
-if WORD == 4:
-    SEGMENT_TL = SEGMENT_GS
-    _instruction = "movl %%gs:0, %0"
-else:
-    SEGMENT_TL = SEGMENT_FS
-    _instruction = "movq %%fs:0, %0"
-
-eci = ExternalCompilationInfo(post_include_bits=['''
-#define RPY_STM_JIT  1
-static long pypy__threadlocal_base(void)
-{
-    /* XXX ONLY LINUX WITH GCC/CLANG FOR NOW XXX */
-    long result;
-    asm("%s" : "=r"(result));
-    return result;
-}
-static long pypy__get_errno_tl(void)
-{
-    return ((long)&errno) - pypy__threadlocal_base();
-}
-''' % _instruction])
-
-
-threadlocal_base = rffi.llexternal(
-    'pypy__threadlocal_base',
-    [], lltype.Signed,
-    compilation_info=eci,
-    _nowrapper=True,
-    ) #transactionsafe=True)
-
-get_errno_tl = rffi.llexternal(
-    'pypy__get_errno_tl',
-    [], lltype.Signed,
-    compilation_info=eci,
-    _nowrapper=True,
-    ) #transactionsafe=True)
diff --git a/rpython/jit/codewriter/effectinfo.py b/rpython/jit/codewriter/effectinfo.py
--- a/rpython/jit/codewriter/effectinfo.py
+++ b/rpython/jit/codewriter/effectinfo.py
@@ -23,8 +23,6 @@
     OS_SHRINK_ARRAY             = 3    # rgc.ll_shrink_array
     OS_DICT_LOOKUP              = 4    # ll_dict_lookup
     OS_THREADLOCALREF_GET       = 5    # llop.threadlocalref_get
-    OS_GET_ERRNO                = 6    # rposix.get_errno
-    OS_SET_ERRNO                = 7    # rposix.set_errno
     OS_NOT_IN_TRACE             = 8    # for calls not recorded in the jit trace
     #
     OS_STR_CONCAT               = 22   # "stroruni.concat"
diff --git a/rpython/jit/codewriter/jitcode.py b/rpython/jit/codewriter/jitcode.py
--- a/rpython/jit/codewriter/jitcode.py
+++ b/rpython/jit/codewriter/jitcode.py
@@ -117,26 +117,6 @@
         raise NotImplementedError
 
 
-class ThreadLocalRefDescr(AbstractDescr):
-    # A special descr used as the extradescr in a call to a
-    # threadlocalref_get function.  If the backend supports it,
-    # it can use this 'get_tlref_addr()' to get the address *in the
-    # current thread* of the thread-local variable.  If, on the current
-    # platform, the "__thread" variables are implemented as an offset
-    # from some base register (e.g. %fs on x86-64), then the backend will
-    # immediately substract the current value of the base register.
-    # This gives an offset from the base register, and this can be
-    # written down in an assembler instruction to load the "__thread"
-    # variable from anywhere.
-
-    def __init__(self, opaque_id):
-        from rpython.rtyper.lltypesystem.lloperation import llop
-        from rpython.rtyper.lltypesystem import llmemory
-        def get_tlref_addr():
-            return llop.threadlocalref_getaddr(llmemory.Address, opaque_id)
-        self.get_tlref_addr = get_tlref_addr
-
-
 class LiveVarsInfo(object):
     def __init__(self, live_i, live_r, live_f):
         self.live_i = live_i
diff --git a/rpython/jit/codewriter/jtransform.py b/rpython/jit/codewriter/jtransform.py
--- a/rpython/jit/codewriter/jtransform.py
+++ b/rpython/jit/codewriter/jtransform.py
@@ -439,8 +439,6 @@
         elif oopspec_name.endswith('dict.lookup'):
             # also ordereddict.lookup
             prepare = self._handle_dict_lookup_call
-        elif oopspec_name.startswith('rposix.'):
-            prepare = self._handle_rposix_call
         else:
             prepare = self.prepare_builtin_call
         try:
@@ -1979,16 +1977,6 @@
         else:
             raise NotImplementedError(oopspec_name)
 
-    def _handle_rposix_call(self, op, oopspec_name, args):
-        if oopspec_name == 'rposix.get_errno':
-            return self._handle_oopspec_call(op, args, EffectInfo.OS_GET_ERRNO,
-                                             EffectInfo.EF_CANNOT_RAISE)
-        elif oopspec_name == 'rposix.set_errno':
-            return self._handle_oopspec_call(op, args, EffectInfo.OS_SET_ERRNO,
-                                             EffectInfo.EF_CANNOT_RAISE)
-        else:
-            raise NotImplementedError(oopspec_name)
-
     def rewrite_op_ll_read_timestamp(self, op):
         op1 = self.prepare_builtin_call(op, "ll_read_timestamp", [])
         return self.handle_residual_call(op1,
@@ -2005,16 +1993,15 @@
         return [op0, op1]
 
     def rewrite_op_threadlocalref_get(self, op):
-        from rpython.jit.codewriter.jitcode import ThreadLocalRefDescr
-        opaqueid = op.args[0].value
-        op1 = self.prepare_builtin_call(op, 'threadlocalref_getter', [],
-                                        extra=(opaqueid,),
-                                        extrakey=opaqueid._obj)
-        extradescr = ThreadLocalRefDescr(opaqueid)
+        # only supports RESTYPE being exactly one word.
+        RESTYPE = op.result.concretetype
+        assert (RESTYPE in (lltype.Signed, lltype.Unsigned, llmemory.Address)
+                or isinstance(RESTYPE, lltype.Ptr))
+        c_offset, = op.args
+        op1 = self.prepare_builtin_call(op, 'threadlocalref_get', [c_offset])
         return self.handle_residual_call(op1,
             oopspecindex=EffectInfo.OS_THREADLOCALREF_GET,
-            extraeffect=EffectInfo.EF_LOOPINVARIANT,
-            extradescr=[extradescr])
+            extraeffect=EffectInfo.EF_LOOPINVARIANT)
 
 # ____________________________________________________________
 
diff --git a/rpython/jit/codewriter/support.py b/rpython/jit/codewriter/support.py
--- a/rpython/jit/codewriter/support.py
+++ b/rpython/jit/codewriter/support.py
@@ -702,10 +702,9 @@
     build_ll_1_raw_free_no_track_allocation = (
         build_raw_free_builder(track_allocation=False))
 
-    def build_ll_0_threadlocalref_getter(opaqueid):
-        def _ll_0_threadlocalref_getter():
-            return llop.threadlocalref_get(rclass.OBJECTPTR, opaqueid)
-        return _ll_0_threadlocalref_getter
+    def _ll_1_threadlocalref_get(TP, offset):
+        return llop.threadlocalref_get(TP, offset)
+    _ll_1_threadlocalref_get.need_result_type = 'exact'   # don't deref
 
     def _ll_1_weakref_create(obj):
         return llop.weakref_create(llmemory.WeakRefPtr, obj)
@@ -818,8 +817,18 @@
     s_result = lltype_to_annotation(ll_res)
     impl = setup_extra_builtin(rtyper, oopspec_name, len(args_s), extra)
     if getattr(impl, 'need_result_type', False):
-        bk = rtyper.annotator.bookkeeper
-        args_s.insert(0, annmodel.SomePBC([bk.getdesc(deref(ll_res))]))
+        if hasattr(rtyper, 'annotator'):
+            bk = rtyper.annotator.bookkeeper
+            ll_restype = ll_res
+            if impl.need_result_type != 'exact':
+                ll_restype = deref(ll_restype)
+            desc = bk.getdesc(ll_restype)
+        else:
+            class TestingDesc(object):
+                knowntype = int
+                pyobj = None
+            desc = TestingDesc()
+        args_s.insert(0, annmodel.SomePBC([desc]))
     #
     if hasattr(rtyper, 'annotator'):  # regular case
         mixlevelann = MixLevelHelperAnnotator(rtyper)
diff --git a/rpython/jit/codewriter/test/test_jtransform.py b/rpython/jit/codewriter/test/test_jtransform.py
--- a/rpython/jit/codewriter/test/test_jtransform.py
+++ b/rpython/jit/codewriter/test/test_jtransform.py
@@ -148,9 +148,7 @@
              EI.OS_UNIEQ_LENGTHOK:       ([PUNICODE, PUNICODE], INT),
              EI.OS_RAW_MALLOC_VARSIZE_CHAR: ([INT], ARRAYPTR),
              EI.OS_RAW_FREE:             ([ARRAYPTR], lltype.Void),
-             EI.OS_THREADLOCALREF_GET:   ([], rclass.OBJECTPTR),
-             EI.OS_GET_ERRNO:            ([], INT),
-             EI.OS_SET_ERRNO:            ([INT], lltype.Void),
+             EI.OS_THREADLOCALREF_GET:   ([INT], INT),   # for example
             }
             argtypes = argtypes[oopspecindex]
             assert argtypes[0] == [v.concretetype for v in op.args[1:]]
@@ -159,9 +157,7 @@
                 assert extraeffect == EI.EF_ELIDABLE_CAN_RAISE
             elif oopspecindex == EI.OS_RAW_MALLOC_VARSIZE_CHAR:
                 assert extraeffect == EI.EF_CAN_RAISE
-            elif oopspecindex in (EI.OS_RAW_FREE,
-                                  EI.OS_GET_ERRNO,
-                                  EI.OS_SET_ERRNO):
+            elif oopspecindex == EI.OS_RAW_FREE:
                 assert extraeffect == EI.EF_CANNOT_RAISE
             elif oopspecindex == EI.OS_THREADLOCALREF_GET:
                 assert extraeffect == EI.EF_LOOPINVARIANT
@@ -1347,53 +1343,20 @@
     assert op2 is None
 
 def test_threadlocalref_get():
-    from rpython.rtyper import rclass
-    from rpython.rlib.rthread import ThreadLocalReference
+    from rpython.rlib.rthread import ThreadLocalField
+    tlfield = ThreadLocalField(lltype.Signed, 'foobar_test_')
     OS_THREADLOCALREF_GET = effectinfo.EffectInfo.OS_THREADLOCALREF_GET
-    class Foo: pass
-    t = ThreadLocalReference(Foo)
-    v2 = varoftype(rclass.OBJECTPTR)
-    c_opaqueid = const(t.opaque_id)
-    op = SpaceOperation('threadlocalref_get', [c_opaqueid], v2)
+    c = const(tlfield.offset)
+    v = varoftype(lltype.Signed)
+    op = SpaceOperation('threadlocalref_get', [c], v)
     tr = Transformer(FakeCPU(), FakeBuiltinCallControl())
     op0 = tr.rewrite_operation(op)
-    assert op0.opname == 'residual_call_r_r'
-    assert op0.args[0].value == 'threadlocalref_getter' # pseudo-function as str
-    assert op0.args[1] == ListOfKind("ref", [])
-    assert op0.args[2] == 'calldescr-%d' % OS_THREADLOCALREF_GET
-    assert op0.result == v2
-
-def test_get_errno():
-    # test that the oopspec is present and correctly transformed
-    from rpython.rlib import rposix
-    FUNC = lltype.FuncType([], lltype.Signed)
-    func = lltype.functionptr(FUNC, 'get_errno', _callable=rposix.get_errno)
-    v3 = varoftype(lltype.Signed)
-    op = SpaceOperation('direct_call', [const(func)], v3)
-    tr = Transformer(FakeCPU(), FakeBuiltinCallControl())
-    op1 = tr.rewrite_operation(op)
-    assert op1.opname == 'residual_call_r_i'
-    assert op1.args[0].value == func
-    assert op1.args[1] == ListOfKind('ref', [])
-    assert op1.args[2] == 'calldescr-%d' % effectinfo.EffectInfo.OS_GET_ERRNO
-    assert op1.result == v3
-
-def test_set_errno():
-    # test that the oopspec is present and correctly transformed
-    from rpython.rlib import rposix
-    FUNC = lltype.FuncType([lltype.Signed], lltype.Void)
-    func = lltype.functionptr(FUNC, 'set_errno', _callable=rposix.set_errno)
-    v1 = varoftype(lltype.Signed)
-    v3 = varoftype(lltype.Void)
-    op = SpaceOperation('direct_call', [const(func), v1], v3)
-    tr = Transformer(FakeCPU(), FakeBuiltinCallControl())
-    op1 = tr.rewrite_operation(op)
-    assert op1.opname == 'residual_call_ir_v'
-    assert op1.args[0].value == func
-    assert op1.args[1] == ListOfKind('int', [v1])
-    assert op1.args[2] == ListOfKind('ref', [])
-    assert op1.args[3] == 'calldescr-%d' % effectinfo.EffectInfo.OS_SET_ERRNO
-    assert op1.result == v3
+    assert op0.opname == 'residual_call_ir_i'
+    assert op0.args[0].value == 'threadlocalref_get' # pseudo-function as str
+    assert op0.args[1] == ListOfKind("int", [c])
+    assert op0.args[2] == ListOfKind("ref", [])
+    assert op0.args[3] == 'calldescr-%d' % OS_THREADLOCALREF_GET
+    assert op0.result == v
 
 def test_unknown_operation():
     op = SpaceOperation('foobar', [], varoftype(lltype.Void))
diff --git a/rpython/jit/metainterp/test/test_threadlocal.py b/rpython/jit/metainterp/test/test_threadlocal.py
--- a/rpython/jit/metainterp/test/test_threadlocal.py
+++ b/rpython/jit/metainterp/test/test_threadlocal.py
@@ -1,29 +1,20 @@
 import py
+from rpython.rlib import rthread
 from rpython.jit.metainterp.test.support import LLJitMixin
-from rpython.rlib.rthread import ThreadLocalReference
-from rpython.rlib.jit import dont_look_inside
+from rpython.rtyper.lltypesystem import lltype
+from rpython.rtyper.lltypesystem.lloperation import llop
 
 
 class ThreadLocalTest(object):
 
     def test_threadlocalref_get(self):
-        class Foo:
-            pass
-        t = ThreadLocalReference(Foo)
-        x = Foo()
-
-        @dont_look_inside
-        def setup():
-            t.set(x)
+        tlfield = rthread.ThreadLocalField(lltype.Signed, 'foobar_test_')
 
         def f():
-            setup()
-            if t.get() is x:
-                return 42
-            return -666
+            return tlfield.getraw()
 
         res = self.interp_operations(f, [])
-        assert res == 42
+        assert res == 0x544c    # magic value returned by llinterp
 
 
 class TestLLtype(ThreadLocalTest, LLJitMixin):
diff --git a/rpython/memory/gctransform/framework.py b/rpython/memory/gctransform/framework.py
--- a/rpython/memory/gctransform/framework.py
+++ b/rpython/memory/gctransform/framework.py
@@ -1080,6 +1080,9 @@
             assert not livevars, "live GC var around %s!" % (hop.spaceop,)
             hop.genop("direct_call", [self.root_walker.thread_run_ptr])
             self.pop_roots(hop, livevars)
+        else:
+            hop.rename("gc_thread_run")     # keep it around for c/gc.py,
+                                            # unless handled specially above
 
     def gct_gc_thread_start(self, hop):
         assert self.translator.config.translation.thread
@@ -1095,6 +1098,7 @@
             assert not livevars, "live GC var around %s!" % (hop.spaceop,)
             hop.genop("direct_call", [self.root_walker.thread_die_ptr])
             self.pop_roots(hop, livevars)
+        hop.rename("gc_thread_die")     # keep it around for c/gc.py
 
     def gct_gc_thread_before_fork(self, hop):
         if (self.translator.config.translation.thread
diff --git a/rpython/memory/gctransform/shadowstack.py b/rpython/memory/gctransform/shadowstack.py
--- a/rpython/memory/gctransform/shadowstack.py
+++ b/rpython/memory/gctransform/shadowstack.py
@@ -132,8 +132,12 @@
             gcdata.root_stack_top/root_stack_base is the one corresponding
             to the current thread.
             No GC operation here, e.g. no mallocs or storing in a dict!
+
+            Note that here specifically we don't call rthread.get_ident(),
+            but rthread.get_or_make_ident().  We are possibly in a fresh
+            new thread, so we need to be careful.
             """
-            tid = get_tid()
+            tid = rthread.get_or_make_ident()
             if gcdata.active_tid != tid:
                 switch_shadow_stacks(tid)
 
diff --git a/rpython/rlib/rposix.py b/rpython/rlib/rposix.py
--- a/rpython/rlib/rposix.py
+++ b/rpython/rlib/rposix.py
@@ -95,12 +95,19 @@
 # the default wrapper for set_errno is not suitable for use in critical places
 # like around GIL handling logic, so we provide our own wrappers.
 
- at jit.oopspec("rposix.get_errno()")
 def get_errno():
+    if jit.we_are_jitted():
+        from rpython.rlib import rthread
+        perrno = rthread.tlfield_p_errno.getraw()
+        return intmask(perrno[0])
     return intmask(_get_errno())
 
- at jit.oopspec("rposix.set_errno(errno)")
 def set_errno(errno):
+    if jit.we_are_jitted():
+        from rpython.rlib import rthread
+        perrno = rthread.tlfield_p_errno.getraw()
+        perrno[0] = rffi.cast(INT, errno)
+        return
     _set_errno(rffi.cast(INT, errno))
 
 if os.name == 'nt':
diff --git a/rpython/rlib/rstack.py b/rpython/rlib/rstack.py
--- a/rpython/rlib/rstack.py
+++ b/rpython/rlib/rstack.py
@@ -1,6 +1,6 @@
 """
 This file defines utilities for manipulating the stack in an
-RPython-compliant way, intended mostly for use by the Stackless PyPy.
+RPython-compliant way.  It is mainly about the stack_check() function.
 """
 
 import py
@@ -10,18 +10,11 @@
 from rpython.rlib import rgc
 from rpython.rtyper.lltypesystem import lltype, rffi
 from rpython.rtyper.lltypesystem.lloperation import llop
-from rpython.translator import cdir
-from rpython.translator.tool.cbuild import ExternalCompilationInfo
 
 # ____________________________________________________________
 
-srcdir = py.path.local(cdir) / 'src'
-compilation_info = ExternalCompilationInfo(
-        includes=['src/stack.h'],
-        separate_module_files=[srcdir / 'stack.c', srcdir / 'threadlocal.c'])
-
 def llexternal(name, args, res, _callable=None):
-    return rffi.llexternal(name, args, res, compilation_info=compilation_info,
+    return rffi.llexternal(name, args, res,
                            sandboxsafe=True, _nowrapper=True,
                            _callable=_callable)
 
diff --git a/rpython/rlib/rthread.py b/rpython/rlib/rthread.py
--- a/rpython/rlib/rthread.py
+++ b/rpython/rlib/rthread.py
@@ -5,8 +5,10 @@
 from rpython.rlib import jit, rgc
 from rpython.rlib.debug import ll_assert
 from rpython.rlib.objectmodel import we_are_translated, specialize
+from rpython.rlib.objectmodel import CDefinedIntSymbolic
 from rpython.rtyper.lltypesystem.lloperation import llop
 from rpython.rtyper.tool import rffi_platform
+from rpython.rtyper.extregistry import ExtRegistryEntry
 
 class RThreadError(Exception):
     pass
@@ -40,8 +42,6 @@
                             releasegil=True)  # release the GIL, but most
                                               # importantly, reacquire it
                                               # around the callback
-c_thread_get_ident = llexternal('RPyThreadGetIdent', [], rffi.LONG,
-                                _nowrapper=True)    # always call directly
 
 TLOCKP = rffi.COpaquePtr('struct RPyOpaque_ThreadLock',
                           compilation_info=eci)
@@ -83,9 +83,16 @@
 
 # wrappers...
 
- at jit.loop_invariant
 def get_ident():
-    return rffi.cast(lltype.Signed, c_thread_get_ident())
+    if we_are_translated():
+        return tlfield_thread_ident.getraw()
+    else:
+        import thread
+        return thread.get_ident()
+
+def get_or_make_ident():
+    assert we_are_translated()
+    return tlfield_thread_ident.get_or_make_raw()
 
 @specialize.arg(0)
 def start_new_thread(x, y):
@@ -265,17 +272,40 @@
 # KEEP THE REFERENCE ALIVE, THE GC DOES NOT FOLLOW THEM SO FAR!
 # We use _make_sure_does_not_move() to make sure the pointer will not move.
 
-ecitl = ExternalCompilationInfo(
-    includes = ['src/threadlocal.h'],
-    separate_module_files = [translator_c_dir / 'src' / 'threadlocal.c'])
-ensure_threadlocal = rffi.llexternal_use_eci(ecitl)
 
-class ThreadLocalReference(object):
+class ThreadLocalField(object):
+    def __init__(self, FIELDTYPE, fieldname):
+        "NOT_RPYTHON: must be prebuilt"
+        self.FIELDTYPE = FIELDTYPE
+        self.fieldname = fieldname
+        offset = CDefinedIntSymbolic('RPY_TLOFS_%s' % self.fieldname,
+                                     default='?')
+        self.offset = offset
+
+        def getraw():
+            _threadlocalref_seeme(self)
+            return llop.threadlocalref_get(FIELDTYPE, offset)
+
+        def get_or_make_raw():
+            _threadlocalref_seeme(self)
+            addr = llop.threadlocalref_addr(llmemory.Address)
+            return llop.raw_load(FIELDTYPE, addr, offset)
+
+        def setraw(value):
+            _threadlocalref_seeme(self)
+            addr = llop.threadlocalref_addr(llmemory.Address)
+            llop.raw_store(lltype.Void, addr, offset, value)
+
+        self.getraw = getraw
+        self.get_or_make_raw = get_or_make_raw
+        self.setraw = setraw
+
+    def _freeze_(self):
+        return True
+
+
+class ThreadLocalReference(ThreadLocalField):
     _COUNT = 1
-    OPAQUEID = lltype.OpaqueType("ThreadLocalRef",
-                                 hints={"threadlocalref": True,
-                                        "external": "C",
-                                        "c_name": "RPyThreadStaticTLS"})
 
     def __init__(self, Cls):
         "NOT_RPYTHON: must be prebuilt"
@@ -284,15 +314,16 @@
         self.local = thread._local()      # <- NOT_RPYTHON
         unique_id = ThreadLocalReference._COUNT
         ThreadLocalReference._COUNT += 1
-        opaque_id = lltype.opaqueptr(ThreadLocalReference.OPAQUEID,
-                                     'tlref%d' % unique_id)
-        self.opaque_id = opaque_id
+        ThreadLocalField.__init__(self, lltype.Signed, 'tlref%d' % unique_id)
+        setraw = self.setraw
+        offset = self.offset
 
         def get():
             if we_are_translated():
                 from rpython.rtyper import rclass
                 from rpython.rtyper.annlowlevel import cast_base_ptr_to_instance
-                ptr = llop.threadlocalref_get(rclass.OBJECTPTR, opaque_id)
+                _threadlocalref_seeme(self)
+                ptr = llop.threadlocalref_get(rclass.OBJECTPTR, offset)
                 return cast_base_ptr_to_instance(Cls, ptr)
             else:
                 return getattr(self.local, 'value', None)
@@ -301,21 +332,34 @@
         def set(value):
             assert isinstance(value, Cls) or value is None
             if we_are_translated():
-                from rpython.rtyper.annlowlevel import cast_instance_to_base_ptr
+                from rpython.rtyper.annlowlevel import cast_instance_to_gcref
                 from rpython.rlib.rgc import _make_sure_does_not_move
                 from rpython.rlib.objectmodel import running_on_llinterp
-                ptr = cast_instance_to_base_ptr(value)
+                gcref = cast_instance_to_gcref(value)
                 if not running_on_llinterp:
-                    gcref = lltype.cast_opaque_ptr(llmemory.GCREF, ptr)
                     if gcref:
                         _make_sure_does_not_move(gcref)
-                llop.threadlocalref_set(lltype.Void, opaque_id, ptr)
-                ensure_threadlocal()
+                value = lltype.cast_ptr_to_int(gcref)
+                setraw(value)
             else:
                 self.local.value = value
 
         self.get = get
         self.set = set
 
-    def _freeze_(self):
-        return True
+
+tlfield_thread_ident = ThreadLocalField(lltype.Signed, "thread_ident")
+tlfield_p_errno = ThreadLocalField(rffi.CArrayPtr(rffi.INT), "p_errno")
+
+def _threadlocalref_seeme(field):
+    "NOT_RPYTHON"
+
+class _Entry(ExtRegistryEntry):
+    _about_ = _threadlocalref_seeme
+
+    def compute_result_annotation(self, s_field):
+        field = s_field.const
+        self.bookkeeper.thread_local_fields.add(field)
+
+    def specialize_call(self, hop):
+        hop.exception_cannot_occur()
diff --git a/rpython/rlib/test/test_rthread.py b/rpython/rlib/test/test_rthread.py
--- a/rpython/rlib/test/test_rthread.py
+++ b/rpython/rlib/test/test_rthread.py
@@ -47,6 +47,10 @@
     time.sleep(0.5)
     assert results == [True] * 15
 
+def test_get_ident():
+    import thread
+    assert get_ident() == thread.get_ident()
+
 
 class AbstractThreadTests(AbstractGCTestClass):
     use_threads = True
diff --git a/rpython/rtyper/llinterp.py b/rpython/rtyper/llinterp.py
--- a/rpython/rtyper/llinterp.py
+++ b/rpython/rtyper/llinterp.py
@@ -919,19 +919,14 @@
     def op_stack_current(self):
         return 0
 
-    def op_threadlocalref_set(self, key, value):
-        try:
-            d = self.llinterpreter.tlrefsdict
-        except AttributeError:
-            d = self.llinterpreter.tlrefsdict = {}
-        d[key._obj] = value
+    def op_threadlocalref_addr(self):
+        raise NotImplementedError("threadlocalref_addr")
 
-    def op_threadlocalref_get(self, key):
-        d = self.llinterpreter.tlrefsdict
-        return d[key._obj]
-
-    def op_threadlocalref_getaddr(self, key):
-        raise NotImplementedError("threadlocalref_getaddr")
+    def op_threadlocalref_get(self, offset):
+        if (type(offset) is CDefinedIntSymbolic and
+                offset.expr == 'RPY_TLOFS_foobar_test_'):   # used in tests
+            return 0x544c
+        raise NotImplementedError("threadlocalref_get")
 
     # __________________________________________________________
     # operations on addresses
@@ -978,6 +973,9 @@
             ll_p = rffi.cast(rffi.CArrayPtr(RESTYPE),
                              rffi.ptradd(ll_p, offset))
             value = ll_p[0]
+        ## elif getattr(addr, 'is_fake_thread_local_addr', False):
+        ##     assert type(offset) is CDefinedIntSymbolic
+        ##     value = self.llinterpreter.tlobj[offset.expr]
         else:
             assert offset.TYPE == RESTYPE
             value = getattr(addr, str(RESTYPE).lower())[offset.repeat]
@@ -998,6 +996,9 @@
             ll_p = rffi.cast(rffi.CArrayPtr(ARGTYPE),
                              rffi.ptradd(ll_p, offset))
             ll_p[0] = value
+        ## elif getattr(addr, 'is_fake_thread_local_addr', False):
+        ##     assert type(offset) is CDefinedIntSymbolic
+        ##     self.llinterpreter.tlobj[offset.expr] = value
         else:
             assert offset.TYPE == ARGTYPE
             getattr(addr, str(ARGTYPE).lower())[offset.repeat] = value
diff --git a/rpython/rtyper/lltypesystem/lloperation.py b/rpython/rtyper/lltypesystem/lloperation.py
--- a/rpython/rtyper/lltypesystem/lloperation.py
+++ b/rpython/rtyper/lltypesystem/lloperation.py
@@ -546,9 +546,8 @@
     'getslice':             LLOp(canraise=(Exception,)),
     'check_and_clear_exc':  LLOp(),
 
-    'threadlocalref_get':   LLOp(sideeffects=False),
-    'threadlocalref_getaddr': LLOp(sideeffects=False),
-    'threadlocalref_set':   LLOp(),
+    'threadlocalref_addr':  LLOp(sideeffects=False),  # get (or make) addr of tl
+    'threadlocalref_get':   LLOp(sideeffects=False),  # read field (no check)
 
     # __________ debugging __________
     'debug_view':           LLOp(),
diff --git a/rpython/rtyper/lltypesystem/test/test_llmemory.py b/rpython/rtyper/lltypesystem/test/test_llmemory.py
--- a/rpython/rtyper/lltypesystem/test/test_llmemory.py
+++ b/rpython/rtyper/lltypesystem/test/test_llmemory.py
@@ -649,3 +649,13 @@
     #assert cast_int_to_adr(i) == adr -- depends on ll2ctypes details
     i = cast_adr_to_int(NULL, mode="forced")
     assert is_valid_int(i) and i == 0
+
+def test_cast_gcref_to_int():
+    A = lltype.GcArray(Address)
+    def f():
+        ptr = lltype.malloc(A, 10)
+        gcref = lltype.cast_opaque_ptr(GCREF, ptr)
+        adr = lltype.cast_ptr_to_int(gcref)
+        assert adr == lltype.cast_ptr_to_int(ptr)
+    f()
+    interpret(f, [])
diff --git a/rpython/translator/c/funcgen.py b/rpython/translator/c/funcgen.py
--- a/rpython/translator/c/funcgen.py
+++ b/rpython/translator/c/funcgen.py
@@ -13,6 +13,7 @@
 from rpython.translator.backendopt.ssa import SSI_to_SSA
 from rpython.translator.backendopt.innerloop import find_inner_loops
 from rpython.tool.identity_dict import identity_dict
+from rpython.rlib.objectmodel import CDefinedIntSymbolic
 
 
 LOCALVAR = 'l_%s'
@@ -900,4 +901,21 @@
         else:
             return None    # use the default
 
+    def OP_THREADLOCALREF_GET(self, op):
+        typename = self.db.gettype(op.result.concretetype)
+        if isinstance(op.args[0], Constant):
+            assert isinstance(op.args[0].value, CDefinedIntSymbolic)
+            fieldname = op.args[0].value.expr
+            assert fieldname.startswith('RPY_TLOFS_')
+            fieldname = fieldname[10:]
+            return '%s = (%s)RPY_THREADLOCALREF_GET(%s);' % (
+                self.expr(op.result),
+                cdecl(typename, ''),
+                fieldname)
+        else:
+            return 'OP_THREADLOCALREF_GET_NONCONST(%s, %s, %s);' % (
+                cdecl(typename, ''),
+                self.expr(op.args[0]),
+                self.expr(op.result))
+
 assert not USESLOTS or '__dict__' not in dir(FunctionCodeGenerator)
diff --git a/rpython/translator/c/gc.py b/rpython/translator/c/gc.py
--- a/rpython/translator/c/gc.py
+++ b/rpython/translator/c/gc.py
@@ -71,13 +71,20 @@
         return ''
 
     def OP_GC_THREAD_RUN(self, funcgen, op):
-        return ''
+        # The gc transformer leaves this operation in the graphs
+        # in all cases except with framework+shadowstack.  In that
+        # case the operation is removed because redundant with
+        # rthread.get_or_make_ident().
+        return 'RPY_THREADLOCALREF_ENSURE();'
 
     def OP_GC_THREAD_START(self, funcgen, op):
         return ''
 
     def OP_GC_THREAD_DIE(self, funcgen, op):
-        return ''
+        # The gc transformer leaves this operation in the graphs
+        # (but may insert a call to a gcrootfinder-specific
+        # function just before).
+        return 'RPython_ThreadLocals_ThreadDie();'
 
     def OP_GC_THREAD_BEFORE_FORK(self, funcgen, op):
         return '%s = NULL;' % funcgen.expr(op.result)
diff --git a/rpython/translator/c/genc.py b/rpython/translator/c/genc.py
--- a/rpython/translator/c/genc.py
+++ b/rpython/translator/c/genc.py
@@ -703,8 +703,27 @@
     for node in structdeflist:
         for line in node.definition():
             print >> f, line
+    gen_threadlocal_structdef(f, database)
     print >> f, "#endif"
 
+def gen_threadlocal_structdef(f, database):
+    from rpython.translator.c.support import cdecl
+    print >> f
+    bk = database.translator.annotator.bookkeeper
+    fields = list(bk.thread_local_fields)
+    fields.sort(key=lambda field: field.fieldname)
+    for field in fields:
+        print >> f, ('#define RPY_TLOFS_%s  offsetof(' % field.fieldname +
+                     'struct pypy_threadlocal_s, %s)' % field.fieldname)
+    print >> f, 'struct pypy_threadlocal_s {'
+    print >> f, '\tint ready;'
+    print >> f, '\tchar *stack_end;'
+    for field in fields:
+        typename = database.gettype(field.FIELDTYPE)
+        print >> f, '\t%s;' % cdecl(typename, field.fieldname)
+    print >> f, '};'
+    print >> f
+
 def gen_forwarddecl(f, database):
     print >> f, '/***********************************************************/'
     print >> f, '/***  Forward declarations                               ***/'
@@ -730,6 +749,11 @@
     # generate the start-up code and put it into a function
     print >> f, 'char *RPython_StartupCode(void) {'
     print >> f, '\tchar *error = NULL;'
+
+    bk = database.translator.annotator.bookkeeper
+    if bk.thread_local_fields:
+        print >> f, '\tRPython_ThreadLocals_ProgramInit();'
+
     for line in database.gcpolicy.gc_startup_code():
         print >> f,"\t" + line
 
@@ -748,6 +772,7 @@
                 print >> f, '\tif (error) return error;'
             for line in lines:
                 print >> f, '\t'+line
+
     print >> f, '\treturn error;'
     print >> f, '}'
 
@@ -770,6 +795,8 @@
         srcdir / 'asm.c',
         srcdir / 'instrument.c',
         srcdir / 'int.c',
+        srcdir / 'stack.c',
+        srcdir / 'threadlocal.c',
     ]
     if _CYGWIN:
         files.append(srcdir / 'cygwin_wait.c')
diff --git a/rpython/translator/c/node.py b/rpython/translator/c/node.py
--- a/rpython/translator/c/node.py
+++ b/rpython/translator/c/node.py
@@ -966,30 +966,12 @@
                 args.append('0')
         yield 'RPyOpaque_SETUP_%s(%s);' % (T.tag, ', '.join(args))
 
-class ThreadLocalRefOpaqueNode(ContainerNode):
-    nodekind = 'tlrefopaque'
-
-    def basename(self):
-        return self.obj._name
-
-    def enum_dependencies(self):
-        return []
-
-    def initializationexpr(self, decoration=''):
-        return ['0']
-
-    def startupcode(self):
-        p = self.getptrname()
-        yield 'RPyThreadStaticTLS_Create(%s);' % (p,)
-
 
 def opaquenode_factory(db, T, obj):
     if T == RuntimeTypeInfo:
         return db.gcpolicy.rtti_node_factory()(db, T, obj)
     if T.hints.get("render_structure", False):
         return ExtType_OpaqueNode(db, T, obj)
-    if T.hints.get("threadlocalref", False):
-        return ThreadLocalRefOpaqueNode(db, T, obj)
     raise Exception("don't know about %r" % (T,))
 
 
diff --git a/rpython/translator/c/src/g_include.h b/rpython/translator/c/src/g_include.h
--- a/rpython/translator/c/src/g_include.h
+++ b/rpython/translator/c/src/g_include.h
@@ -19,6 +19,8 @@
 #include "src/address.h"
 #include "src/unichar.h"
 #include "src/llgroup.h"
+#include "src/stack.h"
+#include "src/threadlocal.h"
 
 #include "src/instrument.h"
 #include "src/asm.h"
diff --git a/rpython/translator/c/src/g_prerequisite.h b/rpython/translator/c/src/g_prerequisite.h
--- a/rpython/translator/c/src/g_prerequisite.h
+++ b/rpython/translator/c/src/g_prerequisite.h
@@ -23,6 +23,3 @@
 # define RPY_LENGTH0     1       /* array decl [0] are bad */
 # define RPY_DUMMY_VARLENGTH     /* nothing */
 #endif
-
-
-#include "src/threadlocal.h"
diff --git a/rpython/translator/c/src/stack.c b/rpython/translator/c/src/stack.c
--- a/rpython/translator/c/src/stack.c
+++ b/rpython/translator/c/src/stack.c
@@ -1,6 +1,8 @@
 /* Stack operation */
+#include "common_header.h"
+#include "structdef.h"       /* for struct pypy_threadlocal_s */
 #include 
-#include 
+#include 
 #include 
 
 
@@ -9,7 +11,6 @@
 char *_LLstacktoobig_stack_end = NULL;
 long _LLstacktoobig_stack_length = MAX_STACK_SIZE;
 char _LLstacktoobig_report_error = 1;
-static RPyThreadStaticTLS end_tls_key;
 
 void LL_stack_set_length_fraction(double fraction)
 {
@@ -20,6 +21,8 @@
 {
 	long diff, max_stack_size;
 	char *baseptr, *curptr = (char*)current;
+	char *tl;
+	struct pypy_threadlocal_s *tl1;
 
 	/* The stack_end variable is updated to match the current value
 	   if it is still 0 or if we later find a 'curptr' position
@@ -27,15 +30,9 @@
 	   thread-local storage, but we try to minimize its overhead by
 	   keeping a local copy in _LLstacktoobig_stack_end. */
 
-	if (_LLstacktoobig_stack_end == NULL) {
-		/* not initialized */
-		/* XXX We assume that initialization is performed early,
-		   when there is still only one thread running.  This
-		   allows us to ignore race conditions here */
-		RPyThreadStaticTLS_Create(&end_tls_key);
-	}
-
-	baseptr = (char *) RPyThreadStaticTLS_Get(end_tls_key);
+	OP_THREADLOCALREF_ADDR(tl);
+	tl1 = (struct pypy_threadlocal_s *)tl;
+	baseptr = tl1->stack_end;
 	max_stack_size = _LLstacktoobig_stack_length;
 	if (baseptr == NULL) {
 		/* first time we see this thread */
@@ -58,7 +55,7 @@
 
 	/* update the stack base pointer to the current value */
 	baseptr = curptr;
-	RPyThreadStaticTLS_Set(end_tls_key, baseptr);
+	tl1->stack_end = baseptr;
 	_LLstacktoobig_stack_end = baseptr;
 	return 0;
 }
diff --git a/rpython/translator/c/src/stack.h b/rpython/translator/c/src/stack.h
--- a/rpython/translator/c/src/stack.h
+++ b/rpython/translator/c/src/stack.h
@@ -2,14 +2,13 @@
 /************************************************************/
  /***  C header subsection: stack operations               ***/
 
+#include 
+
+
 #ifndef MAX_STACK_SIZE
 #    define MAX_STACK_SIZE (3 << 18)    /* 768 kb */
 #endif
 
-/* This include must be done in any case to initialise
- * the header dependencies early (winsock2, before windows.h).
- * It is needed to have RPyThreadStaticTLS, too. */
-#include "threadlocal.h"
 
 RPY_EXTERN char *_LLstacktoobig_stack_end;
 RPY_EXTERN long _LLstacktoobig_stack_length;
diff --git a/rpython/translator/c/src/support.h b/rpython/translator/c/src/support.h
--- a/rpython/translator/c/src/support.h
+++ b/rpython/translator/c/src/support.h
@@ -2,6 +2,9 @@
 /************************************************************/
  /***  C header subsection: support functions              ***/
 
+#ifndef _SRC_SUPPORT_H
+#define _SRC_SUPPORT_H
+
 #define RUNNING_ON_LLINTERP	0
 #define OP_JIT_RECORD_KNOWN_CLASS(i, c, r)  /* nothing */
 
@@ -65,3 +68,5 @@
 #  define RPyNLenItem(array, index)          ((array)->items[index])
 #  define RPyBareItem(array, index)          ((array)[index])
 #endif
+
+#endif  /* _SRC_SUPPORT_H */
diff --git a/rpython/translator/c/src/thread_nt.c b/rpython/translator/c/src/thread_nt.c
--- a/rpython/translator/c/src/thread_nt.c
+++ b/rpython/translator/c/src/thread_nt.c
@@ -26,15 +26,6 @@
 
 static long _pypythread_stacksize = 0;
 
-/*
- * Return the thread Id instead of an handle. The Id is said to uniquely
-   identify the thread in the system
- */
-long RPyThreadGetIdent()
-{
-  return GetCurrentThreadId();
-}
-
 static void
 bootstrap(void *call)
 {
@@ -42,7 +33,7 @@
 	/* copy callobj since other thread might free it before we're done */
 	void (*func)(void) = obj->func;
 
-	obj->id = RPyThreadGetIdent();
+	obj->id = GetCurrentThreadId();
 	ReleaseSemaphore(obj->done, 1, NULL);
 	func();
 }
diff --git a/rpython/translator/c/src/thread_nt.h b/rpython/translator/c/src/thread_nt.h
--- a/rpython/translator/c/src/thread_nt.h
+++ b/rpython/translator/c/src/thread_nt.h
@@ -13,8 +13,6 @@
 
 /* prototypes */
 RPY_EXTERN
-long RPyThreadGetIdent(void);
-RPY_EXTERN
 long RPyThreadStart(void (*func)(void));
 RPY_EXTERN
 int RPyThreadLockInit(struct RPyOpaque_ThreadLock *lock);
diff --git a/rpython/translator/c/src/thread_pthread.c b/rpython/translator/c/src/thread_pthread.c
--- a/rpython/translator/c/src/thread_pthread.c
+++ b/rpython/translator/c/src/thread_pthread.c
@@ -56,30 +56,6 @@
 # endif
 #endif
 
-/* XXX This implementation is considered (to quote Tim Peters) "inherently
-   hosed" because:
-     - It does not guarantee the promise that a non-zero integer is returned.
-     - The cast to long is inherently unsafe.
-     - It is not clear that the 'volatile' (for AIX?) and ugly casting in the
-       latter return statement (for Alpha OSF/1) are any longer necessary.
-*/
-long RPyThreadGetIdent(void)
-{
-	volatile pthread_t threadid;
-	/* Jump through some hoops for Alpha OSF/1 */
-	threadid = pthread_self();
-
-#ifdef __CYGWIN__
-	/* typedef __uint32_t pthread_t; */
-	return (long) threadid;
-#else
-	if (sizeof(pthread_t) <= sizeof(long))
-		return (long) threadid;
-	else
-		return (long) *(long *) &threadid;
-#endif
-}
-
 static long _pypythread_stacksize = 0;
 
 static void *bootstrap_pthread(void *func)
diff --git a/rpython/translator/c/src/thread_pthread.h b/rpython/translator/c/src/thread_pthread.h
--- a/rpython/translator/c/src/thread_pthread.h
+++ b/rpython/translator/c/src/thread_pthread.h
@@ -60,8 +60,6 @@
 /* prototypes */
 
 RPY_EXTERN
-long RPyThreadGetIdent(void);
-RPY_EXTERN
 long RPyThreadStart(void (*func)(void));
 RPY_EXTERN
 int RPyThreadLockInit(struct RPyOpaque_ThreadLock *lock);
diff --git a/rpython/translator/c/src/threadlocal.c b/rpython/translator/c/src/threadlocal.c
--- a/rpython/translator/c/src/threadlocal.c
+++ b/rpython/translator/c/src/threadlocal.c
@@ -1,28 +1,117 @@
+#include "common_header.h"
+#include "structdef.h"       /* for struct pypy_threadlocal_s */
 #include 
 #include 
+#include 
+#include 
+#ifndef _WIN32
+# include 
+#endif
 #include "src/threadlocal.h"
 
+
+static void _RPy_ThreadLocals_Init(void *p)
+{
+    memset(p, 0, sizeof(struct pypy_threadlocal_s));
+#ifdef RPY_TLOFS_p_errno
+    ((struct pypy_threadlocal_s *)p)->p_errno = &errno;
+#endif
+#ifdef RPY_TLOFS_thread_ident
+    ((struct pypy_threadlocal_s *)p)->thread_ident =
+#    ifdef _WIN32
+        GetCurrentThreadId();
+#    else
+        (long)pthread_self();    /* xxx This abuses pthread_self() by
+                  assuming it just returns a integer.  According to
+                  comments in CPython's source code, the platforms
+                  where it is not the case are rather old nowadays. */
+#    endif
+#endif
+    ((struct pypy_threadlocal_s *)p)->ready = 42;
+}
+
+
+/* ------------------------------------------------------------ */
+#ifdef USE___THREAD
+/* ------------------------------------------------------------ */
+
+
+/* in this situation, we always have one full 'struct pypy_threadlocal_s'
+   available, managed by gcc. */
+__thread struct pypy_threadlocal_s pypy_threadlocal;
+
+void RPython_ThreadLocals_ProgramInit(void)
+{
+    _RPy_ThreadLocals_Init(&pypy_threadlocal);
+}
+
+char *_RPython_ThreadLocals_Build(void)
+{
+    RPyAssert(pypy_threadlocal.ready == 0, "corrupted thread-local");
+    _RPy_ThreadLocals_Init(&pypy_threadlocal);
+    return (char *)&pypy_threadlocal;
+}
+
+void RPython_ThreadLocals_ThreadDie(void)
+{
+    memset(&pypy_threadlocal, 0xDD,
+           sizeof(struct pypy_threadlocal_s));  /* debug */
+    pypy_threadlocal.ready = 0;
+}
+
+
+/* ------------------------------------------------------------ */
+#else
+/* ------------------------------------------------------------ */
+
+
+/* this is the case where the 'struct pypy_threadlocal_s' is allocated
+   explicitly, with malloc()/free(), and attached to (a single) thread-
+   local key using the API of Windows or pthread. */
+
+pthread_key_t pypy_threadlocal_key;
+
+
+void RPython_ThreadLocals_ProgramInit(void)
+{
 #ifdef _WIN32
-
-void RPyThreadTLS_Create(RPyThreadTLS *result)
-{
-    *result = TlsAlloc();
-    if (*result == TLS_OUT_OF_INDEXES) {
+    pypy_threadlocal_key = TlsAlloc();
+    if (pypy_threadlocal_key == TLS_OUT_OF_INDEXES)
+#else
+    if (pthread_key_create(&pypy_threadlocal_key, NULL) != 0)
+#endif
+    {
         fprintf(stderr, "Internal RPython error: "
                         "out of thread-local storage indexes");
         abort();
     }
+    _RPython_ThreadLocals_Build();
 }
 
-#else
+char *_RPython_ThreadLocals_Build(void)
+{
+    void *p = malloc(sizeof(struct pypy_threadlocal_s));
+    if (!p) {
+        fprintf(stderr, "Internal RPython error: "
+                        "out of memory for the thread-local storage");
+        abort();
+    }
+    _RPy_ThreadLocals_Init(p);
+    _RPy_ThreadLocals_Set(p);
+    return (char *)p;
+}
 
-void RPyThreadTLS_Create(RPyThreadTLS *result)
+void RPython_ThreadLocals_ThreadDie(void)
 {
-    if (pthread_key_create(result, NULL) != 0) {
-        fprintf(stderr, "Internal RPython error: "
-                        "out of thread-local storage keys");
-        abort();
+    void *p = _RPy_ThreadLocals_Get();
+    if (p != NULL) {
+        _RPy_ThreadLocals_Set(NULL);
+        memset(p, 0xDD, sizeof(struct pypy_threadlocal_s));  /* debug */
+        free(p);
     }
 }
 
+
+/* ------------------------------------------------------------ */
 #endif
+/* ------------------------------------------------------------ */
diff --git a/rpython/translator/c/src/threadlocal.h b/rpython/translator/c/src/threadlocal.h
--- a/rpython/translator/c/src/threadlocal.h
+++ b/rpython/translator/c/src/threadlocal.h
@@ -2,51 +2,98 @@
 #ifndef _SRC_THREADLOCAL_H
 #define _SRC_THREADLOCAL_H
 
-#include 
+#include "src/precommondefs.h"
+#include "src/support.h"
 
 
+/* RPython_ThreadLocals_ProgramInit() is called once at program start-up. */
+RPY_EXTERN void RPython_ThreadLocals_ProgramInit(void);
+
+/* RPython_ThreadLocals_ThreadDie() is called in a thread that is about
+   to die. */
+RPY_EXTERN void RPython_ThreadLocals_ThreadDie(void);
+
+/* There are two llops: 'threadlocalref_addr' and 'threadlocalref_make'.
+   They both return the address of the thread-local structure (of the
+   C type 'struct pypy_threadlocal_s').  The difference is that
+   OP_THREADLOCALREF_MAKE() checks if we have initialized this thread-
+   local structure in the current thread, and if not, calls the following
+   helper. */
+RPY_EXTERN char *_RPython_ThreadLocals_Build(void);
+
+
+/* ------------------------------------------------------------ */
+#ifdef USE___THREAD
+/* ------------------------------------------------------------ */
+
+
+/* Use the '__thread' specifier, so far only on Linux */
+
+RPY_EXTERN __thread struct pypy_threadlocal_s pypy_threadlocal;
+
+#define OP_THREADLOCALREF_ADDR(r)               \
+    do {                                        \
+        r = (char *)&pypy_threadlocal;          \
+        if (pypy_threadlocal.ready != 42)       \
+            r = _RPython_ThreadLocals_Build();  \
+    } while (0)
+
+#define RPY_THREADLOCALREF_ENSURE()             \
+    if (pypy_threadlocal.ready != 42)           \
+        (void)_RPython_ThreadLocals_Build();
+
+#define RPY_THREADLOCALREF_GET(FIELD)   pypy_threadlocal.FIELD
+
+
+/* ------------------------------------------------------------ */
+#else
+/* ------------------------------------------------------------ */
+
+
+/* Don't use '__thread'. */
+
 #ifdef _WIN32
-
-#include 
-#include 
-#define __thread __declspec(thread)
-typedef DWORD RPyThreadTLS;
-#define RPyThreadTLS_Get(key)		TlsGetValue(key)
-#define RPyThreadTLS_Set(key, value)	TlsSetValue(key, value)
-
+#  include 
+#  include 
+#  define _RPy_ThreadLocals_Get()   TlsGetValue(pypy_threadlocal_key)
+#  define _RPy_ThreadLocals_Set(x)  TlsSetValue(pypy_threadlocal_key, x)
+typedef DWORD pthread_key_t;
 #else
-
-#include 
-typedef pthread_key_t RPyThreadTLS;
-#define RPyThreadTLS_Get(key)		pthread_getspecific(key)
-#define RPyThreadTLS_Set(key, value)	pthread_setspecific(key, value)
-
+#  include 
+#  define _RPy_ThreadLocals_Get()   pthread_getspecific(pypy_threadlocal_key)
+#  define _RPy_ThreadLocals_Set(x)  pthread_setspecific(pypy_threadlocal_key, x)
 #endif
 
+RPY_EXTERN pthread_key_t pypy_threadlocal_key;
 
-#ifdef USE___THREAD
 
-#define RPyThreadStaticTLS                  __thread void *
-#define RPyThreadStaticTLS_Create(tls)      (void)0
-#define RPyThreadStaticTLS_Get(tls)         tls
-#define RPyThreadStaticTLS_Set(tls, value)  tls = value
-#define OP_THREADLOCALREF_GETADDR(tlref, ptr)  ptr = tlref
+#define OP_THREADLOCALREF_ADDR(r)               \
+    do {                                        \
+        r = (char *)_RPy_ThreadLocals_Get();    \
+        if (!r)                                 \
+            r = _RPython_ThreadLocals_Build();  \
+    } while (0)
 
+#define RPY_THREADLOCALREF_ENSURE()             \
+    if (!_RPy_ThreadLocals_Get())               \
+        (void)_RPython_ThreadLocals_Build();
+
+#define RPY_THREADLOCALREF_GET(FIELD)           \
+    ((struct pypy_threadlocal_s *)_RPy_ThreadLocals_Get())->FIELD
+
+
+/* ------------------------------------------------------------ */
 #endif
+/* ------------------------------------------------------------ */
 
-#ifndef RPyThreadStaticTLS
 
-#define RPyThreadStaticTLS             RPyThreadTLS
-#define RPyThreadStaticTLS_Create(key) RPyThreadTLS_Create(key)
-#define RPyThreadStaticTLS_Get(key)    RPyThreadTLS_Get(key)
-#define RPyThreadStaticTLS_Set(key, value) RPyThreadTLS_Set(key, value)
-RPY_EXTERN void RPyThreadTLS_Create(RPyThreadTLS *result);
-
-#endif
-
-
-#define OP_THREADLOCALREF_SET(tlref, ptr, _) RPyThreadStaticTLS_Set(*tlref, ptr)
-#define OP_THREADLOCALREF_GET(tlref, ptr)   ptr = RPyThreadStaticTLS_Get(*tlref)
+/* only for the fall-back path in the JIT */
+#define OP_THREADLOCALREF_GET_NONCONST(RESTYPE, offset, r)      \
+    do {                                                        \
+        char *a;                                                \
+        OP_THREADLOCALREF_ADDR(a);                              \
+        r = *(RESTYPE *)(a + offset);                           \
+    } while (0)
 
 
 #endif /* _SRC_THREADLOCAL_H */
diff --git a/rpython/translator/c/test/test_standalone.py b/rpython/translator/c/test/test_standalone.py
--- a/rpython/translator/c/test/test_standalone.py
+++ b/rpython/translator/c/test/test_standalone.py
@@ -2,6 +2,7 @@
 import sys, os, re
 
 from rpython.config.translationoption import get_combined_translation_config
+from rpython.config.translationoption import SUPPORT__THREAD
 from rpython.rlib.objectmodel import keepalive_until_here
 from rpython.rlib.rarithmetic import r_longlong
 from rpython.rlib.debug import ll_assert, have_debug_prints, debug_flush
@@ -1026,11 +1027,12 @@
     gcrootfinder = 'shadowstack'
     config = None
 

From noreply at buildbot.pypy.org  Thu Nov 27 13:19:02 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 27 Nov 2014 13:19:02 +0100 (CET)
Subject: [pypy-commit] pypy default: issue #1921: produce switches on ints
 which don't promote the incoming
Message-ID: <20141127121902.997911D2487@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r74748:cf25c8fc4cdb
Date: 2014-11-27 13:18 +0100
http://bitbucket.org/pypy/pypy/changeset/cf25c8fc4cdb/

Log:	issue #1921: produce switches on ints which don't promote the
	incoming value if it doesn't match any of the cases and goes to
	"default".

diff --git a/rpython/jit/codewriter/assembler.py b/rpython/jit/codewriter/assembler.py
--- a/rpython/jit/codewriter/assembler.py
+++ b/rpython/jit/codewriter/assembler.py
@@ -216,10 +216,11 @@
             self.code[pos  ] = chr(target & 0xFF)
             self.code[pos+1] = chr(target >> 8)
         for descr in self.switchdictdescrs:
-            descr.dict = {}
+            as_dict = {}
             for key, switchlabel in descr._labels:
                 target = self.label_positions[switchlabel.name]
-                descr.dict[key] = target
+                as_dict[key] = target
+            descr.attach(as_dict)
 
     def check_result(self):
         # Limitation of the number of registers, from the single-byte encoding
diff --git a/rpython/jit/codewriter/flatten.py b/rpython/jit/codewriter/flatten.py
--- a/rpython/jit/codewriter/flatten.py
+++ b/rpython/jit/codewriter/flatten.py
@@ -243,55 +243,39 @@
         else:
             # A switch.
             #
-            def emitdefaultpath():
-                if block.exits[-1].exitcase == 'default':
-                    self.make_link(block.exits[-1])
-                else:
-                    self.emitline("unreachable")
-                    self.emitline("---")
-            #
-            self.emitline('-live-')
             switches = [link for link in block.exits
                         if link.exitcase != 'default']
             switches.sort(key=lambda link: link.llexitcase)
             kind = getkind(block.exitswitch.concretetype)
-            if len(switches) >= 5 and kind == 'int':
-                # A large switch on an integer, implementable efficiently
-                # with the help of a SwitchDictDescr
-                from rpython.jit.codewriter.jitcode import SwitchDictDescr
-                switchdict = SwitchDictDescr()
-                switchdict._labels = []
-                self.emitline('switch', self.getcolor(block.exitswitch),
-                                        switchdict)
-                emitdefaultpath()
-                #
-                for switch in switches:
-                    key = lltype.cast_primitive(lltype.Signed,
-                                                switch.llexitcase)
-                    switchdict._labels.append((key, TLabel(switch)))
-                    # emit code for that path
-                    self.emitline(Label(switch))
-                    self.make_link(switch)
+            assert kind == 'int'    # XXX
             #
+            # A switch on an integer, implementable efficiently with the
+            # help of a SwitchDictDescr.  We use this even if there are
+            # very few cases: in pyjitpl.py, opimpl_switch() will promote
+            # the int only if it matches one of the cases.
+            from rpython.jit.codewriter.jitcode import SwitchDictDescr
+            switchdict = SwitchDictDescr()
+            switchdict._labels = []
+            self.emitline('-live-')    # for 'guard_value'
+            self.emitline('switch', self.getcolor(block.exitswitch),
+                                    switchdict)
+            # emit the default path
+            if block.exits[-1].exitcase == 'default':
+                self.make_link(block.exits[-1])
             else:
-                # A switch with several possible answers, though not too
-                # many of them -- a chain of int_eq comparisons is fine
-                assert kind == 'int'    # XXX
-                color = self.getcolor(block.exitswitch)
-                self.emitline('int_guard_value', color)
-                for switch in switches:
-                    # make the case described by 'switch'
-                    self.emitline('goto_if_not_int_eq',
-                                  color,
-                                  Constant(switch.llexitcase,
-                                           block.exitswitch.concretetype),
-                                  TLabel(switch))
-                    # emit code for the "taken" path
-                    self.make_link(switch)
-                    # finally, emit the label for the "non-taken" path
-                    self.emitline(Label(switch))
-                #
-                emitdefaultpath()
+                self.emitline("unreachable")
+                self.emitline("---")
+            #
+            for switch in switches:
+                key = lltype.cast_primitive(lltype.Signed,
+                                            switch.llexitcase)
+                switchdict._labels.append((key, TLabel(switch)))
+                # emit code for that path
+                # note: we need a -live- for all the 'guard_false' we produce
+                # if the switched value doesn't match any case.
+                self.emitline(Label(switch))
+                self.emitline('-live-')
+                self.make_link(switch)
 
     def insert_renamings(self, link):
         renamings = {}
diff --git a/rpython/jit/codewriter/jitcode.py b/rpython/jit/codewriter/jitcode.py
--- a/rpython/jit/codewriter/jitcode.py
+++ b/rpython/jit/codewriter/jitcode.py
@@ -1,4 +1,4 @@
-from rpython.jit.metainterp.history import AbstractDescr
+from rpython.jit.metainterp.history import AbstractDescr, ConstInt
 from rpython.jit.codewriter import heaptracker
 from rpython.rlib.objectmodel import we_are_translated
 
@@ -109,6 +109,10 @@
 class SwitchDictDescr(AbstractDescr):
     "Get a 'dict' attribute mapping integer values to bytecode positions."
 
+    def attach(self, as_dict):
+        self.dict = as_dict
+        self.const_keys_in_order = map(ConstInt, sorted(as_dict.keys()))
+
     def __repr__(self):
         dict = getattr(self, 'dict', '?')
         return '' % (dict,)
diff --git a/rpython/jit/codewriter/test/test_flatten.py b/rpython/jit/codewriter/test/test_flatten.py
--- a/rpython/jit/codewriter/test/test_flatten.py
+++ b/rpython/jit/codewriter/test/test_flatten.py
@@ -282,30 +282,6 @@
             foobar hi_there!
         """)
 
-    def test_switch(self):
-        def f(n):
-            if n == -5:  return 12
-            elif n == 2: return 51
-            elif n == 7: return 1212
-            else:        return 42
-        self.encoding_test(f, [65], """
-            -live-
-            int_guard_value %i0
-            goto_if_not_int_eq %i0, $-5, L1
-            int_return $12
-            ---
-            L1:
-            goto_if_not_int_eq %i0, $2, L2
-            int_return $51
-            ---
-            L2:
-            goto_if_not_int_eq %i0, $7, L3
-            int_return $1212
-            ---
-            L3:
-            int_return $42
-        """)
-
     def test_switch_dict(self):
         def f(x):
             if   x == 1: return 61
diff --git a/rpython/jit/metainterp/pyjitpl.py b/rpython/jit/metainterp/pyjitpl.py
--- a/rpython/jit/metainterp/pyjitpl.py
+++ b/rpython/jit/metainterp/pyjitpl.py
@@ -402,13 +402,26 @@
 
     @arguments("box", "descr", "orgpc")
     def opimpl_switch(self, valuebox, switchdict, orgpc):
-        box = self.implement_guard_value(valuebox, orgpc)
-        search_value = box.getint()
+        search_value = valuebox.getint()
         assert isinstance(switchdict, SwitchDictDescr)
         try:
-            self.pc = switchdict.dict[search_value]
+            target = switchdict.dict[search_value]
         except KeyError:
-            pass
+            # None of the cases match.  Fall back to generating a chain
+            # of 'int_eq'.
+            # xxx as a minor optimization, if that's a bridge, then we would
+            # not need the cases that we already tested (and failed) with
+            # 'guard_value'.  How to do it is not very clear though.
+            for const1 in switchdict.const_keys_in_order:
+                box = self.execute(rop.INT_EQ, valuebox, const1)
+                assert box.getint() == 0
+                target = switchdict.dict[const1.getint()]
+                self.metainterp.generate_guard(rop.GUARD_FALSE, box,
+                                               resumepc=target)
+        else:
+            # found one of the cases
+            self.implement_guard_value(valuebox, orgpc)
+            self.pc = target
 
     @arguments()
     def opimpl_unreachable(self):
@@ -2270,8 +2283,8 @@
         if opnum == rop.GUARD_TRUE:     # a goto_if_not that jumps only now
             if not dont_change_position:
                 frame.pc = frame.jitcode.follow_jump(frame.pc)
-        elif opnum == rop.GUARD_FALSE:     # a goto_if_not that stops jumping
-            pass
+        elif opnum == rop.GUARD_FALSE:     # a goto_if_not that stops jumping;
+            pass                  # or a switch that was in its "default" case
         elif opnum == rop.GUARD_VALUE or opnum == rop.GUARD_CLASS:
             pass        # the pc is already set to the *start* of the opcode
         elif (opnum == rop.GUARD_NONNULL or
diff --git a/rpython/jit/metainterp/test/test_ajit.py b/rpython/jit/metainterp/test/test_ajit.py
--- a/rpython/jit/metainterp/test/test_ajit.py
+++ b/rpython/jit/metainterp/test/test_ajit.py
@@ -698,6 +698,40 @@
         res = self.interp_operations(f, [12311])
         assert res == 42
 
+    def test_switch_bridges(self):
+        from rpython.rlib.rarithmetic import intmask
+        myjitdriver = JitDriver(greens = [], reds = 'auto')
+        lsts = [[-5, 2, 20] * 6,
+                [7, 123, 2] * 6,
+                [12311, -5, 7] * 6,
+                [7, 123, 2] * 4 + [-5, -5, -5] * 2,
+                [7, 123, 2] * 4 + [-5, -5, -5] * 2 + [12311, 12311, 12311],
+                ]
+        def f(case):
+            x = 0
+            i = 0
+            lst = lsts[case]
+            while i < len(lst):
+                myjitdriver.jit_merge_point()
+                n = lst[i]
+                if n == -5:  a = 5
+                elif n == 2: a = 4
+                elif n == 7: a = 3
+                else:        a = 2
+                x = intmask(x * 10 + a)
+                i += 1
+            return x
+        res = self.meta_interp(f, [0], backendopt=True)
+        assert res == intmask(542 * 1001001001001001)
+        res = self.meta_interp(f, [1], backendopt=True)
+        assert res == intmask(324 * 1001001001001001)
+        res = self.meta_interp(f, [2], backendopt=True)
+        assert res == intmask(253 * 1001001001001001)
+        res = self.meta_interp(f, [3], backendopt=True)
+        assert res == intmask(324324324324555555)
+        res = self.meta_interp(f, [4], backendopt=True)
+        assert res == intmask(324324324324555555222)
+
     def test_r_uint(self):
         from rpython.rlib.rarithmetic import r_uint
         myjitdriver = JitDriver(greens = [], reds = ['y'])
@@ -833,23 +867,6 @@
         assert type(res) == bool
         assert not res
 
-    def test_switch_dict(self):
-        def f(x):
-            if   x == 1: return 61
-            elif x == 2: return 511
-            elif x == 3: return -22
-            elif x == 4: return 81
-            elif x == 5: return 17
-            elif x == 6: return 54
-            elif x == 7: return 987
-            elif x == 8: return -12
-            elif x == 9: return 321
-            return -1
-        res = self.interp_operations(f, [5])
-        assert res == 17
-        res = self.interp_operations(f, [15])
-        assert res == -1
-
     def test_int_add_ovf(self):
         def f(x, y):
             try:

From noreply at buildbot.pypy.org  Thu Nov 27 13:46:57 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 27 Nov 2014 13:46:57 +0100 (CET)
Subject: [pypy-commit] pypy default: Add integer bounds to 'int_signext'
	results.
Message-ID: <20141127124657.D9B1B1C3A61@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r74749:eba714c0fe25
Date: 2014-11-27 13:46 +0100
http://bitbucket.org/pypy/pypy/changeset/eba714c0fe25/

Log:	Add integer bounds to 'int_signext' results.

diff --git a/rpython/jit/metainterp/optimizeopt/intbounds.py b/rpython/jit/metainterp/optimizeopt/intbounds.py
--- a/rpython/jit/metainterp/optimizeopt/intbounds.py
+++ b/rpython/jit/metainterp/optimizeopt/intbounds.py
@@ -342,6 +342,13 @@
         else:
             self.emit_operation(op)
 
+    def optimize_INT_SIGNEXT(self, op):
+        self.emit_operation(op)
+        v1 = self.getvalue(op.result)
+        numbits = op.getarg(1).getint() * 8
+        v1.intbound.make_ge(IntLowerBound(-(1 << (numbits - 1))))
+        v1.intbound.make_lt(IntUpperBound(1 << (numbits - 1)))
+
     def optimize_ARRAYLEN_GC(self, op):
         self.emit_operation(op)
         array = self.getvalue(op.getarg(0))
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
@@ -5607,6 +5607,44 @@
         """
         self.optimize_loop(ops, ops, ops)
 
+    def test_bound_backpropagate_int_signext(self):
+        ops = """
+        []
+        i0 = escape()
+        i1 = int_signext(i0, 1)
+        i2 = int_eq(i0, i1)
+        guard_true(i2) []
+        i3 = int_le(i0, 127)    # implied by equality with int_signext
+        guard_true(i3) []
+        i5 = int_gt(i0, -129)   # implied by equality with int_signext
+        guard_true(i5) []
+        jump()
+        """
+        expected = """
+        []
+        i0 = escape()
+        i1 = int_signext(i0, 1)
+        i2 = int_eq(i0, i1)
+        guard_true(i2) []
+        jump()
+        """
+        self.optimize_loop(ops, expected)
+
+    def test_bound_backpropagate_int_signext_2(self):
+        ops = """
+        []
+        i0 = escape()
+        i1 = int_signext(i0, 1)
+        i2 = int_eq(i0, i1)
+        guard_true(i2) []
+        i3 = int_le(i0, 126)    # false for i1 == 127
+        guard_true(i3) []
+        i5 = int_gt(i0, -128)   # false for i1 == -128
+        guard_true(i5) []
+        jump()
+        """
+        self.optimize_loop(ops, ops)
+
     def test_mul_ovf(self):
         ops = """
         [i0, i1]

From noreply at buildbot.pypy.org  Thu Nov 27 13:49:15 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 27 Nov 2014 13:49:15 +0100 (CET)
Subject: [pypy-commit] pypy default: bah
Message-ID: <20141127124915.68C0C1C3A61@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r74750:b6df7b497a7d
Date: 2014-11-27 13:48 +0100
http://bitbucket.org/pypy/pypy/changeset/b6df7b497a7d/

Log:	bah

diff --git a/pypy/module/pypyjit/test_pypy_c/test_ffi.py b/pypy/module/pypyjit/test_pypy_c/test_ffi.py
--- a/pypy/module/pypyjit/test_pypy_c/test_ffi.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_ffi.py
@@ -338,7 +338,7 @@
         i112 = int_signext(i160, 2)
         setfield_gc(p167, ConstPtr(null), descr=)
         setfield_gc(p167, ConstPtr(ptr85), descr=)
-        i114 = int_ne(i112, i160)
+        i114 = int_ne(i160, i112)
         guard_false(i114, descr=...)
         --TICK--
         i119 = call(ConstClass(_ll_1_raw_malloc_varsize__Signed), 6, descr=)

From noreply at buildbot.pypy.org  Thu Nov 27 16:29:49 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 27 Nov 2014 16:29:49 +0100 (CET)
Subject: [pypy-commit] creflect default: Change to group all CREFLECT
 sections inside a single function
Message-ID: <20141127152949.09EFB1D2E15@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r94:aee2e5e2ec38
Date: 2014-11-27 16:30 +0100
http://bitbucket.org/cffi/creflect/changeset/aee2e5e2ec38/

Log:	Change to group all CREFLECT sections inside a single function at
	the end, called by default '_creflect_main()'.

diff --git a/creflect/cmdline.py b/creflect/cmdline.py
--- a/creflect/cmdline.py
+++ b/creflect/cmdline.py
@@ -9,7 +9,8 @@
  from stdin and/or write to stdout.
 
 Options:
-    -m, --main      for debugging, include a main() function that
+    -m/--main=NAME  name of main exported function, default '_creflect_main'
+    -d, --debug     for debugging, include a main() function that
                       prints the recorded reflection information
     -n, --no-copy   output only the translated creflect sections,
                       without copying the text around them
@@ -28,13 +29,14 @@
 
 def main(argv):
     try:
-        options, args = getopt.gnu_getopt(argv, "mnih",
-            ["main", "no-copy", "include", "help", "version"])
+        options, args = getopt.gnu_getopt(argv, "m:dnih",
+            ["main=", "debug", "no-copy", "include", "help", "version"])
     except getopt.GetoptError, e:
         return error(e)
     #
     from . import __version__
-    include_main = False
+    main_name = '_creflect_main'
+    include_debug = False
     include_text_outside_creflect = True
     include_includes = False
     for option, value in options:
@@ -44,7 +46,9 @@
             include_text_outside_creflect = False
             include_includes = True
         elif option == '-m' or option == '--main':
-            include_main = True
+            main_name = value
+        elif option == '-d' or option == '--debug':
+            include_debug = True
         elif option == '-h' or option == '--help':
             print __doc__
             return 0
@@ -73,10 +77,10 @@
     try:
         if include_includes:
             outputf.write('#include "%s"\n' % (inputfile,))
-        blocks = driver.copy_file_and_expand(inputf, outputf,
+        driver.copy_file_and_expand(inputf, outputf, main_name,
             include_text_outside_creflect = include_text_outside_creflect)
-        if include_main:
-            outputf.write(driver.get_debug_code(blocks))
+        if include_debug:
+            outputf.write(driver.get_debug_code())
     except driver.CDefError, e:
         print >> sys.stderr, 'FAILED:', e
         return 1
diff --git a/creflect/codegen.py b/creflect/codegen.py
--- a/creflect/codegen.py
+++ b/creflect/codegen.py
@@ -54,7 +54,7 @@
         return d1
 
 
-def transform_cdef(csource, crx_func_name):
+def transform_cdef(csource, crx_func_name, static=False):
     outerblock = CodeBlock(parent=None)
     outerblock.writeline('#include "creflect.h"')
     outerblock.crx_func_name = crx_func_name
@@ -70,7 +70,11 @@
     for d1 in funcblock.crx_field_vars:
         funcblock.writedecl("crx_field_t %s;" % d1)
 
-    outerblock.writeline('void %s(crx_builder_t *cb)' % (crx_func_name,))
+    static_decl = ""
+    if static:
+        static_decl = "static "
+    outerblock.writeline('%svoid %s(crx_builder_t *cb)' % (static_decl,
+                                                           crx_func_name))
     outerblock.write_subblock(funcblock)
     outerblock.writeline('')     # for the final \n below
     return '\n'.join(outerblock.codelines)
diff --git a/creflect/driver.py b/creflect/driver.py
--- a/creflect/driver.py
+++ b/creflect/driver.py
@@ -5,39 +5,35 @@
 
 def expand_cdef(input, reflection_func_name, first_lineno=1):
     csource = CSource(input, first_lineno)
-    return transform_cdef(csource, reflection_func_name)
+    return transform_cdef(csource, reflection_func_name, static=True)
 
 
-r_comment1 = re.compile(r'\s*/[*/]\s*CREFLECT\s*:\s*([A-Za-z_][A-Za-z0-9_]*)'
-                        r'\s*[(]\s*[)]')
-r_comment2 = re.compile(r'\s*/[*/]\s*CREFLECT\s*:\s*[Ee][Nn][Dd]\b')
-END = object()
+r_directive = re.compile(r'\s*/[*/]\s*CREFLECT\s*:\s*([A-Za-z_][A-Za-z0-9_]*)')
 
-def get_creflect_comment(line):
-    match = r_comment1.match(line)
+def get_creflect_directive(line, expected, lineno):
+    match = r_directive.match(line)
     if match:
-        return match.group(1)
-    if r_comment2.match(line):
-        return END
-    return None
+        if match.group(1).lower() != expected:
+            raise CDefError("line %d: 'CREFLECT: %s' expected, "
+                            "got 'CREFLECT: %s'" % (lineno, expected,
+                                                    match.group(1)))
+        return True
+    return False
 
-def copy_file_and_expand(inputf, outputf, include_text_outside_creflect=True):
+def copy_file_and_expand(inputf, outputf, export_func_name='_creflect_main',
+                         include_text_outside_creflect=True):
     lineno = 0
     blocks = []
     while True:
         # outside 'CREFLECT:' sections
         for line in inputf:
             lineno += 1
-            funcname = get_creflect_comment(line)
-            if funcname is not None:
+            if get_creflect_directive(line, 'start', lineno):
                 break
             if include_text_outside_creflect:
                 outputf.write(line)
         else:
             break    # end of file
-        if funcname is END:
-            raise CDefError("line %d: 'CREFLECT: end' without "
-                            "'CREFLECT: funcname()' before" % lineno)
         if not include_text_outside_creflect:
             outputf.write("\n")
         #
@@ -46,46 +42,52 @@
         first_lineno = lineno + 1
         for line in inputf:
             lineno += 1
-            end = get_creflect_comment(line)
-            if end is not None:
+            if get_creflect_directive(line, 'end', lineno):
                 break
             csource.append(line)
         else:
             raise CDefError("missing 'CREFLECT: end' in the input file")
-        if end is not END:
-            raise CDefError("line %d: 'CREFLECT: funcname()' without "
-                            "'CREFLECT: end' before" % lineno)
         #
         if include_text_outside_creflect:
             outputf.write("/***** CREFLECT block start, "
                           "source line %d *****/\n\n" % (first_lineno - 1,))
+        blocknum = len(blocks)
+        funcname = '_creflect_block%d' % blocknum
         output = expand_cdef(''.join(csource), funcname, first_lineno)
         outputf.write(output)
+        outputf.write('#define _CREFLECT_BLOCK%(blocknum)d\n' % {
+            'blocknum': blocknum})
         if include_text_outside_creflect:
             outputf.write("\n/***** CREFLECT block end, "
                           "source line %d *****/\n" % (lineno,))
-        blocks.append(funcname)
+        blocks.append(CALL_TEMPLATE2 % {'funcname': funcname,
+                                        'blocknum': blocknum})
     #
     if len(blocks) == 0:
         raise CDefError("no 'CREFLECT' block found in the input file")
-    return blocks
+    outputf.write(CALL_TEMPLATE1 % (export_func_name, ''.join(blocks)))
 
 
-_DEBUG_TEMPLATE1 = '''
+CALL_TEMPLATE1 = '''
+/***** CREFLECT main entry point *****/
+
+void %s(crx_builder_t *cb) {
+%s}
+'''
+CALL_TEMPLATE2 = '''\
+#ifdef _CREFLECT_BLOCK%(blocknum)d
+    %(funcname)s(cb);
+#endif
+'''
+
+DEBUG_TEMPLATE = '''\
 /***** CREFLECT debug code *****/
 #include "creflect_print.h"
 int main(void) {
-%s    return 0;
+    %s(&maincb);
+    return 0;
 }
 '''
-_DEBUG_TEMPLATE2 = '''\
-    %(funcname)s(&maincb);
-'''
 
-def get_debug_code(reflection_func_name):
-    if not isinstance(reflection_func_name, list):
-        reflection_func_name = [reflection_func_name]
-    parts = []
-    for funcname in reflection_func_name:
-        parts.append(_DEBUG_TEMPLATE2 % {'funcname': funcname})
-    return _DEBUG_TEMPLATE1 % (''.join(parts),)
+def get_debug_code(export_func_name='_creflect_main'):
+    return DEBUG_TEMPLATE % (export_func_name,)
diff --git a/test/test_driver.py b/test/test_driver.py
--- a/test/test_driver.py
+++ b/test/test_driver.py
@@ -3,10 +3,10 @@
 from creflect.driver import expand_cdef, get_debug_code, copy_file_and_expand
 from .udir import udir
 
-def compile_and_run(code, funcname):
+def compile_and_run(code, funcname, export_func_name='_creflect_main'):
     f = open(str(udir.join(funcname + '.c')), 'w')
     print >> f, code
-    print >> f, get_debug_code(funcname)
+    print >> f, get_debug_code(export_func_name)
     f.close()
     #
     from creflect.driver import __file__
@@ -27,7 +27,7 @@
     real_code = "typedef short a_t[42];\n"
     gen = expand_cdef("typedef long a_t[20];", "test_expand_cdef")
     code = '/* real code */\n%s\n/* generated code */\n%s' % (real_code, gen)
-    data = compile_and_run(code, "test_expand_cdef")
+    data = compile_and_run(code, "test_expand_cdef", "test_expand_cdef")
     assert data == "TYPEDEF a_t = ARRAY[42] short\n"
 
 
@@ -35,7 +35,7 @@
     f = StringIO("""
 typedef unsigned short a_t[5];
 
-/* CREFLECT: inspect1() */
+/* CREFLECT: start */
 typedef int a_t[20];
 /* CREFLECT: end */
 """)
@@ -44,3 +44,23 @@
     #
     data = compile_and_run(g.getvalue(), "inspect1")
     assert data == "TYPEDEF a_t = ARRAY[5] unsigned short\n"
+
+def test_copy_file_and_expand_ifdefs():
+    f = StringIO("""
+typedef unsigned short a_t[5];
+
+#ifdef FOOBARBAZ_NONEXISTENT
+// CREFLECT: start
+typedef int nonexistent_t[20];
+// CREFLECT: end
+#else
+// CREFLECT: start
+typedef int a_t[20];
+// CREFLECT: end
+#endif
+""")
+    g = StringIO()
+    copy_file_and_expand(f, g)
+    #
+    data = compile_and_run(g.getvalue(), "inspect2")
+    assert data == "TYPEDEF a_t = ARRAY[5] unsigned short\n"

From noreply at buildbot.pypy.org  Thu Nov 27 16:38:49 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 27 Nov 2014 16:38:49 +0100 (CET)
Subject: [pypy-commit] creflect default: Tweak the command-line interface
Message-ID: <20141127153849.1FD9A1D2E28@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r95:c4d9348840a6
Date: 2014-11-27 16:39 +0100
http://bitbucket.org/cffi/creflect/changeset/c4d9348840a6/

Log:	Tweak the command-line interface

diff --git a/creflect/cmdline.py b/creflect/cmdline.py
--- a/creflect/cmdline.py
+++ b/creflect/cmdline.py
@@ -1,12 +1,18 @@
 """Usage:
 
-    creflect [options] input{.rfl.c,.h} output.c
+    creflect  [options]  input.crx  output.c
+    creflect  [options]  -i  input.h  output.c
 
-Read the 'input.rfl.c' file and expand the CREFLECT sections in it
+Read the input file and expand the CREFLECT sections in it
 into regular C code.  Write the result to 'output.c'.
 
+The first variant is used to preprocess a 'foo.crx' file into 'foo.c',
+which is identical except with expanded CREFLECT sections.  The second
+variant with the '-i' option is used to generate 'foo.c' containing
+only the expansion of the CREFLECT sections found in 'foo.h'.
+
 The input and/or output file names can be specified as '-' to read
- from stdin and/or write to stdout.
+from stdin and/or write to stdout.
 
 Options:
     -m/--main=NAME  name of main exported function, default '_creflect_main'

From noreply at buildbot.pypy.org  Thu Nov 27 17:12:03 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 27 Nov 2014 17:12:03 +0100 (CET)
Subject: [pypy-commit] creflect default: If the tool crashes,
 output a #error line in the interrupted output file
Message-ID: <20141127161203.AC1211C1366@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r96:6eec004b93e6
Date: 2014-11-27 17:12 +0100
http://bitbucket.org/cffi/creflect/changeset/6eec004b93e6/

Log:	If the tool crashes, output a #error line in the interrupted output
	file

diff --git a/creflect/driver.py b/creflect/driver.py
--- a/creflect/driver.py
+++ b/creflect/driver.py
@@ -22,6 +22,19 @@
 
 def copy_file_and_expand(inputf, outputf, export_func_name='_creflect_main',
                          include_text_outside_creflect=True):
+    try:
+        _copy_file_and_expand(inputf, outputf, export_func_name,
+                              include_text_outside_creflect)
+    except Exception, e:
+        errmsg = str(e)
+        if errmsg:
+            errmsg = ': ' + errmsg
+        errmsg = e.__class__.__name__ + errmsg
+        outputf.write('\n#error %r\n' % (errmsg,))
+        raise
+
+def _copy_file_and_expand(inputf, outputf, export_func_name,
+                         include_text_outside_creflect):
     lineno = 0
     blocks = []
     while True:

From noreply at buildbot.pypy.org  Thu Nov 27 17:12:04 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 27 Nov 2014 17:12:04 +0100 (CET)
Subject: [pypy-commit] creflect default: Add a test and fix for an empty
	struct declaration
Message-ID: <20141127161204.B6BC41C1366@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r97:23788a07cebd
Date: 2014-11-27 17:12 +0100
http://bitbucket.org/cffi/creflect/changeset/23788a07cebd/

Log:	Add a test and fix for an empty struct declaration

diff --git a/creflect/cparser.py b/creflect/cparser.py
--- a/creflect/cparser.py
+++ b/creflect/cparser.py
@@ -28,6 +28,7 @@
 r_comment1 = re.compile(r"//.*")
 r_comment2 = re.compile(r"/[*]")
 r_comment3 = re.compile(r"[*]/")
+r_empty_braces = re.compile(r"{.*}")     # after comments have been removed
 
 def remove_comments(csource):
     csource = r_comment1.sub('', csource)     # remove the '//' comments
@@ -48,6 +49,9 @@
         parts.append(' ' + '\n' * csource.count('\n', start, pos))
     return ''.join(parts)
 
+def expand_empty_braces(csource):
+    return r_empty_braces.sub('{ char __crx_empty__; }', csource)
+
 
 class CSource(object):
 
@@ -58,8 +62,10 @@
 
     def to_ast(self):
         cparser = pycparser.CParser()
+        csource = remove_comments(self.cdefblock)
+        csource = expand_empty_braces(csource)
         try:
-            ast = cparser.parse(remove_comments(self.cdefblock))
+            ast = cparser.parse(csource)
         except pycparser.c_parser.ParseError as e:
             msg, lineno = self.convert_pycparser_error_line(e)
             raise CDefError(msg, self, lineno)
@@ -270,14 +276,9 @@
         fldnames = []
         fldtypes = []
         fldbitsize = []
+        if len(fields) == 1 and fields[0].name == '__crx_empty__':
+            fields = []
         for decl in fields:
-            if (isinstance(decl.type, pycparser.c_ast.IdentifierType) and
-                    ''.join(decl.type.names) == '__dotdotdot__'):
-                # XXX pycparser is inconsistent: 'names' should be a list
-                # of strings, but is sometimes just one string.  Use
-                # str.join() as a way to cope with both.
-                self._make_partial(tp, nested)
-                continue
             if decl.bitsize is None:
                 bitsize = -1
             else:
diff --git a/creflect/model.py b/creflect/model.py
--- a/creflect/model.py
+++ b/creflect/model.py
@@ -606,7 +606,10 @@
                            " - (char *)0" % (realtp,))          # alignment
         else:
             text_offset = '0'
-        d1 = funcblock.write_crx_field_var(len(self.fldnames))
+        if self.fldnames:
+            d1 = funcblock.write_crx_field_var(len(self.fldnames))
+        else:
+            d1 = 'NULL'
         t1 = self.type.get_type_var(funcblock)
         #
         for i, (fldname, fldtype) in enumerate(
diff --git a/test/codegen/struct-006.c b/test/codegen/struct-006.c
new file mode 100644
--- /dev/null
+++ b/test/codegen/struct-006.c
@@ -0,0 +1,17 @@
+typedef struct { } foo_t;
+
+# ____________________________________________________________
+
+void teststruct_006(crx_builder_t *cb)
+{
+    crx_type_t *t1;
+    t1 = cb->get_struct_type(cb, "$foo_t");
+    cb->complete(cb, t1, sizeof(foo_t),
+                 ((char *)&((struct{char a; foo_t b;} *)0)->b) - (char *)0,
+                 NULL, 0);
+#expect STRUCT $foo_t:
+    {
+        cb->define_type(cb, "foo_t", t1);
+#expect TYPEDEF foo_t = STRUCT $foo_t
+    }
+}

From noreply at buildbot.pypy.org  Thu Nov 27 17:46:19 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 27 Nov 2014 17:46:19 +0100 (CET)
Subject: [pypy-commit] creflect default: Use of a typedef'ed name: first in
	globals
Message-ID: <20141127164619.618811C34F4@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r98:76b1612c5824
Date: 2014-11-27 17:46 +0100
http://bitbucket.org/cffi/creflect/changeset/76b1612c5824/

Log:	Use of a typedef'ed name: first in globals

diff --git a/creflect/commontypes.py b/creflect/commontypes.py
--- a/creflect/commontypes.py
+++ b/creflect/commontypes.py
@@ -48,7 +48,4 @@
             return model.const_void_type
         else:
             return model.void_type
-    if ident == '__dotdotdot__':   # XXX
-        raise api.FFIError(':%d: bad usage of "..."' %
-                typenode.coord.line)
     return model.PrimitiveType(ident, const)
diff --git a/creflect/cparser.py b/creflect/cparser.py
--- a/creflect/cparser.py
+++ b/creflect/cparser.py
@@ -28,7 +28,7 @@
 r_comment1 = re.compile(r"//.*")
 r_comment2 = re.compile(r"/[*]")
 r_comment3 = re.compile(r"[*]/")
-r_empty_braces = re.compile(r"{.*}")     # after comments have been removed
+r_empty_braces = re.compile(r"{\s*}")     # after comments have been removed
 
 def remove_comments(csource):
     csource = r_comment1.sub('', csource)     # remove the '//' comments
@@ -83,6 +83,7 @@
 
     def get_declarations(self):
         self.declarations = []
+        self.typedefs = {}
         ast = self.to_ast()
         for decl in ast.ext:
             if isinstance(decl, pycparser.c_ast.Decl):
@@ -138,6 +139,9 @@
             realtype = model.unknown_ptr_type(decl.name)
         else:
             realtype = self._get_type(decl.type, approx_name = '$' + decl.name)
+        if decl.name in self.typedefs:
+            self._parse_error("duplicate typedef declaring %r" % (decl.name,))
+        self.typedefs[decl.name] = model.UserType(decl.name, realtype)
         self.declarations.append(model.TypeDefDecl(decl.name, realtype))
 
     def _get_type(self, typenode, approx_name=None, partial_length_ok=False):
@@ -145,10 +149,8 @@
         if (isinstance(typenode, pycparser.c_ast.TypeDecl) and
             isinstance(typenode.type, pycparser.c_ast.IdentifierType) and
             len(typenode.type.names) == 1 and
-            0): #('typedef ' + typenode.type.names[0]) in self._declarations):
-            #XXX
-            type = self._declarations['typedef ' + typenode.type.names[0]]
-            return type
+            typenode.type.names[0] in self.typedefs):
+            return self.typedefs[typenode.type.names[0]]
         #
         if isinstance(typenode, pycparser.c_ast.ArrayDecl):
             # array type
diff --git a/creflect/model.py b/creflect/model.py
--- a/creflect/model.py
+++ b/creflect/model.py
@@ -76,7 +76,7 @@
         }
 
     def __init__(self, name, const):
-        assert name in self.ALL_PRIMITIVE_TYPES
+        assert name in self.ALL_PRIMITIVE_TYPES, repr(name)
         self.name = name
         self.const = const
         self.c_name_with_marker = name + ' &'
@@ -453,6 +453,29 @@
         return self.get_type_var(block)
 
 
+class UserType(BaseType):
+    _attrs_ = ('name', 'realtype')
+
+    def __init__(self, name, realtype, const=False):
+        self.name = name
+        self.realtype = realtype
+        self.const = const or realtype.const
+        self.c_name_with_marker = name + ' &'
+        if const:
+            self.c_name_with_marker = 'const ' + self.c_name_with_marker
+
+    def inspect_nonconst_type(self, block, inspect):
+        inspect.flush()
+        if isinstance(inspect, VarInspector):
+            decl = self.get_c_name("*p1")
+            block.writeline("%s = &%s;  /* check that '%s' is of type '%s'"
+                            " */" % (decl, inspect.varname,
+                                     inspect.varname, self.get_c_name()))
+            block.writeline("(void)p1;")
+        return block.write_crx_type_var('cb->get_user_type(cb, "%s")' % (
+            self.name,))
+
+
 # ____________________________________________________________
 
 
diff --git a/creflect/src/creflect.h b/creflect/src/creflect.h
--- a/creflect/src/creflect.h
+++ b/creflect/src/creflect.h
@@ -51,6 +51,7 @@
     crx_type_t *(*get_struct_type)(CRX_SELF, const char *);
     crx_type_t *(*get_union_type)(CRX_SELF, const char *);
     crx_type_t *(*get_enum_type)(CRX_SELF, const char *);
+    crx_type_t *(*get_user_type)(CRX_SELF, const char *);
     void (*complete)(CRX_SELF, crx_type_t *,
                      size_t, size_t, crx_field_t[], int);
     void (*complete_enum)(CRX_SELF, crx_type_t *, crx_type_t *);
diff --git a/creflect/src/creflect_print.h b/creflect/src/creflect_print.h
--- a/creflect/src/creflect_print.h
+++ b/creflect/src/creflect_print.h
@@ -162,6 +162,11 @@
     return newtype2("ENUM ", name);
 }
 
+static crx_type_t *tst_get_user_type(crx_builder_t *cb, const char *name)
+{
+    return newtype(name);
+}
+
 static void tst_complete(crx_builder_t *cb, crx_type_t *t,
                          size_t sz, size_t align,
                          crx_field_t fields[], int nfields)
@@ -252,6 +257,7 @@
     tst_get_struct_type,
     tst_get_union_type,
     tst_get_enum_type,
+    tst_get_user_type,
     tst_complete,
     tst_complete_enum,
     tst_define_type,
diff --git a/test/codegen/glob-005.c b/test/codegen/glob-005.c
new file mode 100644
--- /dev/null
+++ b/test/codegen/glob-005.c
@@ -0,0 +1,27 @@
+typedef void *mytype_t;
+
+mytype_t foobar;
+
+# ____________________________________________________________
+
+void testglob_005(crx_builder_t *cb)
+{
+    crx_type_t *t1, *t2, *t3;
+    {
+        mytype_t *p1;
+        char *p2;
+        p1 = (void *)&p2;
+        *p1 = (void *)0;    /* check that 'mytype_t' is a pointer type */
+        t1 = cb->get_void_type(cb);
+        t2 = cb->get_pointer_type(cb, t1);
+        cb->define_type(cb, "mytype_t", t2);
+#expect TYPEDEF mytype_t = PTR void
+    }
+    {
+        mytype_t *p1 = &foobar;  /* check that 'foobar' is of type 'mytype_t' */
+        (void)p1;
+        t3 = cb->get_user_type(cb, "mytype_t");
+        cb->define_var(cb, "foobar", t3, &foobar);
+#expect VAR foobar: mytype_t
+    }
+}

From noreply at buildbot.pypy.org  Fri Nov 28 10:53:48 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Fri, 28 Nov 2014 10:53:48 +0100 (CET)
Subject: [pypy-commit] cffi HERE-in-paths: Allow specifying $HERE in the
 paths given to verify()
Message-ID: <20141128095348.A2CEF1C07BB@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: HERE-in-paths
Changeset: r1582:46d4f4c6b3c4
Date: 2014-11-28 10:53 +0100
http://bitbucket.org/cffi/cffi/changeset/46d4f4c6b3c4/

Log:	Allow specifying $HERE in the paths given to verify()

diff --git a/cffi/api.py b/cffi/api.py
--- a/cffi/api.py
+++ b/cffi/api.py
@@ -336,6 +336,7 @@
         """
         from .verifier import Verifier
         self.verifier = Verifier(self, source, tmpdir, **kwargs)
+        self.verifier.expand_here_in_kwds(frame=sys._getframe(1))
         lib = self.verifier.load_library()
         self._libraries.append(lib)
         return lib
diff --git a/cffi/verifier.py b/cffi/verifier.py
--- a/cffi/verifier.py
+++ b/cffi/verifier.py
@@ -11,6 +11,11 @@
     def _extension_suffixes():
         return [suffix for suffix, _, type in imp.get_suffixes()
                 if type == imp.C_EXTENSION]
+try:
+    basestring
+except NameError:
+    # Python 3.x
+    basestring = str
 
 
 class Verifier(object):
@@ -104,6 +109,24 @@
         modname = self.get_module_name()
         return ffiplatform.get_extension(sourcename, modname, **self.kwds)
 
+    def expand_here_in_kwds(self, here=None, frame=None):
+        if frame is not None:
+            try:
+                here = os.path.dirname(frame.f_globals['__file__'])
+            except (AttributeError, KeyError):
+                pass
+        for key, value in self.kwds.items():
+            if not isinstance(value, list):
+                continue
+            for i in range(len(value)):
+                x = value[i]
+                if isinstance(x, basestring) and (
+                        x.startswith('$HERE/') or x.startswith('$HERE\\')):
+                    if here is None:
+                        raise ValueError("prefix '$HERE' cannot be replaced")
+                    x = os.path.join(here, x[6:])
+                    value[i] = x
+
     def generates_python_module(self):
         return self._vengine._gen_python_module
 
diff --git a/testing/test_verify.py b/testing/test_verify.py
--- a/testing/test_verify.py
+++ b/testing/test_verify.py
@@ -2037,3 +2037,16 @@
             n = (1 << 29) + i
             lib.SetLastError(n)
             assert ffi.getwinerror()[0] == n
+
+def test_verify_include_dir():
+    curdir = os.getcwd()
+    try:
+        os.chdir('..')
+        ffi = FFI()
+        ffi.cdef("int v_include_dir(void);")
+        lib = ffi.verify('#include "verify_include_dir.h"',
+                         include_dirs=['$HERE/snippets/'],
+                         sources=['$HERE/snippets/verify_include_dir.c'])
+        assert lib.v_include_dir() == 42
+    finally:
+        os.chdir(curdir)

From noreply at buildbot.pypy.org  Fri Nov 28 12:58:00 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Fri, 28 Nov 2014 12:58:00 +0100 (CET)
Subject: [pypy-commit] creflect default: Support for "typedef ... sometype; "
Message-ID: <20141128115800.37EC11C07BB@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r99:4c7a226a0be6
Date: 2014-11-28 12:58 +0100
http://bitbucket.org/cffi/creflect/changeset/4c7a226a0be6/

Log:	Support for "typedef ... sometype;"

diff --git a/creflect/cparser.py b/creflect/cparser.py
--- a/creflect/cparser.py
+++ b/creflect/cparser.py
@@ -28,7 +28,8 @@
 r_comment1 = re.compile(r"//.*")
 r_comment2 = re.compile(r"/[*]")
 r_comment3 = re.compile(r"[*]/")
-r_empty_braces = re.compile(r"{\s*}")     # after comments have been removed
+r_empty_braces = re.compile(r"{(\s*)}")     # after comments have been removed
+r_typedef_dotdodot = re.compile(r"\btypedef(\s*)[.][.][.]")
 
 def remove_comments(csource):
     csource = r_comment1.sub('', csource)     # remove the '//' comments
@@ -50,7 +51,10 @@
     return ''.join(parts)
 
 def expand_empty_braces(csource):
-    return r_empty_braces.sub('{ char __crx_empty__; }', csource)
+    return r_empty_braces.sub(r'{ char __crx_empty__;\1}', csource)
+
+def expand_typedef_dotdotdot(csource):
+    return r_typedef_dotdodot.sub(r'typedef\1 __crx_unknown_t ', csource)
 
 
 class CSource(object):
@@ -60,10 +64,15 @@
         self.first_lineno = first_lineno
         self._struct_union_enum_decl = {}
 
-    def to_ast(self):
-        cparser = pycparser.CParser()
+    def prepare_source(self):
         csource = remove_comments(self.cdefblock)
         csource = expand_empty_braces(csource)
+        csource = expand_typedef_dotdotdot(csource)
+        return 'typedef int __crx_unknown_t; ' + csource
+
+    def to_ast(self):
+        csource = self.prepare_source()
+        cparser = pycparser.CParser()
         try:
             ast = cparser.parse(csource)
         except pycparser.c_parser.ParseError as e:
@@ -85,7 +94,8 @@
         self.declarations = []
         self.typedefs = {}
         ast = self.to_ast()
-        for decl in ast.ext:
+        assert ast.ext[0].name == '__crx_unknown_t'
+        for decl in ast.ext[1:]:
             if isinstance(decl, pycparser.c_ast.Decl):
                 self._parse_decl(decl)
             elif isinstance(decl, pycparser.c_ast.Typedef):
@@ -148,9 +158,13 @@
         # first, dereference typedefs, if we have it already parsed, we're good
         if (isinstance(typenode, pycparser.c_ast.TypeDecl) and
             isinstance(typenode.type, pycparser.c_ast.IdentifierType) and
-            len(typenode.type.names) == 1 and
-            typenode.type.names[0] in self.typedefs):
-            return self.typedefs[typenode.type.names[0]]
+            len(typenode.type.names) == 1):
+            typename = typenode.type.names[0]
+            if typename in self.typedefs:
+                return self.typedefs[typenode.type.names[0]]
+            if typename == '__crx_unknown_t':
+                assert approx_name, "XXX"
+                return model.UnknownType(approx_name)
         #
         if isinstance(typenode, pycparser.c_ast.ArrayDecl):
             # array type
diff --git a/creflect/model.py b/creflect/model.py
--- a/creflect/model.py
+++ b/creflect/model.py
@@ -476,6 +476,20 @@
             self.name,))
 
 
+class UnknownType(BaseType):
+    _attrs_ = ('approx_name',)
+    const = False
+
+    def __init__(self, approx_name):
+        self.approx_name = approx_name
+        self.c_name_with_marker = '... &'
+
+    def inspect_nonconst_type(self, block, inspect):
+        inspect.flush()
+        return block.write_crx_type_var('cb->get_unknown_type(cb, "%s")' % (
+            self.approx_name,))
+
+
 # ____________________________________________________________
 
 
diff --git a/creflect/src/creflect.h b/creflect/src/creflect.h
--- a/creflect/src/creflect.h
+++ b/creflect/src/creflect.h
@@ -52,6 +52,7 @@
     crx_type_t *(*get_union_type)(CRX_SELF, const char *);
     crx_type_t *(*get_enum_type)(CRX_SELF, const char *);
     crx_type_t *(*get_user_type)(CRX_SELF, const char *);
+    crx_type_t *(*get_unknown_type)(CRX_SELF, const char *);
     void (*complete)(CRX_SELF, crx_type_t *,
                      size_t, size_t, crx_field_t[], int);
     void (*complete_enum)(CRX_SELF, crx_type_t *, crx_type_t *);
diff --git a/creflect/src/creflect_print.h b/creflect/src/creflect_print.h
--- a/creflect/src/creflect_print.h
+++ b/creflect/src/creflect_print.h
@@ -167,6 +167,11 @@
     return newtype(name);
 }
 
+static crx_type_t *tst_get_unknown_type(crx_builder_t *cb, const char *name)
+{
+    return newtype2("UNKNOWN ", name);
+}
+
 static void tst_complete(crx_builder_t *cb, crx_type_t *t,
                          size_t sz, size_t align,
                          crx_field_t fields[], int nfields)
@@ -258,6 +263,7 @@
     tst_get_union_type,
     tst_get_enum_type,
     tst_get_user_type,
+    tst_get_unknown_type,
     tst_complete,
     tst_complete_enum,
     tst_define_type,
diff --git a/test/codegen/007.c b/test/codegen/007.c
new file mode 100644
--- /dev/null
+++ b/test/codegen/007.c
@@ -0,0 +1,13 @@
+typedef ... num_t;
+
+# ____________________________________________________________
+
+void test007(crx_builder_t *cb)
+{
+    crx_type_t *t1;
+    {
+        t1 = cb->get_unknown_type(cb, "$num_t");
+        cb->define_type(cb, "num_t", t1);
+#expect TYPEDEF num_t = UNKNOWN $num_t
+    }
+}
diff --git a/test/test_cgcompile.py b/test/test_cgcompile.py
--- a/test/test_cgcompile.py
+++ b/test/test_cgcompile.py
@@ -25,6 +25,8 @@
     f.write('\n')
     for line in inputlines:
         if not (line.startswith('# ') or line.startswith('#expect')):
+            if line.startswith('typedef ...'):
+                line = 'typedef struct _missing ' + line[11:]
             f.write(line)
     f.write('\n')
     f.write('#include "%s/../../creflect/src/creflect_print.h"\n' % path)

From noreply at buildbot.pypy.org  Fri Nov 28 13:09:54 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Fri, 28 Nov 2014 13:09:54 +0100 (CET)
Subject: [pypy-commit] creflect default: Support '#define FOO '
Message-ID: <20141128120954.B36701D35D7@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r100:1d714ead0a23
Date: 2014-11-28 13:10 +0100
http://bitbucket.org/cffi/creflect/changeset/1d714ead0a23/

Log:	Support '#define FOO '

diff --git a/creflect/cparser.py b/creflect/cparser.py
--- a/creflect/cparser.py
+++ b/creflect/cparser.py
@@ -30,6 +30,9 @@
 r_comment3 = re.compile(r"[*]/")
 r_empty_braces = re.compile(r"{(\s*)}")     # after comments have been removed
 r_typedef_dotdodot = re.compile(r"\btypedef(\s*)[.][.][.]")
+r_define_var = re.compile(
+    r"^[ \t]*#[ \t]*define[ \t]+([A-Za-z_][A-Za-z0-9_]+)[ \t][^\n]+",
+    re.MULTILINE)
 
 def remove_comments(csource):
     csource = r_comment1.sub('', csource)     # remove the '//' comments
@@ -56,6 +59,9 @@
 def expand_typedef_dotdotdot(csource):
     return r_typedef_dotdodot.sub(r'typedef\1 __crx_unknown_t ', csource)
 
+def expand_define_var(csource):
+    return r_define_var.sub(r'const int \1;', csource)
+
 
 class CSource(object):
 
@@ -68,6 +74,7 @@
         csource = remove_comments(self.cdefblock)
         csource = expand_empty_braces(csource)
         csource = expand_typedef_dotdotdot(csource)
+        csource = expand_define_var(csource)
         return 'typedef int __crx_unknown_t; ' + csource
 
     def to_ast(self):
diff --git a/test/codegen/macro-001.c b/test/codegen/macro-001.c
new file mode 100644
--- /dev/null
+++ b/test/codegen/macro-001.c
@@ -0,0 +1,23 @@
+#define FOO  42
+#define BAR  (40U + 2)
+
+# ____________________________________________________________
+
+void testmacro_001(crx_builder_t *cb)
+{
+    crx_type_t *t1, *t2;
+    {
+        crx_num_const_t v;
+        (void)((FOO) << 1);  /* check that 'FOO' is an integer */
+        t1 = CRX_INT_CONST(cb, FOO, &v, 1);
+        cb->define_num_const(cb, "FOO", t1, &v);
+#expect NUMCONST FOO = int 42
+    }
+    {
+        crx_num_const_t v;
+        (void)((BAR) << 1);  /* check that 'BAR' is an integer */
+        t2 = CRX_INT_CONST(cb, BAR, &v, 1);
+        cb->define_num_const(cb, "BAR", t2, &v);
+#expect NUMCONST BAR = unsigned int 42
+    }
+}

From noreply at buildbot.pypy.org  Fri Nov 28 13:13:36 2014
From: noreply at buildbot.pypy.org (fijal)
Date: Fri, 28 Nov 2014 13:13:36 +0100 (CET)
Subject: [pypy-commit] extradoc extradoc: Thank-you-psf-2
Message-ID: <20141128121336.A4E231D3612@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: extradoc
Changeset: r5467:31bd5aaffc61
Date: 2014-11-28 14:13 +0200
http://bitbucket.org/pypy/extradoc/changeset/31bd5aaffc61/

Log:	Thank-you-psf-2

diff --git a/blog/draft/io-improvements.rst b/blog/draft/io-improvements.rst
--- a/blog/draft/io-improvements.rst
+++ b/blog/draft/io-improvements.rst
@@ -32,7 +32,7 @@
 deviation comes mostly from the warmup at the beginning, true figures
 are smaller):
 
-+----------------+----- ------------+-------------------------+--------------------+
++----------------+------------------+-------------------------+--------------------+
 | benchmark      | CPython          | PyPy 2.4                | PyPy non-zero      |
 +----------------+------------------+-------------------------+--------------------+
 | fibonacci      | 4.8+-0.15 (1.0x) | 0.59+-0.07 (8.1x)       | 0.45+-0.07 (10.6x) |
diff --git a/blog/draft/thank-you-psf-2.rst b/blog/draft/thank-you-psf-2.rst
new file mode 100644
--- /dev/null
+++ b/blog/draft/thank-you-psf-2.rst
@@ -0,0 +1,34 @@
+September donations and thank you to the Python Software Foundation!
+====================================================================
+
+Hello everyone!
+
+We would like to show you a short update on the PyPy funding.
+We gathered a total of $15,986 in the month of September and as per
+`earlier agreement`_, the Python Software Foundation donated $10,000
+to PyPy. We would like to thank everyone participating and the PSF in
+particular for supporting the PyPy project and making our work possible!
+
+We've been working hard on the goals outlined in the funding proposals.
+
+* `PyPy Python 3`_ support has been in beta for a while and it's already
+  being used by many people, as seen per the number of reported bugs.
+  We're currently supporting 3.2, planning on moving towards 3.4 in the
+  future.
+
+* Software Transactional Memory has been a successful research project,
+  with `first real world`_ results shown during the Warsaw sprint.
+
+* More detailed update on numpy will be published soon. A little spoiler is
+  that we're planning on addressing matplotlib, scipy and the larger ecosystem
+  to some extent. Stay tuned!
+
+Again, thanks to everyone who donated and happy Thanksgiving to everyone
+on that side of the world!
+
+Cheers,
+fijal
+
+.. _`earlier agreement`: http://morepypy.blogspot.com/2014/09/python-software-foundation-matching.html
+.. _`first real world`: http://morepypy.blogspot.com/2014/11/tornado-without-gil-on-pypy-stm.html
+.. _`PyPy Python 3`:  http://morepypy.blogspot.com/2014/10/pypy3-240-released.html

From noreply at buildbot.pypy.org  Fri Nov 28 13:40:07 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Fri, 28 Nov 2014 13:40:07 +0100 (CET)
Subject: [pypy-commit] extradoc extradoc: Add my own comments in three places
Message-ID: <20141128124007.E80FE1C03DC@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: extradoc
Changeset: r5468:d9be15557668
Date: 2014-11-17 14:10 +0100
http://bitbucket.org/pypy/extradoc/changeset/d9be15557668/

Log:	Add my own comments in three places

diff --git a/blog/draft/tornado-stm.rst b/blog/draft/tornado-stm.rst
--- a/blog/draft/tornado-stm.rst
+++ b/blog/draft/tornado-stm.rst
@@ -170,7 +170,7 @@
     File "/home/arigo/hg/pypy/stmgc-c7/lib-python/2.7/_weakrefset.py", line 70, in __contains__
     File "/home/arigo/hg/pypy/stmgc-c7/lib-python/2.7/_weakrefset.py", line 70, in __contains__
 
-**FIXME** why does it happen?
+[Armin: It is unclear why this happens so far.  We'll investigate...]
 
 The second conflict (without etag tweaks) originates
 in the transaction module, from this piece of code::
@@ -180,7 +180,9 @@
                     got_exception)
         counter[0] += 1
 
-**FIXME** why does it happen?
+[Armin: This is a conflict in the transaction module itself; ideally,
+it shouldn't have any, but in order to do that we might need a little bit
+of support from RPython or C code.  So this is pending improvement.]
 
 Tornado modification used in this blog post is based on 3.2.dev2.
 As of now, the latest version is 4.0.2, and if we
@@ -188,6 +190,14 @@
 the same changes to this version, then we no longer get any scaling on this benchmark,
 and there are no conflicts that take any substantial time.
 
+[Armin: There are two possible reactions to a conflict.  We can either
+abort one of the two threads, or (depending on the circumstances) just
+pause the current thread until the other one commits, after which the
+thread will likely be able to continue.  The tool ``print_stm_log.py``
+did not report conflicts that cause pauses.  It has been fixed very
+recently.  Chances are that on this test it would report long pauses and
+point to locations that cause them.]
+
 
 Part 2: a more interesting benchmark: A-star
 --------------------------------------------

From noreply at buildbot.pypy.org  Fri Nov 28 13:40:09 2014
From: noreply at buildbot.pypy.org (kostialopuhin)
Date: Fri, 28 Nov 2014 13:40:09 +0100 (CET)
Subject: [pypy-commit] extradoc extradoc: add an image with tornado stm
	benchmark results
Message-ID: <20141128124009.49C4F1C03DC@cobra.cs.uni-duesseldorf.de>

Author: Konstantin Lopuhin 
Branch: extradoc
Changeset: r5469:2a47233c651b
Date: 2014-11-17 16:29 +0300
http://bitbucket.org/pypy/extradoc/changeset/2a47233c651b/

Log:	add an image with tornado stm benchmark results

diff --git a/blog/draft/tornado-stm-results.png b/blog/draft/tornado-stm-results.png
new file mode 100644
index 0000000000000000000000000000000000000000..1913e33f3ded79d5597a5787088581927e2a294f
GIT binary patch

[cut]

diff --git a/blog/draft/tornado-stm.rst b/blog/draft/tornado-stm.rst
--- a/blog/draft/tornado-stm.rst
+++ b/blog/draft/tornado-stm.rst
@@ -124,7 +124,7 @@
 PyPy STM 4      24.2
 ============  =========
 
-.. image:: results-1.png
+.. image:: tornado-stm-results.png
 
 As we can see, in this benchmark PyPy STM using just two cores
 can beat regular PyPy!

From noreply at buildbot.pypy.org  Fri Nov 28 13:40:10 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Fri, 28 Nov 2014 13:40:10 +0100 (CET)
Subject: [pypy-commit] extradoc extradoc: Updates
Message-ID: <20141128124010.6821B1C03DC@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: extradoc
Changeset: r5470:f326c47b119c
Date: 2014-11-17 14:40 +0100
http://bitbucket.org/pypy/extradoc/changeset/f326c47b119c/

Log:	Updates

diff --git a/blog/draft/tornado-stm.rst b/blog/draft/tornado-stm.rst
--- a/blog/draft/tornado-stm.rst
+++ b/blog/draft/tornado-stm.rst
@@ -1,6 +1,9 @@
 Tornado without a GIL on PyPy STM
 =================================
 
+*This post is by Konstantin Lopuhin, who tried PyPy STM during the
+Warsaw sprint.*
+
 Python has a GIL, right? Not quite - PyPy STM is a python implementation
 without a GIL, so it can scale CPU-bound work to several cores.
 PyPy STM is developed by Armin Rigo and Remi Meier,
@@ -170,7 +173,7 @@
     File "/home/arigo/hg/pypy/stmgc-c7/lib-python/2.7/_weakrefset.py", line 70, in __contains__
     File "/home/arigo/hg/pypy/stmgc-c7/lib-python/2.7/_weakrefset.py", line 70, in __contains__
 
-[Armin: It is unclear why this happens so far.  We'll investigate...]
+*Comment by Armin: It is unclear why this happens so far.  We'll investigate...*
 
 The second conflict (without etag tweaks) originates
 in the transaction module, from this piece of code::
@@ -180,9 +183,9 @@
                     got_exception)
         counter[0] += 1
 
-[Armin: This is a conflict in the transaction module itself; ideally,
+*Comment by Armin: This is a conflict in the transaction module itself; ideally,
 it shouldn't have any, but in order to do that we might need a little bit
-of support from RPython or C code.  So this is pending improvement.]
+of support from RPython or C code.  So this is pending improvement.*
 
 Tornado modification used in this blog post is based on 3.2.dev2.
 As of now, the latest version is 4.0.2, and if we
@@ -190,13 +193,13 @@
 the same changes to this version, then we no longer get any scaling on this benchmark,
 and there are no conflicts that take any substantial time.
 
-[Armin: There are two possible reactions to a conflict.  We can either
+*Comment by Armin: There are two possible reactions to a conflict.  We can either
 abort one of the two threads, or (depending on the circumstances) just
 pause the current thread until the other one commits, after which the
 thread will likely be able to continue.  The tool ``print_stm_log.py``
 did not report conflicts that cause pauses.  It has been fixed very
 recently.  Chances are that on this test it would report long pauses and
-point to locations that cause them.]
+point to locations that cause them.*
 
 
 Part 2: a more interesting benchmark: A-star

From noreply at buildbot.pypy.org  Fri Nov 28 13:40:11 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Fri, 28 Nov 2014 13:40:11 +0100 (CET)
Subject: [pypy-commit] extradoc extradoc: merge heads
Message-ID: <20141128124011.8A0A31C03DC@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: extradoc
Changeset: r5471:dde71b6447a4
Date: 2014-11-17 14:40 +0100
http://bitbucket.org/pypy/extradoc/changeset/dde71b6447a4/

Log:	merge heads

diff --git a/blog/draft/tornado-stm-results.png b/blog/draft/tornado-stm-results.png
new file mode 100644
index 0000000000000000000000000000000000000000..1913e33f3ded79d5597a5787088581927e2a294f
GIT binary patch

[cut]

diff --git a/blog/draft/tornado-stm.rst b/blog/draft/tornado-stm.rst
--- a/blog/draft/tornado-stm.rst
+++ b/blog/draft/tornado-stm.rst
@@ -127,7 +127,7 @@
 PyPy STM 4      24.2
 ============  =========
 
-.. image:: results-1.png
+.. image:: tornado-stm-results.png
 
 As we can see, in this benchmark PyPy STM using just two cores
 can beat regular PyPy!

From noreply at buildbot.pypy.org  Fri Nov 28 13:40:12 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Fri, 28 Nov 2014 13:40:12 +0100 (CET)
Subject: [pypy-commit] extradoc extradoc: merge heads
Message-ID: <20141128124012.CA2041C03DC@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: extradoc
Changeset: r5472:255fb1879b2b
Date: 2014-11-28 13:40 +0100
http://bitbucket.org/pypy/extradoc/changeset/255fb1879b2b/

Log:	merge heads

diff --git a/blog/draft/io-improvements.rst b/blog/draft/io-improvements.rst
--- a/blog/draft/io-improvements.rst
+++ b/blog/draft/io-improvements.rst
@@ -32,7 +32,7 @@
 deviation comes mostly from the warmup at the beginning, true figures
 are smaller):
 
-+----------------+----- ------------+-------------------------+--------------------+
++----------------+------------------+-------------------------+--------------------+
 | benchmark      | CPython          | PyPy 2.4                | PyPy non-zero      |
 +----------------+------------------+-------------------------+--------------------+
 | fibonacci      | 4.8+-0.15 (1.0x) | 0.59+-0.07 (8.1x)       | 0.45+-0.07 (10.6x) |
diff --git a/blog/draft/thank-you-psf-2.rst b/blog/draft/thank-you-psf-2.rst
new file mode 100644
--- /dev/null
+++ b/blog/draft/thank-you-psf-2.rst
@@ -0,0 +1,34 @@
+September donations and thank you to the Python Software Foundation!
+====================================================================
+
+Hello everyone!
+
+We would like to show you a short update on the PyPy funding.
+We gathered a total of $15,986 in the month of September and as per
+`earlier agreement`_, the Python Software Foundation donated $10,000
+to PyPy. We would like to thank everyone participating and the PSF in
+particular for supporting the PyPy project and making our work possible!
+
+We've been working hard on the goals outlined in the funding proposals.
+
+* `PyPy Python 3`_ support has been in beta for a while and it's already
+  being used by many people, as seen per the number of reported bugs.
+  We're currently supporting 3.2, planning on moving towards 3.4 in the
+  future.
+
+* Software Transactional Memory has been a successful research project,
+  with `first real world`_ results shown during the Warsaw sprint.
+
+* More detailed update on numpy will be published soon. A little spoiler is
+  that we're planning on addressing matplotlib, scipy and the larger ecosystem
+  to some extent. Stay tuned!
+
+Again, thanks to everyone who donated and happy Thanksgiving to everyone
+on that side of the world!
+
+Cheers,
+fijal
+
+.. _`earlier agreement`: http://morepypy.blogspot.com/2014/09/python-software-foundation-matching.html
+.. _`first real world`: http://morepypy.blogspot.com/2014/11/tornado-without-gil-on-pypy-stm.html
+.. _`PyPy Python 3`:  http://morepypy.blogspot.com/2014/10/pypy3-240-released.html
diff --git a/talk/dls2012/benchmarks/benchmark.sh b/talk/dls2012/benchmarks/benchmark.sh
--- a/talk/dls2012/benchmarks/benchmark.sh
+++ b/talk/dls2012/benchmarks/benchmark.sh
@@ -59,13 +59,21 @@
     #$* ./runner.py $EXTRA_OPTS convolution/convolution.py conv3 1
     #$* ./runner.py $EXTRA_OPTS convolution/convolution.py conv5 1
     $* ./runner.py $EXTRA_OPTS convolution/convolution.py conv3 100
+    $* ./runner.py $EXTRA_OPTS convolution/convolution.py conv3_numpy 100
     $* ./runner.py $EXTRA_OPTS convolution/convolution.py conv5 100
+    $* ./runner.py $EXTRA_OPTS convolution/convolution.py conv5_numpy 100
     $* ./runner.py $EXTRA_OPTS convolution/convolution.py conv3 1000
+    $* ./runner.py $EXTRA_OPTS convolution/convolution.py conv3_numpy 1000
     $* ./runner.py $EXTRA_OPTS convolution/convolution.py conv5 1000
+    $* ./runner.py $EXTRA_OPTS convolution/convolution.py conv5_numpy 1000
     $* ./runner.py $EXTRA_OPTS convolution/convolution.py conv3x3 1000000 3
+    $* ./runner.py $EXTRA_OPTS convolution/convolution.py conv3x3_numpy 1000000 3
     $* ./runner.py $EXTRA_OPTS convolution/convolution.py conv3x3 1000 1000
+    $* ./runner.py $EXTRA_OPTS convolution/convolution.py conv3x3_numpy 1000 1000
     $* ./runner.py $EXTRA_OPTS convolution/convolution.py dilate3x3 1000 1000
+    $* ./runner.py $EXTRA_OPTS convolution/convolution.py dilate3x3_numpy 1000 1000
     $* ./runner.py $EXTRA_OPTS convolution/convolution.py sobel_magnitude 1000 1000
+    $* ./runner.py $EXTRA_OPTS convolution/convolution.py sobel_magnitude_numpy 1000 1000
     #$* ./runner.py $EXTRA_OPTS image/noborder.py main NoBorderImagePadded
     #$* ./runner.py $EXTRA_OPTS image/noborder.py main NoBorderImagePadded iter
     #$* ./runner.py $EXTRA_OPTS image/noborder.py main NoBorderImagePadded range
diff --git a/talk/dls2012/benchmarks/convolution/convolution.py b/talk/dls2012/benchmarks/convolution/convolution.py
--- a/talk/dls2012/benchmarks/convolution/convolution.py
+++ b/talk/dls2012/benchmarks/convolution/convolution.py
@@ -1,5 +1,13 @@
 from array import array
 from math import log10, sqrt
+try:
+    import numpy as np
+except ImportError:
+    try:
+        import numpypy as np
+    except ImportError:
+        print "Cant find nympy"
+
 
 def _conv3(a, k, n=1):
     assert len(k)==3
@@ -14,7 +22,22 @@
     n = int(args[0])
     _conv3(array('d', [1]) * (100000000/n),
            array('d', [-1, 0, 1]), n)
-    return 'conv3(array(1e%d))' % log10(100000000/n)
+    return 'conv3(array.array(1e%d))' % log10(100000000/n)
+
+def _conv3_numpy(a, k, n=1):
+    assert len(k)==3
+    b = np.zeros(len(a) - 2, a.dtype)
+    while n:
+        n -= 1
+        for i in xrange(len(b)):
+            b[i] = k[2]*a[i] + k[1]*a[i+1] + k[0]*a[i+2]
+    return b
+
+def conv3_numpy(args):
+    n = int(args[0])
+    _conv3_numpy(np.ones(100000000/n, 'd'),
+           np.array([-1, 0, 1], 'd'), n)
+    return 'conv3(numpy.array(1e%d))' % log10(100000000/n)
 
 def _conv5(a, k, n=1):
     assert len(k)==5
@@ -29,7 +52,22 @@
     n = int(args[0])
     _conv5(array('d', [1]) * (100000000/n),
            array('d', [1, 4, 6, 4, 1]), n)
-    return 'conv5(array(1e%d))' % log10(100000000/n)
+    return 'conv5(array.array(1e%d))' % log10(100000000/n)
+
+def _conv5_numpy(a, k, n=1):
+    assert len(k)==5
+    b = np.zeros(len(a) - 4, a.dtype)
+    while n:
+        n -= 1
+        for i in xrange(len(b)):
+            b[i] = k[4]*a[i] + k[3]*a[i+1] + k[2]*a[i+2] + k[1]*a[i+3] + k[0]*a[i+4]
+    return b
+
+def conv5_numpy(args):
+    n = int(args[0])
+    _conv5_numpy(np.ones(100000000/n, 'd'),
+           np.array([1, 4, 6, 4, 1], 'd'), n)
+    return 'conv5(numpy.array(1e%d))' % log10(100000000/n)
 
 class Array2D(object):
     def __init__(self, w, h, data=None):
@@ -71,13 +109,16 @@
     def __init__(self, w, h):
         self.width = w
         self.height = h
-        import numpypy
-        self.data = numpypy.zeros([h, w], 'd')
+        self.data = np.zeros([h, w], 'd')
 
     def __getitem__(self, (x, y)):
+        if x < 0 or y < 0:
+            raise IndexError
         return self.data[y, x]
 
     def __setitem__(self, (x, y), val):
+        if x < 0 or y < 0:
+            raise IndexError
         self.data[y, x] = val
 
 def _conv3x3(a, b, k):
@@ -125,6 +166,13 @@
         _dilate3x3(a, b, Array2D(3,3))
     return 'dilate3x3(Array2D(%sx%s))' % tuple(args)
 
+def dilate3x3_numpy(args):
+    a = NumpyArray(int(args[0]), int(args[1]))
+    b = NumpyArray(a.width, a.height)
+    for i in range(10):
+        _dilate3x3(a, b, NumpyArray(3,3))
+    return 'dilate3x3(NumpyArray(%sx%s))' % tuple(args)
+
 def _sobel_magnitude(a):
     b = Array2D(a.width, a.height)    
     for y in xrange(1, a.height-1):
@@ -141,3 +189,8 @@
     for i in range(10):
         _sobel_magnitude(Array2D(int(args[0]), int(args[1])))
     return 'sobel(Array2D(%sx%s))' % tuple(args)
+
+def sobel_magnitude_numpy(args):
+    for i in range(10):
+        _sobel_magnitude(NumpyArray(int(args[0]), int(args[1])))
+    return 'sobel(NumpyArray(%sx%s))' % tuple(args)
diff --git a/talk/dls2012/benchmarks/iter/ndindex.py b/talk/dls2012/benchmarks/iter/ndindex.py
new file mode 100644
--- /dev/null
+++ b/talk/dls2012/benchmarks/iter/ndindex.py
@@ -0,0 +1,145 @@
+from numpy import ndindex, array, ones, tile
+
+range1 = range2 = ndindex
+
+def _sum1d(a):
+    sa = 0
+    for i, in range1(len(a)):
+        sa += a[i]
+
+def _xsum1d(a):
+    sa = 0
+    for i, in range1(len(a)):
+        sa += a[i] + i
+
+def _wsum1d(a):
+    sa = 0
+    for i, in range1(len(a)):
+        sa += a[i] + len(a)
+
+def _sum2d(a, w, h):
+    sa = 0
+    for x, y in range2(w, h):
+        sa += a[y, x]
+
+def _wsum2d(a, w, h):
+    sa = 0
+    for x, y in range2(w, h):
+        sa += a[y, x] + w
+
+def _xsum2d(a, w, h):
+    sa = 0
+    for x, y in range2(w, h):
+        sa += a[y, x] + x
+
+def _whsum2d(a, w, h):
+    sa = 0
+    for x, y in range2(w, h):
+        sa += a[y, x] + w + h
+
+def _xysum2d(a, w, h):
+    sa = 0
+    for x, y in range2(w, h):
+        sa += a[y, x] + x + y
+
+def _mean1d(a):
+    sa = 0
+    for i, in range1(len(a)):
+        sa = (i*sa + a[i])/(i + 1.0);
+        
+def _median1d(a):
+    sa = 0
+    for i, in range1(len(a)):
+        if sa > a[i]:
+            sa -= 1.0/(i + 1.0)
+        elif sa < a[i]:
+            sa += 1.0/(i + 1.0)
+
+def _ripple1d(a):
+    sa = 0
+    for i, in range1(len(a)):
+        if sa > a[i]:
+            sa -= 0.1
+        elif sa < a[i]:
+            sa += 0.1
+
+def _ripple2d(a, w, h):
+    sa = 0
+    for x, y in range2(w, h):
+        if sa > a[y, x]:
+            sa -= 0.1
+        elif sa < a[y, x]:
+            sa += 0.1
+
+def sum1d(args):
+    run1d(args, _sum1d)
+    return "sum1d"
+
+def xsum1d(args):
+    run1d(args, _xsum1d)
+    return "xsum1d"
+
+def wsum1d(args):
+    run1d(args, _wsum1d)
+    return "wsum1d"
+
+def sum2d(args):
+    run2d(args, _sum2d)
+    return "sum2d"
+
+def wsum2d(args):
+    run2d(args, _wsum2d)
+    return "wsum2d"
+
+def xsum2d(args):
+    run2d(args, _xsum2d)
+    return "xsum2d"
+
+def whsum2d(args):
+    run2d(args, _whsum2d)
+    return "whsum2d"
+
+def xysum2d(args):
+    run2d(args, _xysum2d)
+    return "xysum2d"
+
+def mean1d(args):
+    run1d(args, _mean1d, [1, -1])
+    return "mean1d"
+
+def median1d(args):
+    run1d(args, _median1d, [1, -1])
+    return "median1d"
+
+def ripple1d(args):
+    run1d(args, _ripple1d, [1, -1])
+    return "ripple1d"
+
+def ripple2d(args):
+    run2d(args, _ripple2d, [1, -1])
+    return "ripple2d"
+
+def run1d(args, f, data=None):
+    if data:
+        a = tile(array(data), 100000000/len(data))
+    else:
+        a = ones(100000000)
+    n = int(args[0])
+    for i in xrange(n):
+        f(a)
+    return "sum1d"
+
+def run2d(args, f, data=None):
+    if data:
+        a = tile(array(data), 100000000/len(data)).reshape((10000, 10000))
+    else:
+        a = ones(100000000).reshape((10000, 10000))
+    n = int(args[0])
+    for i in xrange(n):
+        f(a, 10000, 10000)
+    return "sum1d"
+
+if __name__ == '__main__':
+    import sys
+    eval(sys.argv[1])(sys.argv[2:])
+    
diff --git a/talk/dls2012/benchmarks/iter/nditer.py b/talk/dls2012/benchmarks/iter/nditer.py
new file mode 100644
--- /dev/null
+++ b/talk/dls2012/benchmarks/iter/nditer.py
@@ -0,0 +1,166 @@
+from numpy import nditer, array, ones, tile
+
+def _sum1d(a):
+    sa = 0
+    it = nditer(a, flags=['f_index'])
+    for v in it:
+        i = it.index    
+        sa += a[i]
+
+def _xsum1d(a):
+    sa = 0
+    it = nditer(a, flags=['f_index'])
+    for v in it:
+        i = it.index    
+        sa += a[i] + i
+
+def _wsum1d(a):
+    sa = 0
+    it = nditer(a, flags=['f_index'])
+    for v in it:
+        i = it.index    
+        sa += a[i] + len(a)
+
+def _sum2d(a, w, h):
+    sa = 0
+    it = nditer(a, flags=['multi_index'])
+    for v in it:
+        y, x = it.multi_index
+        sa += a[y, x]
+
+def _wsum2d(a, w, h):
+    sa = 0
+    it = nditer(a, flags=['multi_index'])
+    for v in it:
+        y, x = it.multi_index
+        sa += a[y, x] + w
+
+def _xsum2d(a, w, h):
+    sa = 0
+    it = nditer(a, flags=['multi_index'])
+    for v in it:
+        y, x = it.multi_index
+        sa += a[y, x] + x
+
+def _whsum2d(a, w, h):
+    sa = 0
+    it = nditer(a, flags=['multi_index'])
+    for v in it:
+        y, x = it.multi_index
+        sa += a[y, x] + w + h
+
+def _xysum2d(a, w, h):
+    sa = 0
+    it = nditer(a, flags=['multi_index'])
+    for v in it:
+        y, x = it.multi_index
+        sa += a[y, x] + x + y
+
+def _mean1d(a):
+    sa = 0
+    it = nditer(a, flags=['f_index'])
+    for v in it:
+        i = it.index    
+        sa = (i*sa + a[i])/(i + 1.0);
+        
+def _median1d(a):
+    sa = 0
+    it = nditer(a, flags=['f_index'])
+    for v in it:
+        i = it.index    
+        if sa > a[i]:
+            sa -= 1.0/(i + 1.0)
+        elif sa < a[i]:
+            sa += 1.0/(i + 1.0)
+
+def _ripple1d(a):
+    sa = 0
+    it = nditer(a, flags=['f_index'])
+    for v in it:
+        i = it.index    
+        if sa > a[i]:
+            sa -= 0.1
+        elif sa < a[i]:
+            sa += 0.1
+
+def _ripple2d(a, w, h):
+    sa = 0
+    it = nditer(a, flags=['multi_index'])
+    for v in it:
+        y, x = it.multi_index
+        if sa > a[y, x]:
+            sa -= 0.1
+        elif sa < a[y, x]:
+            sa += 0.1
+
+def sum1d(args):
+    run1d(args, _sum1d)
+    return "sum1d"
+
+def xsum1d(args):
+    run1d(args, _xsum1d)
+    return "xsum1d"
+
+def wsum1d(args):
+    run1d(args, _wsum1d)
+    return "wsum1d"
+
+def sum2d(args):
+    run2d(args, _sum2d)
+    return "sum2d"
+
+def wsum2d(args):
+    run2d(args, _wsum2d)
+    return "wsum2d"
+
+def xsum2d(args):
+    run2d(args, _xsum2d)
+    return "xsum2d"
+
+def whsum2d(args):
+    run2d(args, _whsum2d)
+    return "whsum2d"
+
+def xysum2d(args):
+    run2d(args, _xysum2d)
+    return "xysum2d"
+
+def mean1d(args):
+    run1d(args, _mean1d, [1, -1])
+    return "mean1d"
+
+def median1d(args):
+    run1d(args, _median1d, [1, -1])
+    return "median1d"
+
+def ripple1d(args):
+    run1d(args, _ripple1d, [1, -1])
+    return "ripple1d"
+
+def ripple2d(args):
+    run2d(args, _ripple2d, [1, -1])
+    return "ripple2d"
+
+def run1d(args, f, data=None):
+    if data:
+        a = tile(array(data), 100000000/len(data))
+    else:
+        a = ones(100000000)
+    n = int(args[0])
+    for i in xrange(n):
+        f(a)
+    return "sum1d"
+
+def run2d(args, f, data=None):
+    if data:
+        a = tile(array(data), 100000000/len(data)).reshape((10000, 10000))
+    else:
+        a = ones(100000000).reshape((10000, 10000))
+    n = int(args[0])
+    for i in xrange(n):
+        f(a, 10000, 10000)
+    return "sum1d"
+
+if __name__ == '__main__':
+    import sys
+    eval(sys.argv[1])(sys.argv[2:])
diff --git a/talk/dls2012/benchmarks/iter/result-2.4.0.txt b/talk/dls2012/benchmarks/iter/result-2.4.0.txt
new file mode 100644
--- /dev/null
+++ b/talk/dls2012/benchmarks/iter/result-2.4.0.txt
@@ -0,0 +1,100 @@
+gcc -O3
+sum1d: 0.83 +- 1.24126707662e-16
+sum2d: 0.83 +- 1.24126707662e-16
+whsum2d: 0.842 +- 0.004472135955
+3wsum1d: 0.836 +- 0.00894427191
+wsum2d: 0.85 +- 0.0308220700148
+xsum1d: 0.842 +- 0.004472135955
+xsum2d: 0.842 +- 0.004472135955
+xysum2d: 1.12 +- 0.0346410161514
+mean1d: 7.428 +- 0.0294957624075
+median1d: 3.818 +- 0.004472135955
+ripple1d: 1.342 +- 0.0109544511501
+ripple2d: 1.336 +- 0.00894427191
+
+pypy iter/generator.py
+sum1d: 5.53084101677 +- 0.00651376226379
+sum2d: 5.7555460453 +- 0.00951369332241
+whsum2d: 5.8612534523 +- 0.0505271222339
+wsum1d: 5.13269457817 +- 0.0823542879822
+wsum2d: 5.99619159698 +- 0.0487867098222
+xsum1d: 5.04685320854 +- 0.0555180883435
+xsum2d: 6.07883496284 +- 0.0389639282491
+xysum2d: 5.83931522369 +- 0.0576320488093
+mean1d: 8.94375824928 +- 0.0108197222492
+median1d: 10.4045877457 +- 0.0285781258496
+ripple1d: 10.2467153549 +- 0.00567696790862
+ripple2d: 11.1841029644 +- 0.139083424095
+
+pypy iter/generator2.py
+sum1d: 5.9797270298 +- 0.0755165051781
+sum2d: 5.80511965752 +- 0.380443555753
+whsum2d: 6.19872779846 +- 0.0262446391517
+wsum1d: 5.0686296463 +- 0.03220581952
+wsum2d: 6.22603621483 +- 0.0765020459155
+xsum1d: 5.03696541786 +- 0.0313417312818
+xsum2d: 6.64942345619 +- 0.0634006175674
+xysum2d: 5.85069346428 +- 0.024646797031
+mean1d: 9.20232362747 +- 0.27107580199
+median1d: 10.5072529793 +- 0.0780365503085
+ripple1d: 10.3291498184 +- 0.0457349492366
+ripple2d: 12.1278275967 +- 0.0184532784891
+
+pypy iter/iterator.py
+sum1d: 2.89783701897 +- 0.0338818402654
+sum2d: 6.21735162735 +- 0.0305362100956
+whsum2d: 5.78359918594 +- 0.0418847806897
+wsum1d: 2.90417222977 +- 0.00550225146282
+wsum2d: 6.1562063694 +- 0.0248465945318
+xsum1d: 3.15220880508 +- 0.0238542345497
+xsum2d: 6.17962999344 +- 0.00522105603458
+xysum2d: 6.54959263802 +- 0.0204275708962
+mean1d: 8.9222530365 +- 0.0569358104413
+median1d: 9.32725701332 +- 0.0118218937952
+ripple1d: 7.84541239738 +- 0.0128802667437
+ripple2d: 11.0337635994 +- 0.0381211395066
+
+pypy iter/range.py
+sum1d: 1.5154399395 +- 0.00218717543831
+sum2d: 1.78169260025 +- 0.0031213150465
+whsum2d: 1.78300223351 +- 0.00343094840578
+wsum1d: 1.51814541817 +- 0.00407496519924
+wsum2d: 1.78870997429 +- 0.0102636422345
+xsum1d: 1.78627576828 +- 0.0141348129086
+xsum2d: 2.04741225243 +- 0.00739649010433
+xysum2d: 2.04407844543 +- 0.00160511612186
+mean1d: 8.38623380661 +- 0.00678559642762
+median1d: 4.52713260651 +- 0.0215238656358
+ripple1d: 3.50191493034 +- 0.0344265982872
+ripple2d: 4.24880318642 +- 0.00425382282478
+
+pypy iter/while.py
+sum1d: 0.987529802322 +- 0.00129671178469
+sum2d: 1.75787782669 +- 0.00201957281403
+whsum2d: 1.75991163254 +- 0.000927917277915
+wsum1d: 0.989894151688 +- 0.00303482111217
+wsum2d: 1.76429200172 +- 0.00496186597436
+xsum1d: 1.23868374825 +- 0.00168004472652
+xsum2d: 2.0280834198 +- 0.00263820277497
+xysum2d: 2.02464160919 +- 0.00174334572372
+mean1d: 8.34133095741 +- 0.0121082272197
+median1d: 4.27610282898 +- 0.0313133386645
+ripple1d: 2.98665003777 +- 0.0134154060239
+ripple2d: 4.04827785492 +- 0.055327379039
+
+pypy iter/ndindex.py
+NotImplementedError: unable to create dtype from objects, "DummyArray" instance not supported
+
+pypy iter/nditer.py
+sum1d: 61.2064362049 +- 0.578041254203
+sum2d: 71.1426748753 +- 1.09482960038
+whsum2d: 73.6043967247 +- 0.592259791238
+wsum1d: 62.5866453171 +- 0.677394146874
+wsum2d: 71.0087907314 +- 0.309482073549
+xsum1d: 63.4650315285 +- 0.844882977961
+xsum2d: 72.2017237663 +- 0.807526950608
+xysum2d: 71.3824438095 +- 0.28602305068
+mean1d: 83.7468230247 +- 2.77559774136
+median1d: 84.9599477768 +- 0.460831996712
+ripple1d: 83.2095424175 +- 0.427835447938
+ripple2d: 107.806012583 +- 1.06730555212
diff --git a/talk/dls2012/benchmarks/runall.sh b/talk/dls2012/benchmarks/runall.sh
--- a/talk/dls2012/benchmarks/runall.sh
+++ b/talk/dls2012/benchmarks/runall.sh
@@ -1,6 +1,8 @@
 #!/bin/bash
 
 #./benchmark.sh pypy
+./benchmark.sh pypy-2.4.0
+./benchmark.sh pypy-2.0
 #./benchmark.sh pypy --jit enable_opts=intbounds:rewrite:virtualize:string:earlyforce:pure:heap:ffi
 #./benchmark.sh pypy-1.5
 #./benchmark.sh pypy-1.5 --jit enable_opts=intbounds:rewrite:virtualize:heap:unroll
@@ -8,10 +10,11 @@
 #./benchmark.sh gcc
 #./benchmark.sh gcc -O2
 #./benchmark.sh gcc -O3 -march=native -fno-tree-vectorize
+./benchmark.sh gcc -O3 -march=native
 #./benchmark.sh python2.7
 #./benchmark.sh python2.6 psyco-wrapper.py
 #./benchmark.sh luajit-2.0.0-beta10
 #./benchmark.sh luajit-2.0.0-beta10 -O-loop
 ./benchmark.sh luajit
-./benchmark.sh luajit -O-loop
+#./benchmark.sh luajit -O-loop
 #./benchmark.sh luajit
diff --git a/talk/dls2012/benchmarks/runiter.sh b/talk/dls2012/benchmarks/runiter.sh
--- a/talk/dls2012/benchmarks/runiter.sh
+++ b/talk/dls2012/benchmarks/runiter.sh
@@ -10,7 +10,7 @@
 for p in iter/*.py; do
     echo pypy $p
     for b in $BENCHMARKS; do
-	/tmp/pypy-trunk ./runner.py -n 5 $p $b 10
+	pypy ./runner.py -n 5 $p $b 10
     done
     echo
 done
\ No newline at end of file

From noreply at buildbot.pypy.org  Fri Nov 28 14:20:08 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Fri, 28 Nov 2014 14:20:08 +0100 (CET)
Subject: [pypy-commit] pypy default: fix test
Message-ID: <20141128132008.DDAAE1C0589@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r74751:4a150158f334
Date: 2014-11-28 14:19 +0100
http://bitbucket.org/pypy/pypy/changeset/4a150158f334/

Log:	fix test

diff --git a/rpython/jit/codewriter/test/test_flatten.py b/rpython/jit/codewriter/test/test_flatten.py
--- a/rpython/jit/codewriter/test/test_flatten.py
+++ b/rpython/jit/codewriter/test/test_flatten.py
@@ -297,21 +297,27 @@
             int_return $-1
             ---
             L1:
+            -live-
             int_return $61
             ---
             L2:
+            -live-
             int_return $511
             ---
             L3:
+            -live-
             int_return $-22
             ---
             L4:
+            -live-
             int_return $81
             ---
             L5:
+            -live-
             int_return $17
             ---
             L6:
+            -live-
             int_return $54
         """)
 

From noreply at buildbot.pypy.org  Fri Nov 28 14:25:18 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Fri, 28 Nov 2014 14:25:18 +0100 (CET)
Subject: [pypy-commit] pypy default: fix test
Message-ID: <20141128132518.60B7F1D3555@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r74752:32796dba665f
Date: 2014-11-28 14:24 +0100
http://bitbucket.org/pypy/pypy/changeset/32796dba665f/

Log:	fix test

diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
@@ -1,4 +1,4 @@
-import py
+import py, sys
 from rpython.rlib.objectmodel import instantiate
 from rpython.jit.metainterp import compile, resume
 from rpython.jit.metainterp.history import AbstractDescr, ConstInt, BoxInt, TreeLoop
@@ -190,6 +190,11 @@
             args = []
             for _ in range(oparity[opnum]):
                 args.append(random.randrange(1, 20))
+            if opnum == rop.INT_SIGNEXT:
+                # 2nd arg is number of bytes to extend from ---
+                # must not be too random
+                args[-1] = random.choice([1, 2] if sys.maxint < 2**32 else
+                                         [1, 2, 4])
             ops = """
             []
             i1 = %s(%s)

From noreply at buildbot.pypy.org  Fri Nov 28 14:57:43 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Fri, 28 Nov 2014 14:57:43 +0100 (CET)
Subject: [pypy-commit] pypy default: Support threadlocalref_{addr,
 get} when running on the llinterp too.
Message-ID: <20141128135743.028721C03DC@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r74753:0331b21a1685
Date: 2014-11-28 14:51 +0100
http://bitbucket.org/pypy/pypy/changeset/0331b21a1685/

Log:	Support threadlocalref_{addr,get} when running on the llinterp too.

diff --git a/rpython/jit/metainterp/test/test_threadlocal.py b/rpython/jit/metainterp/test/test_threadlocal.py
--- a/rpython/jit/metainterp/test/test_threadlocal.py
+++ b/rpython/jit/metainterp/test/test_threadlocal.py
@@ -11,10 +11,11 @@
         tlfield = rthread.ThreadLocalField(lltype.Signed, 'foobar_test_')
 
         def f():
+            tlfield.setraw(0x544c)
             return tlfield.getraw()
 
         res = self.interp_operations(f, [])
-        assert res == 0x544c    # magic value returned by llinterp
+        assert res == 0x544c
 
 
 class TestLLtype(ThreadLocalTest, LLJitMixin):
diff --git a/rpython/rlib/rthread.py b/rpython/rlib/rthread.py
--- a/rpython/rlib/rthread.py
+++ b/rpython/rlib/rthread.py
@@ -286,11 +286,13 @@
             _threadlocalref_seeme(self)
             return llop.threadlocalref_get(FIELDTYPE, offset)
 
+        @jit.dont_look_inside
         def get_or_make_raw():
             _threadlocalref_seeme(self)
             addr = llop.threadlocalref_addr(llmemory.Address)
             return llop.raw_load(FIELDTYPE, addr, offset)
 
+        @jit.dont_look_inside
         def setraw(value):
             _threadlocalref_seeme(self)
             addr = llop.threadlocalref_addr(llmemory.Address)
diff --git a/rpython/rlib/test/test_rthread.py b/rpython/rlib/test/test_rthread.py
--- a/rpython/rlib/test/test_rthread.py
+++ b/rpython/rlib/test/test_rthread.py
@@ -52,6 +52,18 @@
     assert get_ident() == thread.get_ident()
 
 
+def test_threadlocalref_on_llinterp():
+    from rpython.rtyper.test.test_llinterp import interpret
+    tlfield = ThreadLocalField(lltype.Signed, "rthread_test_")
+    #
+    def f():
+        x = tlfield.setraw(42)
+        return tlfield.getraw()
+    #
+    res = interpret(f, [])
+    assert res == 42
+
+
 class AbstractThreadTests(AbstractGCTestClass):
     use_threads = True
 
diff --git a/rpython/rtyper/llinterp.py b/rpython/rtyper/llinterp.py
--- a/rpython/rtyper/llinterp.py
+++ b/rpython/rtyper/llinterp.py
@@ -133,6 +133,19 @@
         for line in lines:
             log.traceback(line)
 
+    def get_tlobj(self):
+        try:
+            return self._tlobj
+        except AttributeError:
+            from rpython.rtyper.lltypesystem import rffi
+            PERRNO = rffi.CArrayPtr(rffi.INT)
+            fake_p_errno = lltype.malloc(PERRNO.TO, 1, flavor='raw', zero=True,
+                                         track_allocation=False)
+            self._tlobj = {'RPY_TLOFS_p_errno': fake_p_errno,
+                           #'thread_ident': ...,
+                           }
+            return self._tlobj
+
     def find_roots(self):
         """Return a list of the addresses of the roots."""
         #log.findroots("starting")
@@ -920,13 +933,11 @@
         return 0
 
     def op_threadlocalref_addr(self):
-        raise NotImplementedError("threadlocalref_addr")
+        return _address_of_thread_local()
 
-    def op_threadlocalref_get(self, offset):
-        if (type(offset) is CDefinedIntSymbolic and
-                offset.expr == 'RPY_TLOFS_foobar_test_'):   # used in tests
-            return 0x544c
-        raise NotImplementedError("threadlocalref_get")
+    def op_threadlocalref_get(self, RESTYPE, offset):
+        return self.op_raw_load(RESTYPE, _address_of_thread_local(), offset)
+    op_threadlocalref_get.need_result_type = True
 
     # __________________________________________________________
     # operations on addresses
@@ -973,9 +984,9 @@
             ll_p = rffi.cast(rffi.CArrayPtr(RESTYPE),
                              rffi.ptradd(ll_p, offset))
             value = ll_p[0]
-        ## elif getattr(addr, 'is_fake_thread_local_addr', False):
-        ##     assert type(offset) is CDefinedIntSymbolic
-        ##     value = self.llinterpreter.tlobj[offset.expr]
+        elif getattr(addr, 'is_fake_thread_local_addr', False):
+            assert type(offset) is CDefinedIntSymbolic
+            value = self.llinterpreter.get_tlobj()[offset.expr]
         else:
             assert offset.TYPE == RESTYPE
             value = getattr(addr, str(RESTYPE).lower())[offset.repeat]
@@ -996,9 +1007,9 @@
             ll_p = rffi.cast(rffi.CArrayPtr(ARGTYPE),
                              rffi.ptradd(ll_p, offset))
             ll_p[0] = value
-        ## elif getattr(addr, 'is_fake_thread_local_addr', False):
-        ##     assert type(offset) is CDefinedIntSymbolic
-        ##     self.llinterpreter.tlobj[offset.expr] = value
+        elif getattr(addr, 'is_fake_thread_local_addr', False):
+            assert type(offset) is CDefinedIntSymbolic
+            self.llinterpreter.get_tlobj()[offset.expr] = value
         else:
             assert offset.TYPE == ARGTYPE
             getattr(addr, str(ARGTYPE).lower())[offset.repeat] = value
@@ -1320,6 +1331,10 @@
             return llmemory.fakeaddress(addr.ptr._obj._ptr)
         return addr
 
+class _address_of_thread_local(object):
+    _TYPE = llmemory.Address
+    is_fake_thread_local_addr = True
+
 
 # by default we route all logging messages to nothingness
 # e.g. tests can then switch on logging to get more help

From noreply at buildbot.pypy.org  Fri Nov 28 14:57:44 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Fri, 28 Nov 2014 14:57:44 +0100 (CET)
Subject: [pypy-commit] pypy default: fix test
Message-ID: <20141128135744.30DE21C03DC@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r74754:ec0b383167ab
Date: 2014-11-28 14:56 +0100
http://bitbucket.org/pypy/pypy/changeset/ec0b383167ab/

Log:	fix test

diff --git a/pypy/module/micronumpy/test/test_zjit.py b/pypy/module/micronumpy/test/test_zjit.py
--- a/pypy/module/micronumpy/test/test_zjit.py
+++ b/pypy/module/micronumpy/test/test_zjit.py
@@ -127,12 +127,13 @@
         assert result == 3 ** 2
         self.check_trace_count(1)
         self.check_simple_loop({
-            'call': 3,
+            'call': 1,
             'float_add': 1,
             'float_eq': 3,
             'float_mul': 2,
             'float_ne': 1,
             'getarrayitem_gc': 1,
+            'getarrayitem_raw': 1,     # read the errno
             'guard_false': 4,
             'guard_not_invalidated': 1,
             'guard_true': 3,
@@ -144,6 +145,7 @@
             'raw_load': 2,
             'raw_store': 1,
             'setarrayitem_gc': 1,
+            'setarrayitem_raw': 1,     # write the errno
         })
 
     def define_pow_int():

From noreply at buildbot.pypy.org  Fri Nov 28 15:14:59 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Fri, 28 Nov 2014 15:14:59 +0100 (CET)
Subject: [pypy-commit] pypy default: Remove 'int_signext' on an argument
 that is already known to be
Message-ID: <20141128141459.D5EEC1C073F@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r74755:f6c90881b77f
Date: 2014-11-28 15:14 +0100
http://bitbucket.org/pypy/pypy/changeset/f6c90881b77f/

Log:	Remove 'int_signext' on an argument that is already known to be
	within bounds. This would be a no-op. Shows up in:

	 getarrayitem_raw(non-full-word)

	 int_signext

	 setarrayitem_raw(non-full-word)

diff --git a/rpython/jit/metainterp/optimizeopt/intbounds.py b/rpython/jit/metainterp/optimizeopt/intbounds.py
--- a/rpython/jit/metainterp/optimizeopt/intbounds.py
+++ b/rpython/jit/metainterp/optimizeopt/intbounds.py
@@ -343,11 +343,17 @@
             self.emit_operation(op)
 
     def optimize_INT_SIGNEXT(self, op):
-        self.emit_operation(op)
-        v1 = self.getvalue(op.result)
+        value = self.getvalue(op.getarg(0))
         numbits = op.getarg(1).getint() * 8
-        v1.intbound.make_ge(IntLowerBound(-(1 << (numbits - 1))))
-        v1.intbound.make_lt(IntUpperBound(1 << (numbits - 1)))
+        start = -(1 << (numbits - 1))
+        stop = 1 << (numbits - 1)
+        bounds = IntBound(start, stop - 1)
+        if bounds.contains_bound(value.intbound):
+            self.make_equal_to(op.result, value)
+        else:
+            self.emit_operation(op)
+            vres = self.getvalue(op.result)
+            vres.intbound.intersect(bounds)
 
     def optimize_ARRAYLEN_GC(self, op):
         self.emit_operation(op)
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py
@@ -5494,6 +5494,41 @@
         """
         self.optimize_loop(ops, expected)
 
+    def test_int_signext_already_in_bounds(self):
+        ops = """
+        [i0]
+        i1 = int_signext(i0, 1)
+        i2 = int_signext(i1, 2)
+        jump(i2)
+        """
+        expected = """
+        [i0]
+        i1 = int_signext(i0, 1)
+        jump(i1)
+        """
+        self.optimize_loop(ops, expected)
+        #
+        ops = """
+        [i0]
+        i1 = int_signext(i0, 1)
+        i2 = int_signext(i1, 1)
+        jump(i2)
+        """
+        expected = """
+        [i0]
+        i1 = int_signext(i0, 1)
+        jump(i1)
+        """
+        self.optimize_loop(ops, expected)
+        #
+        ops = """
+        [i0]
+        i1 = int_signext(i0, 2)
+        i2 = int_signext(i1, 1)
+        jump(i2)
+        """
+        self.optimize_loop(ops, ops)
+
 
 class TestLLtype(BaseTestOptimizeBasic, LLtypeMixin):
     pass

From noreply at buildbot.pypy.org  Fri Nov 28 15:17:41 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Fri, 28 Nov 2014 15:17:41 +0100 (CET)
Subject: [pypy-commit] pypy default: fix test
Message-ID: <20141128141741.69DF01C073F@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r74756:7840fc043a0b
Date: 2014-11-28 15:17 +0100
http://bitbucket.org/pypy/pypy/changeset/7840fc043a0b/

Log:	fix test

diff --git a/pypy/module/pypyjit/test_pypy_c/test_ffi.py b/pypy/module/pypyjit/test_pypy_c/test_ffi.py
--- a/pypy/module/pypyjit/test_pypy_c/test_ffi.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_ffi.py
@@ -342,9 +342,9 @@
         guard_false(i114, descr=...)
         --TICK--
         i119 = call(ConstClass(_ll_1_raw_malloc_varsize__Signed), 6, descr=)
-        raw_store(i119, 0, i116, descr=)
-        raw_store(i119, 2, i116, descr=)
-        raw_store(i119, 4, i116, descr=)
+        raw_store(i119, 0, i112, descr=)
+        raw_store(i119, 2, i112, descr=)
+        raw_store(i119, 4, i112, descr=)
         setfield_gc(p167, i119, descr=)
         i123 = arraylen_gc(p67, descr=)
         jump(..., descr=...)

From noreply at buildbot.pypy.org  Fri Nov 28 17:25:19 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Fri, 28 Nov 2014 17:25:19 +0100 (CET)
Subject: [pypy-commit] creflect default: Transform function argument types
	as C does
Message-ID: <20141128162519.2B17B1D230E@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r101:23d40e0eea11
Date: 2014-11-28 17:25 +0100
http://bitbucket.org/cffi/creflect/changeset/23d40e0eea11/

Log:	Transform function argument types as C does

diff --git a/creflect/cparser.py b/creflect/cparser.py
--- a/creflect/cparser.py
+++ b/creflect/cparser.py
@@ -249,8 +249,10 @@
         return model.FunctionType(tuple(args), result, ellipsis)
 
     def _as_func_arg(self, type):
-        if isinstance(type, (model.ArrayType, model.FunctionType)):
-            return model.PointerType(type.item)
+        if isinstance(type, model.ArrayType):
+            return model.PointerType(type.item, const=False)
+        elif isinstance(type, model.FunctionType):
+            return model.PointerType(type, const=False)
         else:
             return type
 
diff --git a/test/codegen/func-005.c b/test/codegen/func-005.c
new file mode 100644
--- /dev/null
+++ b/test/codegen/func-005.c
@@ -0,0 +1,41 @@
+void f(int[], long(char, short));
+
+# ____________________________________________________________
+
+void f(int a[], long g(char, short)) { }
+
+# ____________________________________________________________
+
+static void testfunc_005__c_f(void *args[], void *result) {
+    f(*(int **)args[0], *(long (**)(char, short))args[1]);
+}
+
+static void testfunc_005__d_f(int *a0, long (*a1)(char, short)) {
+    f(a0, a1);
+}
+
+static void testfunc_005__f4(void *func, void *args[], void *result) {
+    long (*f)(char, short) = func;
+    *(long *)result = f(*(char *)args[0], *(short *)args[1]);
+}
+
+void testfunc_005(crx_builder_t *cb)
+{
+    crx_type_t *t1, *t2, *t3, *t4, *t5, *t6, *a7[2], *t8, *t9, *a10[2];
+    {
+        t1 = cb->get_void_type(cb);
+        t2 = cb->get_signed_type(cb, sizeof(int), "int");
+        t3 = cb->get_pointer_type(cb, t2);
+        t4 = cb->get_signed_type(cb, sizeof(long), "long");
+        t5 = cb->get_char_type(cb);
+        t6 = cb->get_signed_type(cb, sizeof(short), "short");
+        a7[0] = t5;
+        a7[1] = t6;
+        t8 = cb->get_function_type(cb, t4, a7, 2, &testfunc_005__f4);
+        t9 = cb->get_pointer_type(cb, t8);
+        a10[0] = t3;
+        a10[1] = t9;
+        cb->define_func(cb, "f", t1, a10, 2, &testfunc_005__c_f, &testfunc_005__d_f);
+#expect FUNC f: PTR int -> PTR FUNC( char -> short -> long ) -> void
+    }
+}

From noreply at buildbot.pypy.org  Fri Nov 28 17:48:34 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Fri, 28 Nov 2014 17:48:34 +0100 (CET)
Subject: [pypy-commit] creflect default: A tool in C to dump the reflection
 information stored in a library
Message-ID: <20141128164834.45D201C0589@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r102:538bc1da9a17
Date: 2014-11-28 17:48 +0100
http://bitbucket.org/cffi/creflect/changeset/538bc1da9a17/

Log:	A tool in C to dump the reflection information stored in a library
	compiled from the output of "creflect".

diff --git a/creflect/src/creflect_check.c b/creflect/src/creflect_check.c
new file mode 100644
--- /dev/null
+++ b/creflect/src/creflect_check.c
@@ -0,0 +1,54 @@
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+
+#include "creflect.h"
+#include "creflect_print.h"
+
+
+void creflect_dump_information(const char *filename)
+{
+    char filename_ex[PATH_MAX];
+
+    if (strchr(filename, '/') == NULL)
+        strcpy(filename_ex, "./");
+    else
+        strcpy(filename_ex, "");
+    strncat(filename_ex, filename, sizeof(filename_ex) - 3);
+
+    void *lib = dlopen(filename_ex, RTLD_LAZY);
+    if (lib == NULL)
+        goto err;
+
+    void (*crxmain)(crx_builder_t *);
+    crxmain = (void(*)(crx_builder_t *))dlsym(lib, "_creflect_main");
+    if (crxmain == NULL)
+        goto err;
+
+    crxmain(&maincb);
+
+    if (dlclose(lib) != 0) {
+        lib = NULL;
+        goto err;
+    }
+    return;
+
+ err:
+    fprintf(stderr, "error: %s\n", dlerror());
+    if (lib)
+        dlclose(lib);
+}
+
+int main(int argc, char *argv[])
+{
+    if (argc != 2) {
+        fprintf(stderr, "One argument needed: the name of the library file"
+                        " out of which the creflect information is dumped.\n");
+        return 2;
+    }
+    creflect_dump_information(argv[1]);
+    return 0;
+}

From noreply at buildbot.pypy.org  Sat Nov 29 01:35:10 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Sat, 29 Nov 2014 01:35:10 +0100 (CET)
Subject: [pypy-commit] creflect default: Parsing simple C declarations,
	starting
Message-ID: <20141129003510.7D08D1D38B9@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r103:1ad5d2bb1fa5
Date: 2014-11-28 23:11 +0100
http://bitbucket.org/cffi/creflect/changeset/1ad5d2bb1fa5/

Log:	Parsing simple C declarations, starting

diff --git a/creflect/src/c_decl_parser.c b/creflect/src/c_decl_parser.c
new file mode 100644
--- /dev/null
+++ b/creflect/src/c_decl_parser.c
@@ -0,0 +1,197 @@
+#include 
+#include "creflect.h"
+
+
+enum crxp_token_e {
+    TOK_STAR='*',
+    TOK_OPEN_PAREN='(',
+    TOK_CLOSE_PAREN=')',
+    TOK_OPEN_BRACKET='[',
+    TOK_CLOSE_BRACKET=']',
+    TOK_COMMA=',',
+
+    TOK_START=256,
+    TOK_END,
+    TOK_ERROR,
+    TOK_IDENTIFIER,
+    TOK_INTEGER,
+
+    /* keywords */
+    TOK__BOOL,
+    TOK_CHAR,
+    //TOK__COMPLEX,
+    TOK_CONST,
+    TOK_DOUBLE,
+    TOK_FLOAT,
+    //TOK__IMAGINARY,
+    TOK_INT,
+    TOK_LONG,
+    TOK_SHORT,
+    TOK_SIGNED,
+    TOK_STRUCT,
+    TOK_UNION,
+    TOK_UNSIGNED,
+    TOK_VOID,
+};
+
+typedef struct {
+    enum crxp_token_e kind;
+    const char *p;
+    size_t size;
+    crx_builder_t *cb;
+} crxp_token_t;
+
+static int is_space(char x)
+{
+    return (x == ' ' || x == '\f' || x == '\n' || x == '\r' ||
+            x == '\t' || x == '\v');
+}
+
+static int is_ident_first(char x)
+{
+    return ('A' <= x && x <= 'Z' || 'a' <= x && x <= 'z' || x == '_');
+}
+
+static int is_ident_next(char x)
+{
+    return (is_ident_first(x) || '0' <= x && x <= '9');
+}
+
+static void next_token(crxp_token_t *tok)
+{
+    const char *p = tok->p + tok->size;
+    if (tok->kind == TOK_ERROR)
+        return;
+    while (!is_ident_first(*p)) {
+        if (is_space(*p)) {
+            p++;
+        }
+        else if (*p) {
+            tok->kind = *p;
+            tok->p = p;
+            tok->size = 1;
+            return;
+        }
+        else {
+            tok->kind = TOK_END;
+            tok->p = p;
+            tok->size = 0;
+            return;
+        }
+    }
+    tok->p = p;
+    tok->size = 1;
+    while (is_ident_next(p[tok->size]))
+        tok->size++;
+    tok->kind = TOK_IDENTIFIER;
+
+    switch (*p) {
+    case '_':
+        if (tok->size == 5 && !memcmp(p, "_Bool", 5))  tok->kind = TOK__BOOL;
+        break;
+    case 'c':
+        if (tok->size == 4 && !memcmp(p, "char", 4))   tok->kind = TOK_CHAR;
+        if (tok->size == 5 && !memcmp(p, "const", 5))  tok->kind = TOK_CONST;
+        break;
+    case 'd':
+        if (tok->size == 6 && !memcmp(p, "double", 6)) tok->kind = TOK_DOUBLE;
+        break;
+    case 'f':
+        if (tok->size == 5 && !memcmp(p, "float", 5))  tok->kind = TOK_FLOAT;
+        break;
+    case 'i':
+        if (tok->size == 3 && !memcmp(p, "int", 3))    tok->kind = TOK_INT;
+        break;
+    case 'l':
+        if (tok->size == 4 && !memcmp(p, "long", 4))   tok->kind = TOK_LONG;
+        break;
+    case 's':
+        if (tok->size == 5 && !memcmp(p, "short", 5))  tok->kind = TOK_SHORT;
+        if (tok->size == 6 && !memcmp(p, "signed", 6)) tok->kind = TOK_SIGNED;
+        if (tok->size == 6 && !memcmp(p, "struct", 6)) tok->kind = TOK_STRUCT;
+        break;
+    case 'u':
+        if (tok->size == 5 && !memcmp(p, "union", 5))  tok->kind = TOK_UNION;
+        if (tok->size == 8 && !memcmp(p,"unsigned",8)) tok->kind = TOK_UNSIGNED;
+        break;
+    case 'v':
+        if (tok->size == 4 && !memcmp(p, "void", 4))   tok->kind = TOK_VOID;
+        break;
+    }
+}
+
+static crx_type_t *parse_sequel_right(crxp_token_t *tok, crx_type_t *t1)
+{
+    switch (tok->kind) {
+
+    case TOK_OPEN_PAREN:
+        abort();
+
+    case TOK_OPEN_BRACKET:
+        abort();
+
+    default:
+        return t1;
+    }
+}
+
+static crx_type_t *parse_sequel(crxp_token_t *tok, crx_type_t *t1)
+{
+    while (1) {
+        switch (tok->kind) {
+
+        case TOK_STAR:
+            t1 = tok->cb->get_pointer_type(tok->cb, t1);
+            break;
+
+        case TOK_CONST:
+            t1 = tok->cb->get_const_type(tok->cb, t1);
+            break;
+
+        default:
+            return parse_sequel_right(tok, t1);
+        }
+
+        next_token(tok);
+    }
+}
+
+static crx_type_t *parse_complete(crxp_token_t *tok)
+{
+    crx_type_t *t1;
+    int is_const = (tok->kind == TOK_CONST);
+    if (is_const) {
+        next_token(tok);
+    }
+    switch (tok->kind) {
+    case TOK_INT:
+        t1 = tok->cb->get_signed_type(tok->cb, sizeof(int), "int");
+        break;
+    default:
+        tok->kind = TOK_ERROR;
+        return NULL;
+    }
+    next_token(tok);
+
+    if (is_const) {
+        t1 = tok->cb->get_const_type(tok->cb, t1);
+    }
+    return parse_sequel(tok, t1);
+}
+
+crx_type_t *creflect_decl_parser(crx_builder_t *cb, const char **input)
+{
+    crxp_token_t token;
+    crx_type_t *t1;
+    token.kind = TOK_START;
+    token.cb = cb;
+    token.p = *input;
+    token.size = 0;
+    next_token(&token);
+    t1 = parse_complete(&token);
+
+    if (token.kind == TOK_END)
+        return t1;
+    *input = token.p;
+    return NULL;
+}
diff --git a/test/test_c_decl_parser.py b/test/test_c_decl_parser.py
new file mode 100644
--- /dev/null
+++ b/test/test_c_decl_parser.py
@@ -0,0 +1,72 @@
+import os, subprocess
+from .udir import udir
+
+
+TESTER = r"""
+#include "c_decl_parser.c"
+#include "creflect_print.h"
+
+int main(int argc, char *argv[])
+{
+    const char *p = argv[1];
+    crx_type_t *t1 = creflect_decl_parser(&maincb, &p);
+    if (t1 != NULL)
+        printf("%s\n", t1->text);
+    else {
+        printf("error: %s\n       ", argv[1]);
+        while (p > argv[1]) {
+            printf(" ");
+            p--;
+        }
+        printf("^\n");
+    }
+    return 0;
+}
+"""
+
+
+def setup_module(mod):
+    executable = str(udir.join('c_decl_parser_test'))
+    f = open(executable + '.c', 'w')
+    f.write(TESTER)
+    f.close()
+    err = os.system("gcc -g -Werror '%s.c' -o '%s' -I../creflect/src" % (
+        executable, executable))
+    assert not err
+    mod.executable = executable
+
+
+def parse(input, expected_output):
+    global executable
+    got = subprocess.check_output([executable, input])
+    assert got == expected_output + '\n'
+
+def parse_error(input, expected_location):
+    parse(input, 'error: %s\n       %s^' % (input, " " * expected_location))
+
+def test_c_decl_parser():
+    parse("int **", "PTR PTR int")
+    parse("const int **", "PTR PTR CONST int")
+    parse("int const **", "PTR PTR CONST int")
+    parse("int *const *", "PTR CONST PTR int")
+    parse("int ** const", "CONST PTR PTR int")
+    import py; py.test.skip("in-progress")
+    parse("int[2]")
+    parse("int*[2][3]")
+    parse("int(*)[2][3]")
+    parse("int(*[2])[3]")
+    parse("int()")
+    parse("int(void)")
+    parse("int(int)")
+    parse("int(int *, int const *)")
+    parse("int(*)(int)")
+    parse("unsigned int")
+    parse("unsigned long long *")
+    parse("const unsigned long long *")
+    parse("unsigned long long const *")
+    parse("char(*(*)(long))(int)")
+    parse("foo_t[]")
+
+def test_c_decl_error():
+    parse_error("*", 0)
+    parse_error("int ]**", 4)

From noreply at buildbot.pypy.org  Sat Nov 29 01:35:11 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Sat, 29 Nov 2014 01:35:11 +0100 (CET)
Subject: [pypy-commit] creflect default: simple arrays
Message-ID: <20141129003511.923FB1D38B9@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r104:388bb851de61
Date: 2014-11-28 23:18 +0100
http://bitbucket.org/cffi/creflect/changeset/388bb851de61/

Log:	simple arrays

diff --git a/creflect/src/c_decl_parser.c b/creflect/src/c_decl_parser.c
--- a/creflect/src/c_decl_parser.c
+++ b/creflect/src/c_decl_parser.c
@@ -1,4 +1,5 @@
 #include 
+#include 
 #include "creflect.h"
 
 
@@ -52,9 +53,14 @@
     return ('A' <= x && x <= 'Z' || 'a' <= x && x <= 'z' || x == '_');
 }
 
+static int is_digit(char x)
+{
+    return ('0' <= x && x <= '9');
+}
+
 static int is_ident_next(char x)
 {
-    return (is_ident_first(x) || '0' <= x && x <= '9');
+    return (is_ident_first(x) || is_digit(x));
 }
 
 static void next_token(crxp_token_t *tok)
@@ -66,6 +72,14 @@
         if (is_space(*p)) {
             p++;
         }
+        else if (is_digit(*p)) {
+            tok->kind = TOK_INTEGER;
+            tok->p = p;
+            tok->size = 1;
+            while (is_digit(p[tok->size]))
+                tok->size++;
+            return;
+        }
         else if (*p) {
             tok->kind = *p;
             tok->p = p;
@@ -79,11 +93,11 @@
             return;
         }
     }
+    tok->kind = TOK_IDENTIFIER;
     tok->p = p;
     tok->size = 1;
     while (is_ident_next(p[tok->size]))
         tok->size++;
-    tok->kind = TOK_IDENTIFIER;
 
     switch (*p) {
     case '_':
@@ -128,11 +142,20 @@
         abort();
 
     case TOK_OPEN_BRACKET:
-        abort();
+        next_token(tok);
+        assert(tok->kind == TOK_INTEGER);  // XXX
+        unsigned long long length = strtoull(tok->p, NULL, 10);
+        next_token(tok);
+        assert(tok->kind == TOK_CLOSE_BRACKET);  // XXX
+        next_token(tok);
+        t1 = parse_sequel_right(tok, t1);
+        t1 = tok->cb->get_array_type(tok->cb, t1, (size_t)length);
+        break;
 
     default:
-        return t1;
+        break;
     }
+    return t1;
 }
 
 static crx_type_t *parse_sequel(crxp_token_t *tok, crx_type_t *t1)
diff --git a/test/test_c_decl_parser.py b/test/test_c_decl_parser.py
--- a/test/test_c_decl_parser.py
+++ b/test/test_c_decl_parser.py
@@ -50,9 +50,9 @@
     parse("int const **", "PTR PTR CONST int")
     parse("int *const *", "PTR CONST PTR int")
     parse("int ** const", "CONST PTR PTR int")
+    parse("int[2]", "ARRAY[2] int")
+    parse("int*[2][3]", "ARRAY[2] ARRAY[3] PTR int")
     import py; py.test.skip("in-progress")
-    parse("int[2]")
-    parse("int*[2][3]")
     parse("int(*)[2][3]")
     parse("int(*[2])[3]")
     parse("int()")

From noreply at buildbot.pypy.org  Sat Nov 29 01:35:12 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Sat, 29 Nov 2014 01:35:12 +0100 (CET)
Subject: [pypy-commit] creflect default: The next part is a bit annoying and
 requires delaying the calls to
Message-ID: <20141129003512.9D8721D38B9@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r105:aeb1f133cab1
Date: 2014-11-29 01:35 +0100
http://bitbucket.org/cffi/creflect/changeset/aeb1f133cab1/

Log:	The next part is a bit annoying and requires delaying the calls to
	cb->get_xxx_type()

diff --git a/creflect/src/c_decl_parser.c b/creflect/src/c_decl_parser.c
--- a/creflect/src/c_decl_parser.c
+++ b/creflect/src/c_decl_parser.c
@@ -1,4 +1,5 @@
 #include 
+#include 
 #include 
 #include "creflect.h"
 
@@ -35,11 +36,15 @@
     TOK_VOID,
 };
 
+#define NUM_DELAY_SLOTS   5000
+
 typedef struct {
     enum crxp_token_e kind;
     const char *p;
     size_t size;
     crx_builder_t *cb;
+    intptr_t *delay_slots;
+    intptr_t all_delay_slots[NUM_DELAY_SLOTS];
 } crxp_token_t;
 
 static int is_space(char x)
@@ -134,48 +139,95 @@
     }
 }
 
-static crx_type_t *parse_sequel_right(crxp_token_t *tok, crx_type_t *t1)
+static intptr_t *alloc_ds(crxp_token_t *tok, size_t num)
 {
+    intptr_t *result = tok->delay_slots;
+    if (num > (tok->all_delay_slots + NUM_DELAY_SLOTS - tok->delay_slots))
+        abort();  // XXX out of memory
+    tok->delay_slots += num;
+    return result;
+}
+
+static void parse_sequel(crxp_token_t *tok, intptr_t ds_end)
+{
+    while (tok->kind == TOK_STAR || tok->kind == TOK_CONST) {
+        alloc_ds(tok, 1)[0] = tok->kind;
+        next_token(tok);
+    }
+
+    intptr_t *ds, *jump_slot = alloc_ds(tok, 1);
+    *jump_slot = ds_end;
+
+ next_right_part:
     switch (tok->kind) {
 
     case TOK_OPEN_PAREN:
-        abort();
+        /* just parentheses for grouping */
+        next_token(tok);
+
+        ds = tok->delay_slots;
+        parse_sequel(tok, *jump_slot);
+        *jump_slot = -(ds - tok->all_delay_slots);
+
+        assert(tok->kind == TOK_CLOSE_PAREN);  // XXX
+        next_token(tok);
+        goto next_right_part;
 
     case TOK_OPEN_BRACKET:
         next_token(tok);
         assert(tok->kind == TOK_INTEGER);  // XXX
-        unsigned long long length = strtoull(tok->p, NULL, 10);
+
+        uintptr_t length;
+        if (sizeof(uintptr_t) > sizeof(unsigned long))
+            length = strtoull(tok->p, NULL, 10);
+        else
+            length = strtoul(tok->p, NULL, 10);
         next_token(tok);
+
         assert(tok->kind == TOK_CLOSE_BRACKET);  // XXX
         next_token(tok);
-        t1 = parse_sequel_right(tok, t1);
-        t1 = tok->cb->get_array_type(tok->cb, t1, (size_t)length);
-        break;
+
+        ds = alloc_ds(tok, 3);
+        ds[0] = TOK_OPEN_BRACKET;
+        ds[1] = (intptr_t)length;
+        ds[2] = *jump_slot;
+        *jump_slot = -(ds - tok->all_delay_slots);
+        goto next_right_part;
 
     default:
         break;
     }
-    return t1;
 }
 
-static crx_type_t *parse_sequel(crxp_token_t *tok, crx_type_t *t1)
+static crx_type_t *fetch_delay_slots(crxp_token_t *tok, crx_type_t *t1,
+                                     intptr_t *delay_slot)
 {
+    tok->delay_slots = delay_slot;
     while (1) {
-        switch (tok->kind) {
-
-        case TOK_STAR:
-            t1 = tok->cb->get_pointer_type(tok->cb, t1);
-            break;
-
-        case TOK_CONST:
-            t1 = tok->cb->get_const_type(tok->cb, t1);
-            break;
-
-        default:
-            return parse_sequel_right(tok, t1);
+        intptr_t tok_kind = *delay_slot++;
+        if (tok_kind > 0) {
+            switch (tok_kind) {
+            case TOK_END:
+                return t1;
+            case TOK_STAR:
+                t1 = tok->cb->get_pointer_type(tok->cb, t1);
+                break;
+            case TOK_CONST:
+                t1 = tok->cb->get_const_type(tok->cb, t1);
+                break;
+            case TOK_OPEN_BRACKET:   /* array */
+            {
+                uintptr_t length = (uintptr_t)*delay_slot++;
+                t1 = tok->cb->get_array_type(tok->cb, t1, length);
+                break;
+            }
+            default:
+                abort();
+            }
         }
-
-        next_token(tok);
+        else {
+            delay_slot = tok->all_delay_slots + (-tok_kind);
+        }
     }
 }
 
@@ -192,29 +244,32 @@
         break;
     default:
         tok->kind = TOK_ERROR;
-        return NULL;
+        return;
     }
     next_token(tok);
 
     if (is_const) {
         t1 = tok->cb->get_const_type(tok->cb, t1);
     }
-    return parse_sequel(tok, t1);
+
+    intptr_t *orig = tok->delay_slots;
+    parse_sequel(tok, TOK_END);
+    return fetch_delay_slots(tok, t1, orig);
 }
 
-crx_type_t *creflect_decl_parser(crx_builder_t *cb, const char **input)
+const char *creflect_decl_parser(crx_builder_t *cb, const char *input,
+                                 crx_type_t **result)
 {
     crxp_token_t token;
-    crx_type_t *t1;
     token.kind = TOK_START;
     token.cb = cb;
-    token.p = *input;
+    token.p = input;
     token.size = 0;
+    token.delay_slots = token.all_delay_slots;
     next_token(&token);
-    t1 = parse_complete(&token);
+    *result = parse_complete(&token);
 
     if (token.kind == TOK_END)
-        return t1;
-    *input = token.p;
-    return NULL;
+        return NULL;
+    return token.p;
 }
diff --git a/test/test_c_decl_parser.py b/test/test_c_decl_parser.py
--- a/test/test_c_decl_parser.py
+++ b/test/test_c_decl_parser.py
@@ -8,10 +8,11 @@
 
 int main(int argc, char *argv[])
 {
-    const char *p = argv[1];
-    crx_type_t *t1 = creflect_decl_parser(&maincb, &p);
-    if (t1 != NULL)
+    crx_type_t *t1;
+    const char *p = creflect_decl_parser(&maincb, argv[1], &t1);
+    if (p == NULL) {
         printf("%s\n", t1->text);
+    }
     else {
         printf("error: %s\n       ", argv[1]);
         while (p > argv[1]) {
@@ -39,6 +40,8 @@
 def parse(input, expected_output):
     global executable
     got = subprocess.check_output([executable, input])
+    print repr(input)
+    print got.rstrip()
     assert got == expected_output + '\n'
 
 def parse_error(input, expected_location):
@@ -52,9 +55,9 @@
     parse("int ** const", "CONST PTR PTR int")
     parse("int[2]", "ARRAY[2] int")
     parse("int*[2][3]", "ARRAY[2] ARRAY[3] PTR int")
+    parse("int(*)[2][3]", "PTR ARRAY[2] ARRAY[3] int")
+    parse("int(*[2])[3]", "ARRAY[2] PTR ARRAY[3] int")
     import py; py.test.skip("in-progress")
-    parse("int(*)[2][3]")
-    parse("int(*[2])[3]")
     parse("int()")
     parse("int(void)")
     parse("int(int)")

From noreply at buildbot.pypy.org  Sat Nov 29 02:51:00 2014
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Sat, 29 Nov 2014 02:51:00 +0100 (CET)
Subject: [pypy-commit] pypy default: Make python2.6 compatible,
 which I think we still care about?
Message-ID: <20141129015100.08A021C073F@cobra.cs.uni-duesseldorf.de>

Author: Alex Gaynor 
Branch: 
Changeset: r74757:bd8d6cf8f8be
Date: 2014-11-28 19:50 -0600
http://bitbucket.org/pypy/pypy/changeset/bd8d6cf8f8be/

Log:	Make python2.6 compatible, which I think we still care about?

diff --git a/rpython/translator/simplify.py b/rpython/translator/simplify.py
--- a/rpython/translator/simplify.py
+++ b/rpython/translator/simplify.py
@@ -617,7 +617,7 @@
             if simplify_phis(block):
                 progress = True
 
-    renaming = {key: uf[key].rep for key in uf}
+    renaming = dict((key, uf[key].rep) for key in uf)
     for block, links in entrymap.items():
         if inputs[block]:
             new_inputs, new_args = zip(*inputs[block])

From noreply at buildbot.pypy.org  Sat Nov 29 09:21:32 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Sat, 29 Nov 2014 09:21:32 +0100 (CET)
Subject: [pypy-commit] creflect default: simple function types
Message-ID: <20141129082132.2AC991C03A4@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r106:808cb0b998ee
Date: 2014-11-29 09:21 +0100
http://bitbucket.org/cffi/creflect/changeset/808cb0b998ee/

Log:	simple function types

diff --git a/creflect/src/c_decl_parser.c b/creflect/src/c_decl_parser.c
--- a/creflect/src/c_decl_parser.c
+++ b/creflect/src/c_decl_parser.c
@@ -68,6 +68,16 @@
     return (is_ident_first(x) || is_digit(x));
 }
 
+static int is_following_token_this_char(crxp_token_t *tok, char expected)
+{
+    const char *p = tok->p + tok->size;
+    if (tok->kind == TOK_ERROR)
+        return 0;
+    while (is_space(*p))
+        p++;
+    return *p == expected;
+}
+
 static void next_token(crxp_token_t *tok)
 {
     const char *p = tok->p + tok->size;
@@ -148,6 +158,8 @@
     return result;
 }
 
+static crx_type_t *parse_complete(crxp_token_t *tok);
+
 static void parse_sequel(crxp_token_t *tok, intptr_t ds_end)
 {
     while (tok->kind == TOK_STAR || tok->kind == TOK_CONST) {
@@ -162,12 +174,42 @@
     switch (tok->kind) {
 
     case TOK_OPEN_PAREN:
-        /* just parentheses for grouping */
         next_token(tok);
 
-        ds = tok->delay_slots;
-        parse_sequel(tok, *jump_slot);
-        *jump_slot = -(ds - tok->all_delay_slots);
+        if (tok->kind == TOK_STAR ||
+            tok->kind == TOK_CONST ||
+            tok->kind == TOK_OPEN_BRACKET) {
+            /* just parentheses for grouping */
+            ds = tok->delay_slots;
+            parse_sequel(tok, *jump_slot);
+            *jump_slot = -(ds - tok->all_delay_slots);
+        }
+        else {
+            /* function type */
+            ds = alloc_ds(tok, 2);
+            ds[0] = TOK_OPEN_PAREN;
+            ds[1] = 0;
+            if (tok->kind == TOK_VOID &&
+                    is_following_token_this_char(tok, ')')) {
+                next_token(tok);
+            }
+            if (tok->kind != TOK_CLOSE_PAREN) {
+                while (1) {
+                    crx_type_t *t1 = parse_complete(tok);
+                    intptr_t *ds_type = alloc_ds(tok, 1);
+                    assert(ds_type == ds + 2 + ds[1]);
+                    *ds_type = (intptr_t)t1;
+                    ds[1]++;
+                    if (tok->kind != TOK_COMMA)
+                        break;
+                    next_token(tok);
+                }
+            }
+            intptr_t *ds_next = alloc_ds(tok, 1);
+            assert(ds_next == ds + 2 + ds[1]);
+            *ds_next = *jump_slot;
+            *jump_slot = -(ds - tok->all_delay_slots);
+        }
 
         assert(tok->kind == TOK_CLOSE_PAREN);  // XXX
         next_token(tok);
@@ -221,6 +263,15 @@
                 t1 = tok->cb->get_array_type(tok->cb, t1, length);
                 break;
             }
+            case TOK_OPEN_PAREN:   /* function */
+            {
+                intptr_t nbargs = *delay_slot++;
+                crx_type_t **argtypes = (crx_type_t **)delay_slot;
+                delay_slot += nbargs;
+                t1 = tok->cb->get_function_type(tok->cb, t1, argtypes,
+                                                nbargs, NULL);
+                break;
+            }
             default:
                 abort();
             }
@@ -242,6 +293,9 @@
     case TOK_INT:
         t1 = tok->cb->get_signed_type(tok->cb, sizeof(int), "int");
         break;
+    case TOK_VOID:
+        t1 = tok->cb->get_void_type(tok->cb);
+        break;
     default:
         tok->kind = TOK_ERROR;
         return;
diff --git a/test/test_c_decl_parser.py b/test/test_c_decl_parser.py
--- a/test/test_c_decl_parser.py
+++ b/test/test_c_decl_parser.py
@@ -37,10 +37,10 @@
     mod.executable = executable
 
 
-def parse(input, expected_output):
+def parse(input, expected_output='???'):
     global executable
+    print repr(input)
     got = subprocess.check_output([executable, input])
-    print repr(input)
     print got.rstrip()
     assert got == expected_output + '\n'
 
@@ -57,18 +57,21 @@
     parse("int*[2][3]", "ARRAY[2] ARRAY[3] PTR int")
     parse("int(*)[2][3]", "PTR ARRAY[2] ARRAY[3] int")
     parse("int(*[2])[3]", "ARRAY[2] PTR ARRAY[3] int")
+    parse("int()", "FUNC( int )")
+    parse("int(void)", "FUNC( int )")
+    parse("int(int)", "FUNC( int -> int )")
+    parse("int(int *, int const *)", "FUNC( PTR int -> PTR CONST int -> int )")
+    parse("int(*)(int)", "PTR FUNC( int -> int )")
     import py; py.test.skip("in-progress")
-    parse("int()")
-    parse("int(void)")
-    parse("int(int)")
-    parse("int(int *, int const *)")
-    parse("int(*)(int)")
     parse("unsigned int")
     parse("unsigned long long *")
     parse("const unsigned long long *")
     parse("unsigned long long const *")
     parse("char(*(*)(long))(int)")
     parse("foo_t[]")
+    parse("int(int, ...)")
+    parse("int(int[])")
+    parse("int(long(char))")
 
 def test_c_decl_error():
     parse_error("*", 0)

From noreply at buildbot.pypy.org  Sat Nov 29 09:54:57 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Sat, 29 Nov 2014 09:54:57 +0100 (CET)
Subject: [pypy-commit] creflect default: more primitive types
Message-ID: <20141129085457.9CFA81D263B@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r107:0fab3ed49da1
Date: 2014-11-29 09:55 +0100
http://bitbucket.org/cffi/creflect/changeset/0fab3ed49da1/

Log:	more primitive types

diff --git a/creflect/src/c_decl_parser.c b/creflect/src/c_decl_parser.c
--- a/creflect/src/c_decl_parser.c
+++ b/creflect/src/c_decl_parser.c
@@ -282,6 +282,12 @@
     }
 }
 
+static crx_type_t *parse_error(crxp_token_t *tok)
+{
+    tok->kind = TOK_ERROR;
+    return NULL;
+}
+
 static crx_type_t *parse_complete(crxp_token_t *tok)
 {
     crx_type_t *t1;
@@ -289,18 +295,91 @@
     if (is_const) {
         next_token(tok);
     }
+
+    int modifiers_length = 0;
+    int modifiers_sign = 0;
+ modifiers:
     switch (tok->kind) {
-    case TOK_INT:
-        t1 = tok->cb->get_signed_type(tok->cb, sizeof(int), "int");
+
+    case TOK_SHORT:
+        if (modifiers_length != 0)
+            return parse_error(tok);
+        modifiers_length--;
+        next_token(tok);
+        goto modifiers;
+
+    case TOK_LONG:
+        if (modifiers_length != 0 && modifiers_length != 1)
+            return parse_error(tok);
+        modifiers_length++;
+        next_token(tok);
+        goto modifiers;
+
+    case TOK_SIGNED:
+        if (modifiers_sign)
+            return parse_error(tok);
+        modifiers_sign++;
+        next_token(tok);
+        goto modifiers;
+
+    case TOK_UNSIGNED:
+        if (modifiers_sign)
+            return parse_error(tok);
+        modifiers_sign--;
+        next_token(tok);
+        goto modifiers;
+
+    default:
         break;
-    case TOK_VOID:
-        t1 = tok->cb->get_void_type(tok->cb);
-        break;
-    default:
-        tok->kind = TOK_ERROR;
-        return;
     }
-    next_token(tok);
+
+    if (modifiers_length || modifiers_sign) {
+        size_t size;
+        char *name;
+
+        switch (tok->kind) {
+
+        case TOK_CHAR:
+            if (modifiers_length != 0)
+                return parse_error(tok);
+            size = sizeof(char);
+            name = "char";
+            next_token(tok);
+            break;
+
+        case TOK_INT:
+            next_token(tok);
+            /* fall-through */
+        default:
+            switch (modifiers_length) {
+            case -1: size = sizeof(short);     name = "short"; break;
+            case 1:  size = sizeof(long);      name = "long"; break;
+            case 2:  size = sizeof(long long); name = "long long"; break;
+            default: size = sizeof(int);       name = "int"; break;
+            }
+        }
+        if (modifiers_sign < 0)
+            t1 = tok->cb->get_unsigned_type(tok->cb, size, name);
+        else
+            t1 = tok->cb->get_signed_type(tok->cb, size, name);
+    }
+    else {
+        switch (tok->kind) {
+        case TOK_INT:
+            t1 = tok->cb->get_signed_type(tok->cb, sizeof(int), "int");
+            break;
+        case TOK_CHAR:
+            t1 = tok->cb->get_char_type(tok->cb);
+            break;
+        case TOK_VOID:
+            t1 = tok->cb->get_void_type(tok->cb);
+            break;
+        default:
+            tok->kind = TOK_ERROR;
+            return NULL;
+        }
+        next_token(tok);
+    }
 
     if (is_const) {
         t1 = tok->cb->get_const_type(tok->cb, t1);
diff --git a/test/codegen/003i.c b/test/codegen/003i.c
new file mode 100644
--- /dev/null
+++ b/test/codegen/003i.c
@@ -0,0 +1,18 @@
+typedef signed char num_t;
+
+# ____________________________________________________________
+
+void test003i(crx_builder_t *cb)
+{
+    crx_type_t *t1;
+    {
+        num_t *p1;
+        char b[sizeof(*p1)];
+        p1 = (void *)b;
+        (void)(*p1 << 1);  /* check that 'num_t' is an integer type */
+        *p1 = -1;  /* check that 'num_t' is not declared 'const' */
+        t1 = CRX_INT_TYPE(cb, *p1, "char");
+        cb->define_type(cb, "num_t", t1);
+#expect TYPEDEF num_t = signed char
+    }
+}
diff --git a/test/codegen/glob-003j.c b/test/codegen/glob-003j.c
new file mode 100644
--- /dev/null
+++ b/test/codegen/glob-003j.c
@@ -0,0 +1,15 @@
+signed char const ab = 42;
+
+# ____________________________________________________________
+
+void testglob_003j(crx_builder_t *cb)
+{
+    crx_type_t *t1;
+    {
+        crx_num_const_t v;
+        (void)((ab) << 1);  /* check that 'ab' is an integer */
+        t1 = CRX_INT_CONST(cb, ab, &v, 1);
+        cb->define_num_const(cb, "ab", t1, &v);
+#expect NUMCONST ab = int 42
+    }
+}
diff --git a/test/test_c_decl_parser.py b/test/test_c_decl_parser.py
--- a/test/test_c_decl_parser.py
+++ b/test/test_c_decl_parser.py
@@ -62,12 +62,19 @@
     parse("int(int)", "FUNC( int -> int )")
     parse("int(int *, int const *)", "FUNC( PTR int -> PTR CONST int -> int )")
     parse("int(*)(int)", "PTR FUNC( int -> int )")
+    parse("unsigned int", "unsigned int")
+    parse("long", "long")
+    parse("long int", "long")
+    parse("short int", "short")
+    parse("long long int", "long long")
+    parse("unsigned long long *", "PTR unsigned long long")
+    parse("const unsigned long long *", "PTR CONST unsigned long long")
+    parse("unsigned long long const *", "PTR CONST unsigned long long")
+    parse("char", "char")
+    parse("signed char", "signed char")
+    parse("unsigned char", "unsigned char")
+    parse("char(*(*)(long))(int)","PTR FUNC( long -> PTR FUNC( int -> char ) )")
     import py; py.test.skip("in-progress")
-    parse("unsigned int")
-    parse("unsigned long long *")
-    parse("const unsigned long long *")
-    parse("unsigned long long const *")
-    parse("char(*(*)(long))(int)")
     parse("foo_t[]")
     parse("int(int, ...)")
     parse("int(int[])")

From noreply at buildbot.pypy.org  Sat Nov 29 10:09:12 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Sat, 29 Nov 2014 10:09:12 +0100 (CET)
Subject: [pypy-commit] creflect default: incomplete arrays
Message-ID: <20141129090912.B4E841C03A4@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r108:a18484dc4bd9
Date: 2014-11-29 09:58 +0100
http://bitbucket.org/cffi/creflect/changeset/a18484dc4bd9/

Log:	incomplete arrays

diff --git a/creflect/src/c_decl_parser.c b/creflect/src/c_decl_parser.c
--- a/creflect/src/c_decl_parser.c
+++ b/creflect/src/c_decl_parser.c
@@ -216,16 +216,18 @@
         goto next_right_part;
 
     case TOK_OPEN_BRACKET:
+    {
+        uintptr_t length = (uintptr_t)-1;
         next_token(tok);
-        assert(tok->kind == TOK_INTEGER);  // XXX
+        if (tok->kind != TOK_CLOSE_BRACKET) {
+            assert(tok->kind == TOK_INTEGER);  // XXX
 
-        uintptr_t length;
-        if (sizeof(uintptr_t) > sizeof(unsigned long))
-            length = strtoull(tok->p, NULL, 10);
-        else
-            length = strtoul(tok->p, NULL, 10);
-        next_token(tok);
-
+            if (sizeof(uintptr_t) > sizeof(unsigned long))
+                length = strtoull(tok->p, NULL, 10);
+            else
+                length = strtoul(tok->p, NULL, 10);
+            next_token(tok);
+        }
         assert(tok->kind == TOK_CLOSE_BRACKET);  // XXX
         next_token(tok);
 
@@ -235,7 +237,7 @@
         ds[2] = *jump_slot;
         *jump_slot = -(ds - tok->all_delay_slots);
         goto next_right_part;
-
+    }
     default:
         break;
     }
@@ -260,7 +262,10 @@
             case TOK_OPEN_BRACKET:   /* array */
             {
                 uintptr_t length = (uintptr_t)*delay_slot++;
-                t1 = tok->cb->get_array_type(tok->cb, t1, length);
+                if (length != (uintptr_t)-1)
+                    t1 = tok->cb->get_array_type(tok->cb, t1, length);
+                else
+                    t1 = tok->cb->get_incomplete_array_type(tok->cb, t1);
                 break;
             }
             case TOK_OPEN_PAREN:   /* function */
diff --git a/test/test_c_decl_parser.py b/test/test_c_decl_parser.py
--- a/test/test_c_decl_parser.py
+++ b/test/test_c_decl_parser.py
@@ -74,8 +74,9 @@
     parse("signed char", "signed char")
     parse("unsigned char", "unsigned char")
     parse("char(*(*)(long))(int)","PTR FUNC( long -> PTR FUNC( int -> char ) )")
+    parse("int[]", "ARRAY[] int")
     import py; py.test.skip("in-progress")
-    parse("foo_t[]")
+    parse("foo_t")
     parse("int(int, ...)")
     parse("int(int[])")
     parse("int(long(char))")

From noreply at buildbot.pypy.org  Sat Nov 29 10:09:13 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Sat, 29 Nov 2014 10:09:13 +0100 (CET)
Subject: [pypy-commit] creflect default: typedef'ed identifiers
Message-ID: <20141129090913.D28281C03A4@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r109:add8f0e282f4
Date: 2014-11-29 10:01 +0100
http://bitbucket.org/cffi/creflect/changeset/add8f0e282f4/

Log:	typedef'ed identifiers

diff --git a/creflect/src/c_decl_parser.c b/creflect/src/c_decl_parser.c
--- a/creflect/src/c_decl_parser.c
+++ b/creflect/src/c_decl_parser.c
@@ -379,6 +379,16 @@
         case TOK_VOID:
             t1 = tok->cb->get_void_type(tok->cb);
             break;
+        case TOK_IDENTIFIER:
+        {
+            char identifier[1024];
+            if (tok->size >= 1024)
+                return parse_error(tok);   // XXX
+            memcpy(identifier, tok->p, tok->size);
+            identifier[tok->size] = 0;
+            t1 = tok->cb->get_user_type(tok->cb, identifier);
+            break;
+        }
         default:
             tok->kind = TOK_ERROR;
             return NULL;
diff --git a/test/test_c_decl_parser.py b/test/test_c_decl_parser.py
--- a/test/test_c_decl_parser.py
+++ b/test/test_c_decl_parser.py
@@ -75,8 +75,9 @@
     parse("unsigned char", "unsigned char")
     parse("char(*(*)(long))(int)","PTR FUNC( long -> PTR FUNC( int -> char ) )")
     parse("int[]", "ARRAY[] int")
+    parse("foo_t", "foo_t")
+    parse("foo_t*", "PTR foo_t")
     import py; py.test.skip("in-progress")
-    parse("foo_t")
     parse("int(int, ...)")
     parse("int(int[])")
     parse("int(long(char))")

From noreply at buildbot.pypy.org  Sat Nov 29 10:09:14 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Sat, 29 Nov 2014 10:09:14 +0100 (CET)
Subject: [pypy-commit] creflect default: ellipsis in function types
Message-ID: <20141129090914.E84881C03A4@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r110:0bfe3f8ba02d
Date: 2014-11-29 10:04 +0100
http://bitbucket.org/cffi/creflect/changeset/0bfe3f8ba02d/

Log:	ellipsis in function types

diff --git a/creflect/src/c_decl_parser.c b/creflect/src/c_decl_parser.c
--- a/creflect/src/c_decl_parser.c
+++ b/creflect/src/c_decl_parser.c
@@ -17,6 +17,7 @@
     TOK_ERROR,
     TOK_IDENTIFIER,
     TOK_INTEGER,
+    TOK_DOTDOTDOT,
 
     /* keywords */
     TOK__BOOL,
@@ -95,6 +96,12 @@
                 tok->size++;
             return;
         }
+        else if (p[0] == '.' && p[1] == '.' && p[2] == '.') {
+            tok->kind = TOK_DOTDOTDOT;
+            tok->p = p;
+            tok->size = 3;
+            return;
+        }
         else if (*p) {
             tok->kind = *p;
             tok->p = p;
@@ -195,6 +202,11 @@
             }
             if (tok->kind != TOK_CLOSE_PAREN) {
                 while (1) {
+                    if (tok->kind == TOK_DOTDOTDOT) {
+                        ds[0] = TOK_DOTDOTDOT;
+                        next_token(tok);
+                        break;
+                    }
                     crx_type_t *t1 = parse_complete(tok);
                     intptr_t *ds_type = alloc_ds(tok, 1);
                     assert(ds_type == ds + 2 + ds[1]);
@@ -277,6 +289,15 @@
                                                 nbargs, NULL);
                 break;
             }
+            case TOK_DOTDOTDOT:   /* function ending with a '...' */
+            {
+                intptr_t nbargs = *delay_slot++;
+                crx_type_t **argtypes = (crx_type_t **)delay_slot;
+                delay_slot += nbargs;
+                t1 = tok->cb->get_ellipsis_function_type(tok->cb, t1,
+                                                         argtypes, nbargs);
+                break;
+            }
             default:
                 abort();
             }
diff --git a/test/test_c_decl_parser.py b/test/test_c_decl_parser.py
--- a/test/test_c_decl_parser.py
+++ b/test/test_c_decl_parser.py
@@ -77,8 +77,8 @@
     parse("int[]", "ARRAY[] int")
     parse("foo_t", "foo_t")
     parse("foo_t*", "PTR foo_t")
+    parse("int(int, ...)", "FUNC( int -> ... -> int )")
     import py; py.test.skip("in-progress")
-    parse("int(int, ...)")
     parse("int(int[])")
     parse("int(long(char))")
 

From noreply at buildbot.pypy.org  Sat Nov 29 10:09:15 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Sat, 29 Nov 2014 10:09:15 +0100 (CET)
Subject: [pypy-commit] creflect default: The two special rules for function
	argument types
Message-ID: <20141129090915.ECC1C1C03A4@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r111:9eb51bd08098
Date: 2014-11-29 10:09 +0100
http://bitbucket.org/cffi/creflect/changeset/9eb51bd08098/

Log:	The two special rules for function argument types

diff --git a/creflect/src/c_decl_parser.c b/creflect/src/c_decl_parser.c
--- a/creflect/src/c_decl_parser.c
+++ b/creflect/src/c_decl_parser.c
@@ -165,7 +165,7 @@
     return result;
 }
 
-static crx_type_t *parse_complete(crxp_token_t *tok);
+static crx_type_t *parse_complete(crxp_token_t *tok, int is_func_arg);
 
 static void parse_sequel(crxp_token_t *tok, intptr_t ds_end)
 {
@@ -207,7 +207,7 @@
                         next_token(tok);
                         break;
                     }
-                    crx_type_t *t1 = parse_complete(tok);
+                    crx_type_t *t1 = parse_complete(tok, 1);
                     intptr_t *ds_type = alloc_ds(tok, 1);
                     assert(ds_type == ds + 2 + ds[1]);
                     *ds_type = (intptr_t)t1;
@@ -256,7 +256,7 @@
 }
 
 static crx_type_t *fetch_delay_slots(crxp_token_t *tok, crx_type_t *t1,
-                                     intptr_t *delay_slot)
+                                     intptr_t *delay_slot, int is_func_arg)
 {
     tok->delay_slots = delay_slot;
     while (1) {
@@ -274,6 +274,11 @@
             case TOK_OPEN_BRACKET:   /* array */
             {
                 uintptr_t length = (uintptr_t)*delay_slot++;
+                if (is_func_arg && *delay_slot == TOK_END) {
+                    /* function argument: replace an array with a ptr */
+                    t1 = tok->cb->get_pointer_type(tok->cb, t1);
+                    break;
+                }
                 if (length != (uintptr_t)-1)
                     t1 = tok->cb->get_array_type(tok->cb, t1, length);
                 else
@@ -281,21 +286,21 @@
                 break;
             }
             case TOK_OPEN_PAREN:   /* function */
+            case TOK_DOTDOTDOT:    /* function ending with a '...' */
             {
                 intptr_t nbargs = *delay_slot++;
                 crx_type_t **argtypes = (crx_type_t **)delay_slot;
                 delay_slot += nbargs;
-                t1 = tok->cb->get_function_type(tok->cb, t1, argtypes,
-                                                nbargs, NULL);
-                break;
-            }
-            case TOK_DOTDOTDOT:   /* function ending with a '...' */
-            {
-                intptr_t nbargs = *delay_slot++;
-                crx_type_t **argtypes = (crx_type_t **)delay_slot;
-                delay_slot += nbargs;
-                t1 = tok->cb->get_ellipsis_function_type(tok->cb, t1,
-                                                         argtypes, nbargs);
+                if (tok_kind == TOK_DOTDOTDOT)
+                    t1 = tok->cb->get_ellipsis_function_type(tok->cb, t1,
+                                                             argtypes, nbargs);
+                else
+                    t1 = tok->cb->get_function_type(tok->cb, t1, argtypes,
+                                                    nbargs, NULL);
+                if (is_func_arg && *delay_slot == TOK_END) {
+                    /* function argument: replace a func with a ptr-to-func */
+                    t1 = tok->cb->get_pointer_type(tok->cb, t1);
+                }
                 break;
             }
             default:
@@ -314,7 +319,7 @@
     return NULL;
 }
 
-static crx_type_t *parse_complete(crxp_token_t *tok)
+static crx_type_t *parse_complete(crxp_token_t *tok, int is_func_arg)
 {
     crx_type_t *t1;
     int is_const = (tok->kind == TOK_CONST);
@@ -423,7 +428,7 @@
 
     intptr_t *orig = tok->delay_slots;
     parse_sequel(tok, TOK_END);
-    return fetch_delay_slots(tok, t1, orig);
+    return fetch_delay_slots(tok, t1, orig, is_func_arg);
 }
 
 const char *creflect_decl_parser(crx_builder_t *cb, const char *input,
@@ -436,7 +441,7 @@
     token.size = 0;
     token.delay_slots = token.all_delay_slots;
     next_token(&token);
-    *result = parse_complete(&token);
+    *result = parse_complete(&token, 0);
 
     if (token.kind == TOK_END)
         return NULL;
diff --git a/test/test_c_decl_parser.py b/test/test_c_decl_parser.py
--- a/test/test_c_decl_parser.py
+++ b/test/test_c_decl_parser.py
@@ -78,9 +78,8 @@
     parse("foo_t", "foo_t")
     parse("foo_t*", "PTR foo_t")
     parse("int(int, ...)", "FUNC( int -> ... -> int )")
-    import py; py.test.skip("in-progress")
-    parse("int(int[])")
-    parse("int(long(char))")
+    parse("int(int[])", "FUNC( PTR int -> int )")
+    parse("int(long(char))", "FUNC( PTR FUNC( char -> long ) -> int )")
 
 def test_c_decl_error():
     parse_error("*", 0)

From noreply at buildbot.pypy.org  Sat Nov 29 10:33:56 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Sat, 29 Nov 2014 10:33:56 +0100 (CET)
Subject: [pypy-commit] creflect default: Fix error cases
Message-ID: <20141129093356.BFE361C03A4@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r112:873c4b857b75
Date: 2014-11-29 10:34 +0100
http://bitbucket.org/cffi/creflect/changeset/873c4b857b75/

Log:	Fix error cases

diff --git a/creflect/src/c_decl_parser.c b/creflect/src/c_decl_parser.c
--- a/creflect/src/c_decl_parser.c
+++ b/creflect/src/c_decl_parser.c
@@ -156,11 +156,21 @@
     }
 }
 
+static crx_type_t *parse_error(crxp_token_t *tok, const char *msg)
+{
+    if (tok->kind != TOK_ERROR)
+        tok->cb->error(tok->cb, msg);
+    tok->kind = TOK_ERROR;
+    return NULL;
+}
+
 static intptr_t *alloc_ds(crxp_token_t *tok, size_t num)
 {
     intptr_t *result = tok->delay_slots;
-    if (num > (tok->all_delay_slots + NUM_DELAY_SLOTS - tok->delay_slots))
-        abort();  // XXX out of memory
+    if (num > (tok->all_delay_slots + NUM_DELAY_SLOTS - tok->delay_slots)) {
+        parse_error(tok, "type too lengthy");
+        return NULL;
+    }
     tok->delay_slots += num;
     return result;
 }
@@ -169,12 +179,18 @@
 
 static void parse_sequel(crxp_token_t *tok, intptr_t ds_end)
 {
+    intptr_t *ds;
     while (tok->kind == TOK_STAR || tok->kind == TOK_CONST) {
-        alloc_ds(tok, 1)[0] = tok->kind;
+        ds = alloc_ds(tok, 1);
+        if (ds == NULL)
+            return;
+        ds[0] = tok->kind;
         next_token(tok);
     }
 
-    intptr_t *ds, *jump_slot = alloc_ds(tok, 1);
+    intptr_t *jump_slot = alloc_ds(tok, 1);
+    if (jump_slot == NULL)
+        return;
     *jump_slot = ds_end;
 
  next_right_part:
@@ -194,6 +210,8 @@
         else {
             /* function type */
             ds = alloc_ds(tok, 2);
+            if (ds == NULL)
+                return;
             ds[0] = TOK_OPEN_PAREN;
             ds[1] = 0;
             if (tok->kind == TOK_VOID &&
@@ -209,6 +227,8 @@
                     }
                     crx_type_t *t1 = parse_complete(tok, 1);
                     intptr_t *ds_type = alloc_ds(tok, 1);
+                    if (ds_type == NULL)
+                        return;
                     assert(ds_type == ds + 2 + ds[1]);
                     *ds_type = (intptr_t)t1;
                     ds[1]++;
@@ -218,12 +238,17 @@
                 }
             }
             intptr_t *ds_next = alloc_ds(tok, 1);
+            if (ds_next == NULL)
+                return;
             assert(ds_next == ds + 2 + ds[1]);
             *ds_next = *jump_slot;
             *jump_slot = -(ds - tok->all_delay_slots);
         }
 
-        assert(tok->kind == TOK_CLOSE_PAREN);  // XXX
+        if (tok->kind != TOK_CLOSE_PAREN) {
+            parse_error(tok, "expected ')'");
+            return;
+        }
         next_token(tok);
         goto next_right_part;
 
@@ -232,7 +257,10 @@
         uintptr_t length = (uintptr_t)-1;
         next_token(tok);
         if (tok->kind != TOK_CLOSE_BRACKET) {
-            assert(tok->kind == TOK_INTEGER);  // XXX
+            if (tok->kind != TOK_INTEGER) {
+                parse_error(tok, "expected an integer constant");
+                return;
+            }
 
             if (sizeof(uintptr_t) > sizeof(unsigned long))
                 length = strtoull(tok->p, NULL, 10);
@@ -240,10 +268,16 @@
                 length = strtoul(tok->p, NULL, 10);
             next_token(tok);
         }
-        assert(tok->kind == TOK_CLOSE_BRACKET);  // XXX
+
+        if (tok->kind != TOK_CLOSE_BRACKET) {
+            parse_error(tok, "expected ']'");
+            return;
+        }
         next_token(tok);
 
         ds = alloc_ds(tok, 3);
+        if (ds == NULL)
+            return;
         ds[0] = TOK_OPEN_BRACKET;
         ds[1] = (intptr_t)length;
         ds[2] = *jump_slot;
@@ -258,6 +292,8 @@
 static crx_type_t *fetch_delay_slots(crxp_token_t *tok, crx_type_t *t1,
                                      intptr_t *delay_slot, int is_func_arg)
 {
+    if (tok->kind == TOK_ERROR)
+        return NULL;
     tok->delay_slots = delay_slot;
     while (1) {
         intptr_t tok_kind = *delay_slot++;
@@ -313,12 +349,6 @@
     }
 }
 
-static crx_type_t *parse_error(crxp_token_t *tok)
-{
-    tok->kind = TOK_ERROR;
-    return NULL;
-}
-
 static crx_type_t *parse_complete(crxp_token_t *tok, int is_func_arg)
 {
     crx_type_t *t1;
@@ -334,28 +364,30 @@
 
     case TOK_SHORT:
         if (modifiers_length != 0)
-            return parse_error(tok);
+            return parse_error(tok, "'short' after another 'short' or 'long'");
         modifiers_length--;
         next_token(tok);
         goto modifiers;
 
     case TOK_LONG:
-        if (modifiers_length != 0 && modifiers_length != 1)
-            return parse_error(tok);
+        if (modifiers_length < 0)
+            return parse_error(tok, "'long' after 'short'");
+        if (modifiers_length >= 2)
+            return parse_error(tok, "'long long long' is too long");
         modifiers_length++;
         next_token(tok);
         goto modifiers;
 
     case TOK_SIGNED:
         if (modifiers_sign)
-            return parse_error(tok);
+            return parse_error(tok, "multiple 'signed' or 'unsigned'");
         modifiers_sign++;
         next_token(tok);
         goto modifiers;
 
     case TOK_UNSIGNED:
         if (modifiers_sign)
-            return parse_error(tok);
+            return parse_error(tok, "multiple 'signed' or 'unsigned'");
         modifiers_sign--;
         next_token(tok);
         goto modifiers;
@@ -372,7 +404,7 @@
 
         case TOK_CHAR:
             if (modifiers_length != 0)
-                return parse_error(tok);
+                return parse_error(tok,"cannot combine with 'short' or 'long'");
             size = sizeof(char);
             name = "char";
             next_token(tok);
@@ -409,15 +441,14 @@
         {
             char identifier[1024];
             if (tok->size >= 1024)
-                return parse_error(tok);   // XXX
+                return parse_error(tok, "identifier name too long");
             memcpy(identifier, tok->p, tok->size);
             identifier[tok->size] = 0;
             t1 = tok->cb->get_user_type(tok->cb, identifier);
             break;
         }
         default:
-            tok->kind = TOK_ERROR;
-            return NULL;
+            return parse_error(tok, "identifier expected");
         }
         next_token(tok);
     }
@@ -445,5 +476,6 @@
 
     if (token.kind == TOK_END)
         return NULL;
+    parse_error(&token, "unexpected symbol");
     return token.p;
 }
diff --git a/test/test_c_decl_parser.py b/test/test_c_decl_parser.py
--- a/test/test_c_decl_parser.py
+++ b/test/test_c_decl_parser.py
@@ -14,7 +14,7 @@
         printf("%s\n", t1->text);
     }
     else {
-        printf("error: %s\n       ", argv[1]);
+        printf("%s\n", argv[1]);
         while (p > argv[1]) {
             printf(" ");
             p--;
@@ -44,8 +44,9 @@
     print got.rstrip()
     assert got == expected_output + '\n'
 
-def parse_error(input, expected_location):
-    parse(input, 'error: %s\n       %s^' % (input, " " * expected_location))
+def parse_error(input, expected_msg, expected_location):
+    exp = 'ERROR: %s\n%s\n%s^' % (expected_msg, input, " " * expected_location)
+    parse(input, exp)
 
 def test_c_decl_parser():
     parse("int **", "PTR PTR int")
@@ -82,5 +83,23 @@
     parse("int(long(char))", "FUNC( PTR FUNC( char -> long ) -> int )")
 
 def test_c_decl_error():
-    parse_error("*", 0)
-    parse_error("int ]**", 4)
+    parse_error("short short int", "'short' after another 'short' or 'long'", 6)
+    parse_error("long long long", "'long long long' is too long", 10)
+    parse_error("short long", "'long' after 'short'", 6)
+    parse_error("signed unsigned int", "multiple 'signed' or 'unsigned'", 7)
+    parse_error("unsigned signed int", "multiple 'signed' or 'unsigned'", 9)
+    parse_error("long char", "cannot combine with 'short' or 'long'", 5)
+    parse_error("short char", "cannot combine with 'short' or 'long'", 6)
+    #
+    parse_error("", "identifier expected", 0)
+    parse_error("]", "identifier expected", 0)
+    parse_error("*", "identifier expected", 0)
+    parse_error("int ]**", "unexpected symbol", 4)
+    parse_error("int(int]", "expected ')'", 7)
+    parse_error("int(*]", "expected ')'", 5)
+    parse_error("int(]", "identifier expected", 4)
+    parse_error("int[?]", "expected an integer constant", 4)
+    parse_error("int[24)", "expected ']'", 6)
+    #
+    parse_error("int" + "[]" * 2500, "type too lengthy", 3337)
+    parse_error("f" * 1200, "identifier name too long", 0)

From noreply at buildbot.pypy.org  Sat Nov 29 11:18:47 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Sat, 29 Nov 2014 11:18:47 +0100 (CET)
Subject: [pypy-commit] creflect default: The remaining primitive types
Message-ID: <20141129101847.C8E211C0589@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r113:deab50fe7ed9
Date: 2014-11-29 11:15 +0100
http://bitbucket.org/cffi/creflect/changeset/deab50fe7ed9/

Log:	The remaining primitive types

diff --git a/creflect/src/c_decl_parser.c b/creflect/src/c_decl_parser.c
--- a/creflect/src/c_decl_parser.c
+++ b/creflect/src/c_decl_parser.c
@@ -402,29 +402,41 @@
 
         switch (tok->kind) {
 
+        case TOK_VOID:
+        case TOK__BOOL:
+        case TOK_FLOAT:
+        case TOK_IDENTIFIER:
+            return parse_error(tok, "invalid combination of types");
+
+        case TOK_DOUBLE:
+            if (modifiers_sign != 0 || modifiers_length != 1)
+                return parse_error(tok, "invalid combination of types");
+            next_token(tok);
+            t1 = tok->cb->get_float_type(tok->cb, sizeof(long double),
+                                         "long double");
+            break;
+
         case TOK_CHAR:
             if (modifiers_length != 0)
-                return parse_error(tok,"cannot combine with 'short' or 'long'");
-            size = sizeof(char);
-            name = "char";
-            next_token(tok);
-            break;
-
+                return parse_error(tok, "invalid combination of types");
+            modifiers_length = -2;
+            /* fall-through */
         case TOK_INT:
             next_token(tok);
             /* fall-through */
         default:
             switch (modifiers_length) {
-            case -1: size = sizeof(short);     name = "short"; break;
-            case 1:  size = sizeof(long);      name = "long"; break;
+            case -2: size = sizeof(char);      name = "char";      break;
+            case -1: size = sizeof(short);     name = "short";     break;
+            case 1:  size = sizeof(long);      name = "long";      break;
             case 2:  size = sizeof(long long); name = "long long"; break;
-            default: size = sizeof(int);       name = "int"; break;
+            default: size = sizeof(int);       name = "int";       break;
             }
+            if (modifiers_sign < 0)
+                t1 = tok->cb->get_unsigned_type(tok->cb, size, name);
+            else
+                t1 = tok->cb->get_signed_type(tok->cb, size, name);
         }
-        if (modifiers_sign < 0)
-            t1 = tok->cb->get_unsigned_type(tok->cb, size, name);
-        else
-            t1 = tok->cb->get_signed_type(tok->cb, size, name);
     }
     else {
         switch (tok->kind) {
@@ -437,6 +449,15 @@
         case TOK_VOID:
             t1 = tok->cb->get_void_type(tok->cb);
             break;
+        case TOK__BOOL:
+            t1 = tok->cb->get_bool_type(tok->cb);
+            break;
+        case TOK_FLOAT:
+            t1 = tok->cb->get_float_type(tok->cb, sizeof(float), "float");
+            break;
+        case TOK_DOUBLE:
+            t1 = tok->cb->get_float_type(tok->cb, sizeof(double), "double");
+            break;
         case TOK_IDENTIFIER:
         {
             char identifier[1024];
diff --git a/test/test_c_decl_parser.py b/test/test_c_decl_parser.py
--- a/test/test_c_decl_parser.py
+++ b/test/test_c_decl_parser.py
@@ -81,6 +81,10 @@
     parse("int(int, ...)", "FUNC( int -> ... -> int )")
     parse("int(int[])", "FUNC( PTR int -> int )")
     parse("int(long(char))", "FUNC( PTR FUNC( char -> long ) -> int )")
+    parse("_Bool", "_Bool")
+    parse("float", "float")
+    parse("double", "double")
+    parse("long double", "long double")
 
 def test_c_decl_error():
     parse_error("short short int", "'short' after another 'short' or 'long'", 6)
@@ -88,13 +92,16 @@
     parse_error("short long", "'long' after 'short'", 6)
     parse_error("signed unsigned int", "multiple 'signed' or 'unsigned'", 7)
     parse_error("unsigned signed int", "multiple 'signed' or 'unsigned'", 9)
-    parse_error("long char", "cannot combine with 'short' or 'long'", 5)
-    parse_error("short char", "cannot combine with 'short' or 'long'", 6)
+    parse_error("long char", "invalid combination of types", 5)
+    parse_error("short char", "invalid combination of types", 6)
+    parse_error("signed void", "invalid combination of types", 7)
+    parse_error("unsigned foobar_t", "invalid combination of types", 9)
     #
     parse_error("", "identifier expected", 0)
     parse_error("]", "identifier expected", 0)
     parse_error("*", "identifier expected", 0)
     parse_error("int ]**", "unexpected symbol", 4)
+    parse_error("char char", "unexpected symbol", 5)
     parse_error("int(int]", "expected ')'", 7)
     parse_error("int(*]", "expected ')'", 5)
     parse_error("int(]", "identifier expected", 4)

From noreply at buildbot.pypy.org  Sat Nov 29 11:18:48 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Sat, 29 Nov 2014 11:18:48 +0100 (CET)
Subject: [pypy-commit] creflect default: structs and unions
Message-ID: <20141129101848.D26EC1C0589@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r114:5948e2d049a7
Date: 2014-11-29 11:19 +0100
http://bitbucket.org/cffi/creflect/changeset/5948e2d049a7/

Log:	structs and unions

diff --git a/creflect/src/c_decl_parser.c b/creflect/src/c_decl_parser.c
--- a/creflect/src/c_decl_parser.c
+++ b/creflect/src/c_decl_parser.c
@@ -406,6 +406,8 @@
         case TOK__BOOL:
         case TOK_FLOAT:
         case TOK_IDENTIFIER:
+        case TOK_STRUCT:
+        case TOK_UNION:
             return parse_error(tok, "invalid combination of types");
 
         case TOK_DOUBLE:
@@ -468,6 +470,25 @@
             t1 = tok->cb->get_user_type(tok->cb, identifier);
             break;
         }
+        case TOK_STRUCT:
+        case TOK_UNION:
+        {
+            char identifier[1024];
+            crx_type_t *(*get_type)(crx_builder_t *, const char *);
+            if (tok->kind == TOK_STRUCT)
+                get_type = tok->cb->get_struct_type;
+            else
+                get_type = tok->cb->get_union_type;
+            next_token(tok);
+            if (tok->kind != TOK_IDENTIFIER)
+                return parse_error(tok, "struct or union name expected");
+            if (tok->size >= 1024)
+                return parse_error(tok, "struct or union name too long");
+            memcpy(identifier, tok->p, tok->size);
+            identifier[tok->size] = 0;
+            t1 = get_type(tok->cb, identifier);
+            break;
+        }
         default:
             return parse_error(tok, "identifier expected");
         }
diff --git a/test/test_c_decl_parser.py b/test/test_c_decl_parser.py
--- a/test/test_c_decl_parser.py
+++ b/test/test_c_decl_parser.py
@@ -85,6 +85,8 @@
     parse("float", "float")
     parse("double", "double")
     parse("long double", "long double")
+    parse("struct foo_s", "STRUCT foo_s")
+    parse("union foo_u *", "PTR UNION foo_u")
 
 def test_c_decl_error():
     parse_error("short short int", "'short' after another 'short' or 'long'", 6)
@@ -107,6 +109,8 @@
     parse_error("int(]", "identifier expected", 4)
     parse_error("int[?]", "expected an integer constant", 4)
     parse_error("int[24)", "expected ']'", 6)
+    parse_error("struct", "struct or union name expected", 6)
+    parse_error("struct 24", "struct or union name expected", 7)
     #
     parse_error("int" + "[]" * 2500, "type too lengthy", 3337)
     parse_error("f" * 1200, "identifier name too long", 0)

From noreply at buildbot.pypy.org  Sat Nov 29 11:26:05 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Sat, 29 Nov 2014 11:26:05 +0100 (CET)
Subject: [pypy-commit] creflect default: small rewrites
Message-ID: <20141129102605.1AA551D2837@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r115:6318702e8b28
Date: 2014-11-29 11:26 +0100
http://bitbucket.org/cffi/creflect/changeset/6318702e8b28/

Log:	small rewrites

diff --git a/creflect/src/c_decl_parser.c b/creflect/src/c_decl_parser.c
--- a/creflect/src/c_decl_parser.c
+++ b/creflect/src/c_decl_parser.c
@@ -297,17 +297,20 @@
     tok->delay_slots = delay_slot;
     while (1) {
         intptr_t tok_kind = *delay_slot++;
-        if (tok_kind > 0) {
-            switch (tok_kind) {
-            case TOK_END:
-                return t1;
-            case TOK_STAR:
-                t1 = tok->cb->get_pointer_type(tok->cb, t1);
-                break;
-            case TOK_CONST:
-                t1 = tok->cb->get_const_type(tok->cb, t1);
-                break;
-            case TOK_OPEN_BRACKET:   /* array */
+        if (tok_kind <= 0) {
+            delay_slot = tok->all_delay_slots + (-tok_kind);
+            continue;
+        }
+        switch (tok_kind) {
+        case TOK_END:
+            return t1;
+        case TOK_STAR:
+            t1 = tok->cb->get_pointer_type(tok->cb, t1);
+            break;
+        case TOK_CONST:
+            t1 = tok->cb->get_const_type(tok->cb, t1);
+            break;
+        case TOK_OPEN_BRACKET:   /* array */
             {
                 uintptr_t length = (uintptr_t)*delay_slot++;
                 if (is_func_arg && *delay_slot == TOK_END) {
@@ -321,8 +324,8 @@
                     t1 = tok->cb->get_incomplete_array_type(tok->cb, t1);
                 break;
             }
-            case TOK_OPEN_PAREN:   /* function */
-            case TOK_DOTDOTDOT:    /* function ending with a '...' */
+        case TOK_OPEN_PAREN:   /* function */
+        case TOK_DOTDOTDOT:    /* function ending with a '...' */
             {
                 intptr_t nbargs = *delay_slot++;
                 crx_type_t **argtypes = (crx_type_t **)delay_slot;
@@ -339,12 +342,8 @@
                 }
                 break;
             }
-            default:
-                abort();
-            }
-        }
-        else {
-            delay_slot = tok->all_delay_slots + (-tok_kind);
+        default:
+            assert(!"missing delay slot case");
         }
     }
 }
@@ -397,8 +396,6 @@
     }
 
     if (modifiers_length || modifiers_sign) {
-        size_t size;
-        char *name;
 
         switch (tok->kind) {
 
@@ -427,17 +424,21 @@
             next_token(tok);
             /* fall-through */
         default:
-            switch (modifiers_length) {
-            case -2: size = sizeof(char);      name = "char";      break;
-            case -1: size = sizeof(short);     name = "short";     break;
-            case 1:  size = sizeof(long);      name = "long";      break;
-            case 2:  size = sizeof(long long); name = "long long"; break;
-            default: size = sizeof(int);       name = "int";       break;
+            {
+                size_t size;
+                char *name;
+                switch (modifiers_length) {
+                case -2: size = sizeof(char);      name = "char";      break;
+                case -1: size = sizeof(short);     name = "short";     break;
+                case 1:  size = sizeof(long);      name = "long";      break;
+                case 2:  size = sizeof(long long); name = "long long"; break;
+                default: size = sizeof(int);       name = "int";       break;
+                }
+                if (modifiers_sign < 0)
+                    t1 = tok->cb->get_unsigned_type(tok->cb, size, name);
+                else
+                    t1 = tok->cb->get_signed_type(tok->cb, size, name);
             }
-            if (modifiers_sign < 0)
-                t1 = tok->cb->get_unsigned_type(tok->cb, size, name);
-            else
-                t1 = tok->cb->get_signed_type(tok->cb, size, name);
         }
     }
     else {
@@ -474,11 +475,7 @@
         case TOK_UNION:
         {
             char identifier[1024];
-            crx_type_t *(*get_type)(crx_builder_t *, const char *);
-            if (tok->kind == TOK_STRUCT)
-                get_type = tok->cb->get_struct_type;
-            else
-                get_type = tok->cb->get_union_type;
+            int kind = tok->kind;
             next_token(tok);
             if (tok->kind != TOK_IDENTIFIER)
                 return parse_error(tok, "struct or union name expected");
@@ -486,7 +483,10 @@
                 return parse_error(tok, "struct or union name too long");
             memcpy(identifier, tok->p, tok->size);
             identifier[tok->size] = 0;
-            t1 = get_type(tok->cb, identifier);
+            if (kind == TOK_STRUCT)
+                t1 = tok->cb->get_struct_type(tok->cb, identifier);
+            else
+                t1 = tok->cb->get_union_type(tok->cb, identifier);
             break;
         }
         default:

From noreply at buildbot.pypy.org  Sat Nov 29 12:08:32 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Sat, 29 Nov 2014 12:08:32 +0100 (CET)
Subject: [pypy-commit] creflect default: More corner cases
Message-ID: <20141129110832.C5AD01C073F@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r116:61c8089f6f6b
Date: 2014-11-29 12:08 +0100
http://bitbucket.org/cffi/creflect/changeset/61c8089f6f6b/

Log:	More corner cases

diff --git a/creflect/src/c_decl_parser.c b/creflect/src/c_decl_parser.c
--- a/creflect/src/c_decl_parser.c
+++ b/creflect/src/c_decl_parser.c
@@ -69,14 +69,14 @@
     return (is_ident_first(x) || is_digit(x));
 }
 
-static int is_following_token_this_char(crxp_token_t *tok, char expected)
+static char get_following_char(crxp_token_t *tok)
 {
     const char *p = tok->p + tok->size;
     if (tok->kind == TOK_ERROR)
         return 0;
     while (is_space(*p))
         p++;
-    return *p == expected;
+    return *p;
 }
 
 static void next_token(crxp_token_t *tok)
@@ -188,20 +188,28 @@
         next_token(tok);
     }
 
+    int check_for_grouping = -1;
+    if (tok->kind == TOK_IDENTIFIER) {
+        next_token(tok);    /* skip a potential variable name */
+        check_for_grouping = 1;
+    }
+
     intptr_t *jump_slot = alloc_ds(tok, 1);
     if (jump_slot == NULL)
         return;
     *jump_slot = ds_end;
 
  next_right_part:
+    check_for_grouping++;
+
     switch (tok->kind) {
 
     case TOK_OPEN_PAREN:
         next_token(tok);
 
-        if (tok->kind == TOK_STAR ||
-            tok->kind == TOK_CONST ||
-            tok->kind == TOK_OPEN_BRACKET) {
+        if (check_for_grouping == 0 && (tok->kind == TOK_STAR ||
+                                        tok->kind == TOK_CONST ||
+                                        tok->kind == TOK_OPEN_BRACKET)) {
             /* just parentheses for grouping */
             ds = tok->delay_slots;
             parse_sequel(tok, *jump_slot);
@@ -214,8 +222,7 @@
                 return;
             ds[0] = TOK_OPEN_PAREN;
             ds[1] = 0;
-            if (tok->kind == TOK_VOID &&
-                    is_following_token_this_char(tok, ')')) {
+            if (tok->kind == TOK_VOID && get_following_char(tok) == ')') {
                 next_token(tok);
             }
             if (tok->kind != TOK_CLOSE_PAREN) {
diff --git a/test/test_c_decl_parser.py b/test/test_c_decl_parser.py
--- a/test/test_c_decl_parser.py
+++ b/test/test_c_decl_parser.py
@@ -87,6 +87,10 @@
     parse("long double", "long double")
     parse("struct foo_s", "STRUCT foo_s")
     parse("union foo_u *", "PTR UNION foo_u")
+    parse("void *", "PTR void")
+    parse("int a", "int")
+    parse("int *abc", "PTR int")
+    parse("int myfunc(int a[])", "FUNC( PTR int -> int )")
 
 def test_c_decl_error():
     parse_error("short short int", "'short' after another 'short' or 'long'", 6)
@@ -111,6 +115,8 @@
     parse_error("int[24)", "expected ']'", 6)
     parse_error("struct", "struct or union name expected", 6)
     parse_error("struct 24", "struct or union name expected", 7)
+    parse_error("int[5](*)", "identifier expected", 7)
+    parse_error("int a(*)", "identifier expected", 6)
     #
     parse_error("int" + "[]" * 2500, "type too lengthy", 3337)
     parse_error("f" * 1200, "identifier name too long", 0)

From noreply at buildbot.pypy.org  Sat Nov 29 12:18:39 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Sat, 29 Nov 2014 12:18:39 +0100 (CET)
Subject: [pypy-commit] creflect default: tweak
Message-ID: <20141129111839.B3A531D2705@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r117:8781274565f2
Date: 2014-11-29 12:19 +0100
http://bitbucket.org/cffi/creflect/changeset/8781274565f2/

Log:	tweak

diff --git a/creflect/src/creflect_check.c b/creflect/src/creflect_check.c
--- a/creflect/src/creflect_check.c
+++ b/creflect/src/creflect_check.c
@@ -9,7 +9,7 @@
 #include "creflect_print.h"
 
 
-void creflect_dump_information(const char *filename)
+void creflect_dump_information(const char *filename, const char *creflect_main)
 {
     char filename_ex[PATH_MAX];
 
@@ -24,7 +24,7 @@
         goto err;
 
     void (*crxmain)(crx_builder_t *);
-    crxmain = (void(*)(crx_builder_t *))dlsym(lib, "_creflect_main");
+    crxmain = (void(*)(crx_builder_t *))dlsym(lib, creflect_main);
     if (crxmain == NULL)
         goto err;
 
@@ -44,11 +44,14 @@
 
 int main(int argc, char *argv[])
 {
-    if (argc != 2) {
-        fprintf(stderr, "One argument needed: the name of the library file"
-                        " out of which the creflect information is dumped.\n");
+    if (argc != 2 && argc != 3) {
+        fprintf(stderr, 
+            "Usage: %s libfilename [entrypoint]\n\n"
+            "Dumps the creflect information out of the library by calling the\n"
+            "entry point.  It is named '_creflect_main' by default, but can\n"
+            "also be specified as second argument.\n", argv[0]);
         return 2;
     }
-    creflect_dump_information(argv[1]);
+    creflect_dump_information(argv[1], argc == 2 ? "_creflect_main" : argv[2]);
     return 0;
 }

From noreply at buildbot.pypy.org  Sat Nov 29 12:34:32 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Sat, 29 Nov 2014 12:34:32 +0100 (CET)
Subject: [pypy-commit] creflect default: Fix test
Message-ID: <20141129113432.AD4921D2705@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r118:a280890af84f
Date: 2014-11-29 12:22 +0100
http://bitbucket.org/cffi/creflect/changeset/a280890af84f/

Log:	Fix test

diff --git a/test/test_cparser.py b/test/test_cparser.py
--- a/test/test_cparser.py
+++ b/test/test_cparser.py
@@ -29,6 +29,9 @@
     ast.show(f)
     assert f.getvalue() == """\
 FileAST: 
+  Typedef: __crx_unknown_t, [], ['typedef']
+    TypeDecl: __crx_unknown_t, []
+      IdentifierType: ['int']
   Decl: f, [], [], []
     FuncDecl: 
       ParamList: 

From noreply at buildbot.pypy.org  Sat Nov 29 12:34:33 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Sat, 29 Nov 2014 12:34:33 +0100 (CET)
Subject: [pypy-commit] creflect default: tweaks
Message-ID: <20141129113433.B6FB01D2705@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r119:cc5e3f72c5b9
Date: 2014-11-29 12:34 +0100
http://bitbucket.org/cffi/creflect/changeset/cc5e3f72c5b9/

Log:	tweaks

diff --git a/creflect/src/c_decl_parser.c b/creflect/src/c_decl_parser.c
--- a/creflect/src/c_decl_parser.c
+++ b/creflect/src/c_decl_parser.c
@@ -265,7 +265,7 @@
         next_token(tok);
         if (tok->kind != TOK_CLOSE_BRACKET) {
             if (tok->kind != TOK_INTEGER) {
-                parse_error(tok, "expected an integer constant");
+                parse_error(tok, "expected a positive integer constant");
                 return;
             }
 
@@ -273,6 +273,10 @@
                 length = strtoull(tok->p, NULL, 10);
             else
                 length = strtoul(tok->p, NULL, 10);
+            if (length == (uintptr_t)-1) {
+                parse_error(tok, "number too large");
+                return;
+            }
             next_token(tok);
         }
 
diff --git a/test/test_c_decl_parser.py b/test/test_c_decl_parser.py
--- a/test/test_c_decl_parser.py
+++ b/test/test_c_decl_parser.py
@@ -91,6 +91,7 @@
     parse("int a", "int")
     parse("int *abc", "PTR int")
     parse("int myfunc(int a[])", "FUNC( PTR int -> int )")
+    parse("abcd efgh", "abcd")    # 'efgh' is the variable name, ignored
 
 def test_c_decl_error():
     parse_error("short short int", "'short' after another 'short' or 'long'", 6)
@@ -111,12 +112,13 @@
     parse_error("int(int]", "expected ')'", 7)
     parse_error("int(*]", "expected ')'", 5)
     parse_error("int(]", "identifier expected", 4)
-    parse_error("int[?]", "expected an integer constant", 4)
+    parse_error("int[?]", "expected a positive integer constant", 4)
     parse_error("int[24)", "expected ']'", 6)
     parse_error("struct", "struct or union name expected", 6)
     parse_error("struct 24", "struct or union name expected", 7)
     parse_error("int[5](*)", "identifier expected", 7)
     parse_error("int a(*)", "identifier expected", 6)
+    parse_error("int[123456789012345678901234567890]", "number too large", 4)
     #
     parse_error("int" + "[]" * 2500, "type too lengthy", 3337)
     parse_error("f" * 1200, "identifier name too long", 0)

From noreply at buildbot.pypy.org  Sat Nov 29 13:49:38 2014
From: noreply at buildbot.pypy.org (fijal)
Date: Sat, 29 Nov 2014 13:49:38 +0100 (CET)
Subject: [pypy-commit] pypy optresult: a minor rewrite + minor progress here,
 not sure about the correct API yet
Message-ID: <20141129124938.878981C0589@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: optresult
Changeset: r74758:4ce71001d704
Date: 2014-11-29 14:49 +0200
http://bitbucket.org/pypy/pypy/changeset/4ce71001d704/

Log:	a minor rewrite + minor progress here, not sure about the correct
	API yet

diff --git a/rpython/jit/metainterp/history.py b/rpython/jit/metainterp/history.py
--- a/rpython/jit/metainterp/history.py
+++ b/rpython/jit/metainterp/history.py
@@ -185,6 +185,9 @@
 class Const(AbstractValue):
     __slots__ = ()
 
+    is_source_op = True
+    source_op = None
+
     @staticmethod
     def _new(x):
         "NOT_RPYTHON"
diff --git a/rpython/jit/metainterp/optimizeopt/intbounds.py b/rpython/jit/metainterp/optimizeopt/intbounds.py
--- a/rpython/jit/metainterp/optimizeopt/intbounds.py
+++ b/rpython/jit/metainterp/optimizeopt/intbounds.py
@@ -49,11 +49,10 @@
         assert not op.is_ovf()
         self.emit_operation(op)
 
-    def propagate_bounds_backward(self, box):
+    def propagate_bounds_backward(self, box, v):
         # FIXME: This takes care of the instruction where box is the reuslt
         #        but the bounds produced by all instructions where box is
         #        an argument might also be tighten
-        v = self.getvalue(box)
         b = v.intbound
         if b.has_lower and b.has_upper and b.lower == b.upper:
             v.make_constant(ConstInt(b.lower))
@@ -62,8 +61,9 @@
             dispatch_bounds_ops(self, box)
 
     def optimize_GUARD_TRUE(self, op):
+        v = self.getvalue(op.getarg(0))
         self.emit_operation(op)
-        self.propagate_bounds_backward(op.getarg(0))
+        self.propagate_bounds_backward(op.getarg(0), v)
 
     optimize_GUARD_FALSE = optimize_GUARD_TRUE
     optimize_GUARD_VALUE = optimize_GUARD_TRUE
@@ -237,25 +237,33 @@
         v1 = self.getvalue(op.getarg(0))
         v2 = self.getvalue(op.getarg(1))
         resbound = v1.intbound.add_bound(v2.intbound)
+        r = self.getvalue(op)
         if resbound.bounded():
             # Transform into INT_ADD.  The following guard will be killed
             # by optimize_GUARD_NO_OVERFLOW; if we see instead an
             # optimize_GUARD_OVERFLOW, then InvalidLoop.
-            op = self.optimizer.replace_op_with(op, rop.INT_ADD)
-        self.emit_operation(op) # emit the op
+            newop = op.copy_and_change(rop.INT_ADD)
+            r.box = newop
+        else:
+            newop = op
+        self.emit_operation(newop) # emit the op
         r = self.getvalue(op)
         r.intbound.intersect(resbound)
 
     def optimize_INT_SUB_OVF(self, op):
         v1 = self.getvalue(op.getarg(0))
         v2 = self.getvalue(op.getarg(1))
+        r = self.getvalue(op)
         if v1 is v2:
             self.make_constant_int(op, 0)
             return
         resbound = v1.intbound.sub_bound(v2.intbound)
         if resbound.bounded():
-            op = self.optimizer.replace_op_with(op, rop.INT_SUB)
-        self.emit_operation(op) # emit the op
+            newop = op.copy_and_change(rop.INT_SUB)
+            r.box = newop
+        else:
+            newop = op
+        self.emit_operation(newop) # emit the op
         r = self.getvalue(op)
         r.intbound.intersect(resbound)
 
@@ -410,17 +418,17 @@
         v1 = self.getvalue(box1)
         v2 = self.getvalue(box2)
         if v1.intbound.make_lt(v2.intbound):
-            self.propagate_bounds_backward(box1)
+            self.propagate_bounds_backward(box1, v1)
         if v2.intbound.make_gt(v1.intbound):
-            self.propagate_bounds_backward(box2)
+            self.propagate_bounds_backward(box2, v2)
 
     def make_int_le(self, box1, box2):
         v1 = self.getvalue(box1)
         v2 = self.getvalue(box2)
         if v1.intbound.make_le(v2.intbound):
-            self.propagate_bounds_backward(box1)
+            self.propagate_bounds_backward(box1, v1)
         if v2.intbound.make_ge(v1.intbound):
-            self.propagate_bounds_backward(box2)
+            self.propagate_bounds_backward(box2, v2)
 
     def make_int_gt(self, box1, box2):
         self.make_int_lt(box2, box1)
@@ -467,9 +475,9 @@
                 v1 = self.getvalue(op.getarg(0))
                 v2 = self.getvalue(op.getarg(1))
                 if v1.intbound.intersect(v2.intbound):
-                    self.propagate_bounds_backward(op.getarg(0))
+                    self.propagate_bounds_backward(op.getarg(0), v1)
                 if v2.intbound.intersect(v1.intbound):
-                    self.propagate_bounds_backward(op.getarg(1))
+                    self.propagate_bounds_backward(op.getarg(1), v2)
 
     def propagate_bounds_INT_NE(self, op):
         r = self.getvalue(op)
@@ -489,7 +497,7 @@
                 v1 = self.getvalue(op.getarg(0))
                 if v1.intbound.known_ge(IntBound(0, 0)):
                     v1.intbound.make_gt(IntBound(0, 0))
-                    self.propagate_bounds_backward(op.getarg(0))
+                    self.propagate_bounds_backward(op.getarg(0), v1)
 
     def propagate_bounds_INT_IS_ZERO(self, op):
         r = self.getvalue(op)
@@ -501,7 +509,7 @@
                 # an assert, this is a clever way of expressing the same thing.
                 v1.intbound.make_ge(IntBound(0, 0))
                 v1.intbound.make_lt(IntBound(1, 1))
-                self.propagate_bounds_backward(op.getarg(0))
+                self.propagate_bounds_backward(op.getarg(0), v1)
 
     def propagate_bounds_INT_ADD(self, op):
         v1 = self.getvalue(op.getarg(0))
@@ -509,10 +517,10 @@
         r = self.getvalue(op)
         b = r.intbound.sub_bound(v2.intbound)
         if v1.intbound.intersect(b):
-            self.propagate_bounds_backward(op.getarg(0))
+            self.propagate_bounds_backward(op.getarg(0), v1)
         b = r.intbound.sub_bound(v1.intbound)
         if v2.intbound.intersect(b):
-            self.propagate_bounds_backward(op.getarg(1))
+            self.propagate_bounds_backward(op.getarg(1), v2)
 
     def propagate_bounds_INT_SUB(self, op):
         v1 = self.getvalue(op.getarg(0))
@@ -520,10 +528,10 @@
         r = self.getvalue(op)
         b = r.intbound.add_bound(v2.intbound)
         if v1.intbound.intersect(b):
-            self.propagate_bounds_backward(op.getarg(0))
+            self.propagate_bounds_backward(op.getarg(0), v1)
         b = r.intbound.sub_bound(v1.intbound).mul(-1)
         if v2.intbound.intersect(b):
-            self.propagate_bounds_backward(op.getarg(1))
+            self.propagate_bounds_backward(op.getarg(1), v2)
 
     def propagate_bounds_INT_MUL(self, op):
         v1 = self.getvalue(op.getarg(0))
diff --git a/rpython/jit/metainterp/optimizeopt/optimizer.py b/rpython/jit/metainterp/optimizeopt/optimizer.py
--- a/rpython/jit/metainterp/optimizeopt/optimizer.py
+++ b/rpython/jit/metainterp/optimizeopt/optimizer.py
@@ -64,15 +64,6 @@
             self.make_constant(box)
         # invariant: box is a Const if and only if level == LEVEL_CONSTANT
 
-    def copy_attributes_from(self, other):
-        assert other.__class__ is OptValue
-        self.level = other.level
-        self.known_class = other.known_class
-        self.intbound = other.intbound
-        self.lenbound = other.lenbound
-        self.box = other.box
-        self.last_guard = other.last_guard
-
     def make_len_gt(self, mode, descr, val):
         if self.lenbound:
             assert self.lenbound.mode == mode
@@ -454,6 +445,9 @@
 
     @specialize.argtype(0)
     def getvalue(self, box):
+        while box.source_op is not None:
+            box = box.source_op
+        assert box.is_source_op
         box = self.getinterned(box)
         try:
             value = self.values[box]
@@ -487,19 +481,13 @@
     def clear_newoperations(self):
         self._newoperations = []
 
-    def make_equal_to(self, box, newvalue):
-        if box in self.values:
-            v = self.getvalue(box)
-            v.copy_attributes_from(newvalue)
-        else:
-            self.values[box] = newvalue
-
-    def replace_op_with(self, oldop, newopnum, args=None):
-        newop = oldop._copy_and_change(newopnum, args=args)
-        v = self.getvalue(oldop)
-        v.box = newop
-        self.values[newop] = v
-        return newop
+    def make_equal_to(self, box, value, replace=False):
+        assert isinstance(value, OptValue)
+        assert replace or box not in self.values
+        while box.source_op is not None:
+            box = box.source_op
+        assert box.is_source_op
+        self.values[box] = value
 
     def make_constant(self, box, constbox):
         try:
@@ -574,6 +562,10 @@
     @specialize.argtype(0)
     def _emit_operation(self, op):
         assert not op.is_call_pure()
+        if op.getopnum() == rop.GUARD_VALUE:
+            val = self.getvalue(op.getarg(0))
+        else:
+            val = None
         for i in range(op.numargs()):
             arg = op.getarg(i)
             try:
@@ -593,7 +585,7 @@
                 del self.replaces_guard[op]
                 return
             else:
-                op = self.store_final_boxes_in_guard(op, pendingfields)
+                op = self.store_final_boxes_in_guard(op, pendingfields, val)
         elif op.can_raise():
             self.exception_might_have_happened = True
         self._newoperations.append(op)
@@ -609,7 +601,7 @@
         else:
             assert False
 
-    def store_final_boxes_in_guard(self, op, pendingfields):
+    def store_final_boxes_in_guard(self, op, pendingfields, val):
         assert pendingfields is not None
         descr = op.getdescr()
         assert isinstance(descr, compile.ResumeGuardDescr)
@@ -624,7 +616,7 @@
         descr.store_final_boxes(op, newboxes, self.metainterp_sd)
         #
         if op.getopnum() == rop.GUARD_VALUE:
-            if self.getvalue(op.getarg(0)) in self.bool_boxes:
+            if val in self.bool_boxes:
                 # Hack: turn guard_value(bool) into guard_true/guard_false.
                 # This is done after the operation is emitted to let
                 # store_final_boxes_in_guard set the guard_opnum field of the
diff --git a/rpython/jit/metainterp/optimizeopt/pure.py b/rpython/jit/metainterp/optimizeopt/pure.py
--- a/rpython/jit/metainterp/optimizeopt/pure.py
+++ b/rpython/jit/metainterp/optimizeopt/pure.py
@@ -42,7 +42,7 @@
                                                 op.getarglist(), op.getdescr())
             oldop = self.pure_operations.get(args, None)
             if oldop is not None and oldop.getdescr() is op.getdescr():
-                self.optimizer.make_equal_to(op, self.getvalue(oldop))
+                self.optimizer.make_equal_to(op, self.getvalue(oldop), True)
                 return
             else:
                 self.pure_operations[args] = op
@@ -84,6 +84,7 @@
         args = op.getarglist()
         opnum = OpHelpers.call_for_descr(op.getdescr())
         newop = ResOperation(opnum, args, op.getdescr())
+        newop.source_op = op
         self.getvalue(op).box = newop
         self.emit_operation(newop)
     optimize_CALL_PURE_R = optimize_CALL_PURE_I
diff --git a/rpython/jit/metainterp/optimizeopt/rewrite.py b/rpython/jit/metainterp/optimizeopt/rewrite.py
--- a/rpython/jit/metainterp/optimizeopt/rewrite.py
+++ b/rpython/jit/metainterp/optimizeopt/rewrite.py
@@ -220,8 +220,8 @@
                     self.make_equal_to(op, v2)
                     return
                 elif v1.box.getfloatstorage() == -1.0:
-                    newop = self.optimizer.replace_op_with(op, rop.FLOAT_NEG,
-                                                           args=[rhs])
+                    newop = op.copy_and_change(rop.FLOAT_NEG, args=[rhs])
+                    self.getvalue(op).box = newop
                     self.emit_operation(newop)
                     return
         self.emit_operation(op)
@@ -233,6 +233,7 @@
         v2 = self.getvalue(arg2)
 
         # replace "x / const" by "x * (1/const)" if possible
+        newop = op
         if v2.is_constant():
             divisor = v2.box.getfloatstorage()
             fraction = math.frexp(divisor)[0]
@@ -243,9 +244,9 @@
                 rfraction = math.frexp(reciprocal)[0]
                 if rfraction == 0.5 or rfraction == -0.5:
                     c = ConstFloat(longlong.getfloatstorage(reciprocal))
-                    op = self.optimizer.replace_op_with(op, rop.FLOAT_MUL,
-                                                        args=[arg1, c])
-        self.emit_operation(op)
+                    newop = op.copy_and_change(rop.FLOAT_MUL, args=[arg1, c])
+                    self.getvalue(op).box = newop
+        self.emit_operation(newop)
 
     def optimize_FLOAT_NEG(self, op):
         v1 = op.getarg(0)
@@ -291,6 +292,7 @@
 
     def optimize_GUARD_VALUE(self, op):
         value = self.getvalue(op.getarg(0))
+        opv = self.getvalue(op)
         if value.is_virtual():
             arg = value.get_constant_class(self.optimizer.cpu)
             if arg:
@@ -316,10 +318,8 @@
                     r = self.optimizer.metainterp_sd.logger_ops.repr_of_resop(op)
                     raise InvalidLoop('A GUARD_VALUE (%s) was proven to always fail' % r)
             arglist = [old_guard_op.getarg(0), op.getarg(1)]
-            op = self.optimizer.replace_op_with(old_guard_op,
-                                                rop.GUARD_VALUE,
-                                                args=arglist)
-            self.getvalue(old_guard_op).box = op
+            op = old_guard_op.copy_and_change(rop.GUARD_VALUE, args=arglist)
+            opv.box = op
             self.optimizer.replaces_guard[op] = old_guard_op
             # hack hack hack.  Change the guard_opnum on
             # new_guard_op.getdescr() so that when resuming,
@@ -351,6 +351,7 @@
         value.make_constant_class(expectedclassbox, None)
 
     def optimize_GUARD_CLASS(self, op):
+        opv = self.getvalue(op)
         value = self.getvalue(op.getarg(0))
         expectedclassbox = op.getarg(1)
         assert isinstance(expectedclassbox, Const)
@@ -369,9 +370,8 @@
                 # it was a guard_nonnull, which we replace with a
                 # guard_nonnull_class.
                 args = [old_guard_op.getarg(0), op.getarg(1)]
-                op = self.optimizer.replace_op_with(old_guard_op,
-                                                    rop.GUARD_NONNULL_CLASS,
-                                                    args)
+                op = old_guard_op.copy_and_change(rop.GUARD_NONNULL_CLASS, args)
+                opv.box = op
                 self.optimizer.replaces_guard[op] = old_guard_op
                 # hack hack hack.  Change the guard_opnum on
                 # new_guard_op.getdescr() so that when resuming,
@@ -406,10 +406,10 @@
         # there is no reason to have a separate operation for this
         self.loop_invariant_producer[key] = op
         opnum = OpHelpers.call_for_descr(op.getdescr())
-        newop = self.optimizer.replace_op_with(op, opnum)
+        newop = op.copy_and_change(opnum)
+        resvalue = self.optimizer.getvalue(op)
+        resvalue.box = newop
         self.emit_operation(newop)
-        resvalue = self.getvalue(op)
-        resvalue.box = newop
         self.loop_invariant_results[key] = resvalue
     optimize_CALL_LOOPINVARIANT_R = optimize_CALL_LOOPINVARIANT_I
     optimize_CALL_LOOPINVARIANT_F = optimize_CALL_LOOPINVARIANT_I
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py
@@ -27,7 +27,7 @@
     snapshot0 = resume.Snapshot(None, [b0])
     fdescr.rd_snapshot = resume.Snapshot(snapshot0, [b1])
     #
-    opt.store_final_boxes_in_guard(op, [])
+    opt.store_final_boxes_in_guard(op, [], None)
     if op.getfailargs() == [b0, b1]:
         assert list(fdescr.rd_numb.nums)      == [tag(1, TAGBOX)]
         assert list(fdescr.rd_numb.prev.nums) == [tag(0, TAGBOX)]
diff --git a/rpython/jit/metainterp/resoperation.py b/rpython/jit/metainterp/resoperation.py
--- a/rpython/jit/metainterp/resoperation.py
+++ b/rpython/jit/metainterp/resoperation.py
@@ -39,6 +39,9 @@
     boolreflex = -1
     boolinverse = -1
 
+    is_source_op = False
+    source_op = None
+
     _attrs_ = ()
 
     def getopnum(self):
@@ -87,7 +90,7 @@
     # common methods
     # --------------
 
-    def _copy_and_change(self, opnum, args=None, descr=None):
+    def copy_and_change(self, opnum, args=None, descr=None):
         "shallow copy: the returned operation is meant to be used in place of self"
         if args is None:
             args = self.getarglist()
@@ -96,6 +99,7 @@
         newop = ResOperation(opnum, args, descr)
         if self.type != 'v':
             newop.copy_value_from(self)
+        newop.source_op = self
         return newop
 
     @specialize.argtype(1)
@@ -279,8 +283,8 @@
     def setfailargs(self, fail_args):
         self._fail_args = fail_args
 
-    def _copy_and_change(self, opnum, args=None, descr=None):
-        newop = AbstractResOp._copy_and_change(self, opnum, args, descr)
+    def copy_and_change(self, opnum, args=None, descr=None):
+        newop = AbstractResOp.copy_and_change(self, opnum, args, descr)
         newop.setfailargs(self.getfailargs())
         return newop
 
@@ -359,7 +363,10 @@
     def nonnull(self):
         return bool(self._resref)
 
-class AbstractInputArg(AbstractValue):    
+class AbstractInputArg(AbstractValue):
+    is_source_op = True
+    source_op = None
+    
     def repr(self, memo):
         try:
             return memo[self]
diff --git a/rpython/jit/tool/oparser.py b/rpython/jit/tool/oparser.py
--- a/rpython/jit/tool/oparser.py
+++ b/rpython/jit/tool/oparser.py
@@ -17,6 +17,7 @@
     OPNUM = -123
 
     type = 'i'
+    is_source_op = True
 
     def getopnum(self):
         return self.OPNUM
@@ -32,6 +33,7 @@
 class FORCE_SPILL(UnaryOp, PlainResOp):
 
     OPNUM = -124
+    is_source_op = True
 
     def getopnum(self):
         return self.OPNUM
@@ -277,7 +279,9 @@
             assert descr is None
             return op
         else:
-            return ResOperation(opnum, args, descr)
+            res = ResOperation(opnum, args, descr)
+            res.is_source_op = True
+            return res
 
     def parse_result_op(self, line):
         res, op = line.split("=", 1)

From noreply at buildbot.pypy.org  Sat Nov 29 15:16:31 2014
From: noreply at buildbot.pypy.org (fijal)
Date: Sat, 29 Nov 2014 15:16:31 +0100 (CET)
Subject: [pypy-commit] pypy optresult: small progress,
 I might reverse the whole approach though
Message-ID: <20141129141631.B4ECE1C03A4@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: optresult
Changeset: r74759:4080e1ba1276
Date: 2014-11-29 16:16 +0200
http://bitbucket.org/pypy/pypy/changeset/4080e1ba1276/

Log:	small progress, I might reverse the whole approach though

diff --git a/rpython/jit/metainterp/optimizeopt/intbounds.py b/rpython/jit/metainterp/optimizeopt/intbounds.py
--- a/rpython/jit/metainterp/optimizeopt/intbounds.py
+++ b/rpython/jit/metainterp/optimizeopt/intbounds.py
@@ -237,13 +237,11 @@
         v1 = self.getvalue(op.getarg(0))
         v2 = self.getvalue(op.getarg(1))
         resbound = v1.intbound.add_bound(v2.intbound)
-        r = self.getvalue(op)
         if resbound.bounded():
             # Transform into INT_ADD.  The following guard will be killed
             # by optimize_GUARD_NO_OVERFLOW; if we see instead an
             # optimize_GUARD_OVERFLOW, then InvalidLoop.
-            newop = op.copy_and_change(rop.INT_ADD)
-            r.box = newop
+            newop = self.replace_op_with(op, rop.INT_ADD)
         else:
             newop = op
         self.emit_operation(newop) # emit the op
@@ -253,14 +251,12 @@
     def optimize_INT_SUB_OVF(self, op):
         v1 = self.getvalue(op.getarg(0))
         v2 = self.getvalue(op.getarg(1))
-        r = self.getvalue(op)
         if v1 is v2:
             self.make_constant_int(op, 0)
             return
         resbound = v1.intbound.sub_bound(v2.intbound)
         if resbound.bounded():
-            newop = op.copy_and_change(rop.INT_SUB)
-            r.box = newop
+            newop = self.replace_op_with(op, rop.INT_SUB)
         else:
             newop = op
         self.emit_operation(newop) # emit the op
diff --git a/rpython/jit/metainterp/optimizeopt/optimizer.py b/rpython/jit/metainterp/optimizeopt/optimizer.py
--- a/rpython/jit/metainterp/optimizeopt/optimizer.py
+++ b/rpython/jit/metainterp/optimizeopt/optimizer.py
@@ -293,6 +293,9 @@
     def getvalue(self, box):
         return self.optimizer.getvalue(box)
 
+    def replace_op_with(self, op, newopnum, args=None):
+        return self.optimizer.replace_op_with(op, newopnum, args)
+
     def make_constant(self, box, constbox):
         return self.optimizer.make_constant(box, constbox)
 
@@ -497,6 +500,12 @@
         except KeyError:
             self.values[box] = ConstantValue(constbox)
 
+    def replace_op_with(self, op, newopnum, args=None):
+        v = self.getvalue(op)
+        newop = op.copy_and_change(newopnum, args=args)
+        v.box = newop
+        return newop
+
     def make_constant_int(self, box, intvalue):
         self.make_constant(box, ConstInt(intvalue))
 
diff --git a/rpython/jit/metainterp/optimizeopt/pure.py b/rpython/jit/metainterp/optimizeopt/pure.py
--- a/rpython/jit/metainterp/optimizeopt/pure.py
+++ b/rpython/jit/metainterp/optimizeopt/pure.py
@@ -41,7 +41,7 @@
             args = self.optimizer.make_args_key(op.getopnum(),
                                                 op.getarglist(), op.getdescr())
             oldop = self.pure_operations.get(args, None)
-            if oldop is not None and oldop.getdescr() is op.getdescr():
+            if oldop is not None:
                 self.optimizer.make_equal_to(op, self.getvalue(oldop), True)
                 return
             else:
diff --git a/rpython/jit/metainterp/optimizeopt/rewrite.py b/rpython/jit/metainterp/optimizeopt/rewrite.py
--- a/rpython/jit/metainterp/optimizeopt/rewrite.py
+++ b/rpython/jit/metainterp/optimizeopt/rewrite.py
@@ -220,8 +220,7 @@
                     self.make_equal_to(op, v2)
                     return
                 elif v1.box.getfloatstorage() == -1.0:
-                    newop = op.copy_and_change(rop.FLOAT_NEG, args=[rhs])
-                    self.getvalue(op).box = newop
+                    newop = self.replace_op_with(op, rop.FLOAT_NEG, args=[rhs])
                     self.emit_operation(newop)
                     return
         self.emit_operation(op)
@@ -244,8 +243,8 @@
                 rfraction = math.frexp(reciprocal)[0]
                 if rfraction == 0.5 or rfraction == -0.5:
                     c = ConstFloat(longlong.getfloatstorage(reciprocal))
-                    newop = op.copy_and_change(rop.FLOAT_MUL, args=[arg1, c])
-                    self.getvalue(op).box = newop
+                    newop = self.replace_op_with(op, rop.FLOAT_MUL,
+                                                 args=[arg1, c])
         self.emit_operation(newop)
 
     def optimize_FLOAT_NEG(self, op):
@@ -405,10 +404,9 @@
         # change the op to be a normal call, from the backend's point of view
         # there is no reason to have a separate operation for this
         self.loop_invariant_producer[key] = op
-        opnum = OpHelpers.call_for_descr(op.getdescr())
-        newop = op.copy_and_change(opnum)
+        newop = self.replace_op_with(op,
+                                     OpHelpers.call_for_descr(op.getdescr()))
         resvalue = self.optimizer.getvalue(op)
-        resvalue.box = newop
         self.emit_operation(newop)
         self.loop_invariant_results[key] = resvalue
     optimize_CALL_LOOPINVARIANT_R = optimize_CALL_LOOPINVARIANT_I
@@ -521,6 +519,7 @@
                                       [op.getarg(1),
                                        ConstInt(index + source_start)],
                                        descr=arraydescr)
+                    newop.is_source_op = True
                     self.optimizer.send_extra_operation(newop)
                     val = self.getvalue(newop)
                 if val is None:
diff --git a/rpython/jit/metainterp/optimizeopt/virtualize.py b/rpython/jit/metainterp/optimizeopt/virtualize.py
--- a/rpython/jit/metainterp/optimizeopt/virtualize.py
+++ b/rpython/jit/metainterp/optimizeopt/virtualize.py
@@ -606,10 +606,10 @@
         # 'jit_virtual_ref'.  The jit_virtual_ref structure may be forced soon,
         # but the point is that doing so does not force the original structure.
         newop = ResOperation(rop.NEW_WITH_VTABLE, [c_cls])
+        newop.source_op = op
         vrefvalue = self.make_virtual(c_cls, newop)
-        assert op not in self.optimizer.values
-        self.optimizer.values[op] = vrefvalue
         token = ResOperation(rop.FORCE_TOKEN, [])
+        token.is_source_op = True
         self.emit_operation(token)
         vrefvalue.setfield(descr_virtual_token, self.getvalue(token))
         vrefvalue.setfield(descr_forced, self.optimizer.cpu.ts.CVAL_NULLREF)

From noreply at buildbot.pypy.org  Sat Nov 29 16:11:30 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Sat, 29 Nov 2014 16:11:30 +0100 (CET)
Subject: [pypy-commit] creflect default: Kill a feature,
	with comments for why
Message-ID: <20141129151130.D99B11C03A4@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r120:315cd9372b22
Date: 2014-11-29 16:06 +0100
http://bitbucket.org/cffi/creflect/changeset/315cd9372b22/

Log:	Kill a feature, with comments for why

diff --git a/creflect/src/c_decl_parser.c b/creflect/src/c_decl_parser.c
--- a/creflect/src/c_decl_parser.c
+++ b/creflect/src/c_decl_parser.c
@@ -175,7 +175,7 @@
     return result;
 }
 
-static crx_type_t *parse_complete(crxp_token_t *tok, int is_func_arg);
+static crx_type_t *parse_complete(crxp_token_t *tok);
 
 static void parse_sequel(crxp_token_t *tok, intptr_t ds_end)
 {
@@ -232,7 +232,7 @@
                         next_token(tok);
                         break;
                     }
-                    crx_type_t *t1 = parse_complete(tok, 1);
+                    crx_type_t *t1 = parse_complete(tok);
                     intptr_t *ds_type = alloc_ds(tok, 1);
                     if (ds_type == NULL)
                         return;
@@ -301,7 +301,7 @@
 }
 
 static crx_type_t *fetch_delay_slots(crxp_token_t *tok, crx_type_t *t1,
-                                     intptr_t *delay_slot, int is_func_arg)
+                                     intptr_t *delay_slot)
 {
     if (tok->kind == TOK_ERROR)
         return NULL;
@@ -324,11 +324,6 @@
         case TOK_OPEN_BRACKET:   /* array */
             {
                 uintptr_t length = (uintptr_t)*delay_slot++;
-                if (is_func_arg && *delay_slot == TOK_END) {
-                    /* function argument: replace an array with a ptr */
-                    t1 = tok->cb->get_pointer_type(tok->cb, t1);
-                    break;
-                }
                 if (length != (uintptr_t)-1)
                     t1 = tok->cb->get_array_type(tok->cb, t1, length);
                 else
@@ -347,10 +342,6 @@
                 else
                     t1 = tok->cb->get_function_type(tok->cb, t1, argtypes,
                                                     nbargs, NULL);
-                if (is_func_arg && *delay_slot == TOK_END) {
-                    /* function argument: replace a func with a ptr-to-func */
-                    t1 = tok->cb->get_pointer_type(tok->cb, t1);
-                }
                 break;
             }
         default:
@@ -359,7 +350,7 @@
     }
 }
 
-static crx_type_t *parse_complete(crxp_token_t *tok, int is_func_arg)
+static crx_type_t *parse_complete(crxp_token_t *tok)
 {
     crx_type_t *t1;
     int is_const = (tok->kind == TOK_CONST);
@@ -512,7 +503,7 @@
 
     intptr_t *orig = tok->delay_slots;
     parse_sequel(tok, TOK_END);
-    return fetch_delay_slots(tok, t1, orig, is_func_arg);
+    return fetch_delay_slots(tok, t1, orig);
 }
 
 const char *creflect_decl_parser(crx_builder_t *cb, const char *input,
@@ -525,7 +516,7 @@
     token.size = 0;
     token.delay_slots = token.all_delay_slots;
     next_token(&token);
-    *result = parse_complete(&token, 0);
+    *result = parse_complete(&token);
 
     if (token.kind == TOK_END)
         return NULL;
diff --git a/test/test_c_decl_parser.py b/test/test_c_decl_parser.py
--- a/test/test_c_decl_parser.py
+++ b/test/test_c_decl_parser.py
@@ -79,8 +79,11 @@
     parse("foo_t", "foo_t")
     parse("foo_t*", "PTR foo_t")
     parse("int(int, ...)", "FUNC( int -> ... -> int )")
-    parse("int(int[])", "FUNC( PTR int -> int )")
-    parse("int(long(char))", "FUNC( PTR FUNC( char -> long ) -> int )")
+    # The following should be interpreted like 'int(int *)', but it is not
+    # the job of c_decl_parser to change it -- it can't if the array type
+    # is typedef'ed.  Same for 'int(long(char))' => 'int(long(*)(char))'.
+    parse("int(int[])", "FUNC( ARRAY[] int -> int )")
+    parse("int(long(char))", "FUNC( FUNC( char -> long ) -> int )")
     parse("_Bool", "_Bool")
     parse("float", "float")
     parse("double", "double")
@@ -90,7 +93,7 @@
     parse("void *", "PTR void")
     parse("int a", "int")
     parse("int *abc", "PTR int")
-    parse("int myfunc(int a[])", "FUNC( PTR int -> int )")
+    parse("int myfunc(int a[])", "FUNC( ARRAY[] int -> int )")
     parse("abcd efgh", "abcd")    # 'efgh' is the variable name, ignored
 
 def test_c_decl_error():

From noreply at buildbot.pypy.org  Sat Nov 29 18:14:35 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Sat, 29 Nov 2014 18:14:35 +0100 (CET)
Subject: [pypy-commit] creflect default: Standardize the location and usage
 of the .c/.h files
Message-ID: <20141129171435.43E4D1D370F@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r121:f71be8e76e58
Date: 2014-11-29 18:10 +0100
http://bitbucket.org/cffi/creflect/changeset/f71be8e76e58/

Log:	Standardize the location and usage of the .c/.h files

diff too long, truncating to 2000 out of 2035 lines

diff --git a/creflect/codegen.py b/creflect/codegen.py
--- a/creflect/codegen.py
+++ b/creflect/codegen.py
@@ -65,16 +65,16 @@
         decl.write_declaration(funcblock)
 
     if funcblock.crx_type_vars:
-        funcblock.writedecl("crx_type_t %s;" % (
+        funcblock.writedecl("_crx_type_t %s;" % (
             ', '.join(funcblock.crx_type_vars),))
     for d1 in funcblock.crx_field_vars:
-        funcblock.writedecl("crx_field_t %s;" % d1)
+        funcblock.writedecl("_crx_field_t %s;" % d1)
 
     static_decl = ""
     if static:
         static_decl = "static "
-    outerblock.writeline('%svoid %s(crx_builder_t *cb)' % (static_decl,
-                                                           crx_func_name))
+    outerblock.writeline('%svoid %s(_crx_builder_t *cb)' % (static_decl,
+                                                            crx_func_name))
     outerblock.write_subblock(funcblock)
     outerblock.writeline('')     # for the final \n below
     return '\n'.join(outerblock.codelines)
diff --git a/creflect/src/creflect.h b/creflect/creflect.h
rename from creflect/src/creflect.h
rename to creflect/creflect.h
--- a/creflect/src/creflect.h
+++ b/creflect/creflect.h
@@ -1,20 +1,20 @@
-#ifndef _CREFLECT_H_
-#define _CREFLECT_H_
+#ifndef _CRX_H_
+#define _CRX_H_
 
 #include 
 #include 
 
 
-typedef struct crx_type_s crx_type_t;   /* opaque */
-typedef void (*crx_trampoline0_fn)(void *args[], void *res);
-typedef void (*crx_trampoline1_fn)(void *fn, void *args[], void *res);
+typedef struct _crx_type_s _crx_type_t;   /* opaque */
+typedef void (*_crx_trampoline0_fn)(void *args[], void *res);
+typedef void (*_crx_trampoline1_fn)(void *fn, void *args[], void *res);
 
 typedef struct {
     const char *name;
-    crx_type_t *type;
+    _crx_type_t *type;
     size_t offset;
     int numbits, bitshift;
-} crx_field_t;
+} _crx_field_t;
 
 typedef union {
     int as_int;
@@ -27,52 +27,53 @@
     float as_float;
     double as_double;
     long double as_long_double;
-} crx_num_const_t;
+} _crx_num_const_t;
 
-#define CRX_SELF struct _crx_builder_s *
+#define _CRX_SELF struct _crx_builder_s *
 typedef struct _crx_builder_s {
-    crx_type_t *(*get_void_type)(CRX_SELF);
-    crx_type_t *(*get_char_type)(CRX_SELF);
-    crx_type_t *(*get_bool_type)(CRX_SELF);
-    crx_type_t *(*get_signed_type)(CRX_SELF, size_t,
+    _crx_type_t *(*get_void_type)(_CRX_SELF);
+    _crx_type_t *(*get_char_type)(_CRX_SELF);
+    _crx_type_t *(*get_bool_type)(_CRX_SELF);
+    _crx_type_t *(*get_signed_type)(_CRX_SELF, size_t,
+                                    const char *);
+    _crx_type_t *(*get_unsigned_type)(_CRX_SELF, size_t,
+                                      const char *);
+    _crx_type_t *(*get_float_type)(_CRX_SELF, size_t,
                                    const char *);
-    crx_type_t *(*get_unsigned_type)(CRX_SELF, size_t,
-                                     const char *);
-    crx_type_t *(*get_float_type)(CRX_SELF, size_t,
-                                  const char *);
-    crx_type_t *(*get_function_type)(CRX_SELF, crx_type_t *,
-                                     crx_type_t *[], int, crx_trampoline1_fn);
-    crx_type_t *(*get_ellipsis_function_type)(CRX_SELF, crx_type_t *,
-                                              crx_type_t *[], int);
-    crx_type_t *(*get_pointer_type)(CRX_SELF, crx_type_t *);
-    crx_type_t *(*get_const_type)(CRX_SELF, crx_type_t *);
-    crx_type_t *(*get_array_type)(CRX_SELF, crx_type_t *, size_t);
-    crx_type_t *(*get_incomplete_array_type)(CRX_SELF, crx_type_t *);
-    crx_type_t *(*get_struct_type)(CRX_SELF, const char *);
-    crx_type_t *(*get_union_type)(CRX_SELF, const char *);
-    crx_type_t *(*get_enum_type)(CRX_SELF, const char *);
-    crx_type_t *(*get_user_type)(CRX_SELF, const char *);
-    crx_type_t *(*get_unknown_type)(CRX_SELF, const char *);
-    void (*complete)(CRX_SELF, crx_type_t *,
-                     size_t, size_t, crx_field_t[], int);
-    void (*complete_enum)(CRX_SELF, crx_type_t *, crx_type_t *);
-    void (*define_type)(CRX_SELF, const char *, crx_type_t *);
-    void (*define_var)(CRX_SELF, const char *, crx_type_t *,
+    _crx_type_t *(*get_function_type)(_CRX_SELF, _crx_type_t *,
+                                      _crx_type_t *[], int,
+                                      _crx_trampoline1_fn);
+    _crx_type_t *(*get_ellipsis_function_type)(_CRX_SELF, _crx_type_t *,
+                                               _crx_type_t *[], int);
+    _crx_type_t *(*get_pointer_type)(_CRX_SELF, _crx_type_t *);
+    _crx_type_t *(*get_const_type)(_CRX_SELF, _crx_type_t *);
+    _crx_type_t *(*get_array_type)(_CRX_SELF, _crx_type_t *, size_t);
+    _crx_type_t *(*get_incomplete_array_type)(_CRX_SELF, _crx_type_t *);
+    _crx_type_t *(*get_struct_type)(_CRX_SELF, const char *);
+    _crx_type_t *(*get_union_type)(_CRX_SELF, const char *);
+    _crx_type_t *(*get_enum_type)(_CRX_SELF, const char *);
+    _crx_type_t *(*get_user_type)(_CRX_SELF, const char *);
+    _crx_type_t *(*get_unknown_type)(_CRX_SELF, const char *);
+    void (*complete)(_CRX_SELF, _crx_type_t *,
+                     size_t, size_t, _crx_field_t[], int);
+    void (*complete_enum)(_CRX_SELF, _crx_type_t *, _crx_type_t *);
+    void (*define_type)(_CRX_SELF, const char *, _crx_type_t *);
+    void (*define_var)(_CRX_SELF, const char *, _crx_type_t *,
                        void *);
-    void (*define_func)(CRX_SELF, const char *, crx_type_t *,
-                        crx_type_t *[], int, crx_trampoline0_fn, void *);
-    void (*define_num_const)(CRX_SELF, const char *, crx_type_t *,
-                             crx_num_const_t *);
-    void (*error)(CRX_SELF, const char *);
-} crx_builder_t;
-#undef CRX_SELF
+    void (*define_func)(_CRX_SELF, const char *, _crx_type_t *,
+                        _crx_type_t *[], int, _crx_trampoline0_fn, void *);
+    void (*define_num_const)(_CRX_SELF, const char *, _crx_type_t *,
+                             _crx_num_const_t *);
+    void (*error)(_CRX_SELF, const char *);
+} _crx_builder_t;
+#undef _CRX_SELF
 
 
-#define CRX_INT_TYPE(cb, expr, guessname)                               \
-    _creflect__int_type(cb, expr > 0, sizeof(expr), expr == 1, guessname)
+#define _CRX_INT_TYPE(cb, expr, guessname)                               \
+    _crx_int_type(cb, expr > 0, sizeof(expr), expr == 1, guessname)
 
-#define CRX_INT_CONST(cb, expr, vp, pn)                 \
-    _creflect__int_const(cb, vp, pn,                    \
+#define _CRX_INT_CONST(cb, expr, vp, pn)                 \
+    _crx_int_const(cb, vp, pn,                    \
         "integer constant '" #expr "' is too large",    \
         !(((expr) * 0 + 4) << (sizeof(int)*8-2)),       \
         !(((expr) * 0L + 4L) << (sizeof(long)*8-2)),    \
@@ -81,19 +82,19 @@
         (expr) == (long long)(expr),                    \
         (unsigned long long)(expr))
 
-#define CRX_CHAR_CONST(cb, expr, vp)                                    \
-    _creflect__char_const(cb, vp,                                       \
+#define _CRX_CHAR_CONST(cb, expr, vp)                                    \
+    _crx_char_const(cb, vp,                                       \
         "char constant '" #expr "' is too large",                       \
         (expr) == (signed char)(expr) || (expr) == (unsigned char)(expr), \
         (char)(expr))
 
-#define CRX_FLOAT_CONST(cb, expr, vp)                                   \
-    _creflect__float_const(cb, vp, sizeof(expr), (long double)(expr))
+#define _CRX_FLOAT_CONST(cb, expr, vp)                                   \
+    _crx_float_const(cb, vp, sizeof(expr), (long double)(expr))
 
 __attribute__((unused))
-static crx_type_t *_creflect__int_type(crx_builder_t *cb, int expr_positive,
-                                       size_t size_of_expr, int expr_equal_one,
-                                       const char *guessname)
+static _crx_type_t *_crx_int_type(_crx_builder_t *cb, int expr_positive,
+                                  size_t size_of_expr, int expr_equal_one,
+                                  const char *guessname)
 {
     if (!expr_positive)
         return cb->get_signed_type(cb, size_of_expr, guessname);
@@ -104,12 +105,12 @@
 }
 
 __attribute__((unused))
-static crx_type_t *_creflect__int_const(crx_builder_t *cb, crx_num_const_t *vp,
-                                        int preference_number,
-                                        const char *toobig,
-                                        int fits_int, int fits_long,
-                                        int unsign, int fits_ull, int fits_ll,
-                                        unsigned long long value)
+static _crx_type_t *_crx_int_const(_crx_builder_t *cb, _crx_num_const_t *vp,
+                                   int preference_number,
+                                   const char *toobig,
+                                   int fits_int, int fits_long,
+                                   int unsign, int fits_ull, int fits_ll,
+                                   unsigned long long value)
 {
     size_t size;
     const char *name;
@@ -170,9 +171,9 @@
 }
 
 __attribute__((unused))
-static void _creflect__char_const(crx_builder_t *cb, crx_num_const_t *vp,
-                                  const char *toobig,
-                                  int fits_char, char value)
+static void _crx_char_const(_crx_builder_t *cb, _crx_num_const_t *vp,
+                            const char *toobig,
+                            int fits_char, char value)
 {
     if (!fits_char)
         cb->error(cb, toobig);
@@ -181,9 +182,9 @@
 }
 
 __attribute__((unused))
-static crx_type_t *_creflect__float_const(crx_builder_t *cb,
-                                          crx_num_const_t *vp,
-                                          size_t size, long double value)
+static _crx_type_t *_crx_float_const(_crx_builder_t *cb,
+                                     _crx_num_const_t *vp,
+                                     size_t size, long double value)
 {
     const char *name;
 
@@ -202,4 +203,4 @@
     return cb->get_float_type(cb, size, name);
 }
 
-#endif  /* _CREFLECT_H_ */
+#endif  /* _CRX_H_ */
diff --git a/creflect/src/c_decl_parser.c b/creflect/creflect_cdecl.c
rename from creflect/src/c_decl_parser.c
rename to creflect/creflect_cdecl.c
--- a/creflect/src/c_decl_parser.c
+++ b/creflect/creflect_cdecl.c
@@ -1,10 +1,10 @@
 #include 
 #include 
 #include 
-#include "creflect.h"
+#include "creflect_cdecl.h"
 
 
-enum crxp_token_e {
+enum token_e {
     TOK_STAR='*',
     TOK_OPEN_PAREN='(',
     TOK_CLOSE_PAREN=')',
@@ -40,13 +40,13 @@
 #define NUM_DELAY_SLOTS   5000
 
 typedef struct {
-    enum crxp_token_e kind;
-    const char *p;
+    enum token_e kind;
+    const char *p, **error_location;
     size_t size;
-    crx_builder_t *cb;
+    _crx_builder_t *cb;
     intptr_t *delay_slots;
     intptr_t all_delay_slots[NUM_DELAY_SLOTS];
-} crxp_token_t;
+} token_t;
 
 static int is_space(char x)
 {
@@ -69,7 +69,7 @@
     return (is_ident_first(x) || is_digit(x));
 }
 
-static char get_following_char(crxp_token_t *tok)
+static char get_following_char(token_t *tok)
 {
     const char *p = tok->p + tok->size;
     if (tok->kind == TOK_ERROR)
@@ -79,7 +79,7 @@
     return *p;
 }
 
-static void next_token(crxp_token_t *tok)
+static void next_token(token_t *tok)
 {
     const char *p = tok->p + tok->size;
     if (tok->kind == TOK_ERROR)
@@ -156,15 +156,18 @@
     }
 }
 
-static crx_type_t *parse_error(crxp_token_t *tok, const char *msg)
+static _crx_type_t *parse_error(token_t *tok, const char *msg)
 {
-    if (tok->kind != TOK_ERROR)
+    if (tok->kind != TOK_ERROR) {
+        tok->kind = TOK_ERROR;
+        if (tok->error_location)
+            *tok->error_location = tok->p;
         tok->cb->error(tok->cb, msg);
-    tok->kind = TOK_ERROR;
+    }
     return NULL;
 }
 
-static intptr_t *alloc_ds(crxp_token_t *tok, size_t num)
+static intptr_t *alloc_ds(token_t *tok, size_t num)
 {
     intptr_t *result = tok->delay_slots;
     if (num > (tok->all_delay_slots + NUM_DELAY_SLOTS - tok->delay_slots)) {
@@ -175,9 +178,9 @@
     return result;
 }
 
-static crx_type_t *parse_complete(crxp_token_t *tok);
+static _crx_type_t *parse_complete(token_t *tok);
 
-static void parse_sequel(crxp_token_t *tok, intptr_t ds_end)
+static void parse_sequel(token_t *tok, intptr_t ds_end)
 {
     intptr_t *ds;
     while (tok->kind == TOK_STAR || tok->kind == TOK_CONST) {
@@ -232,7 +235,7 @@
                         next_token(tok);
                         break;
                     }
-                    crx_type_t *t1 = parse_complete(tok);
+                    _crx_type_t *t1 = parse_complete(tok);
                     intptr_t *ds_type = alloc_ds(tok, 1);
                     if (ds_type == NULL)
                         return;
@@ -300,7 +303,7 @@
     }
 }
 
-static crx_type_t *fetch_delay_slots(crxp_token_t *tok, crx_type_t *t1,
+static _crx_type_t *fetch_delay_slots(token_t *tok, _crx_type_t *t1,
                                      intptr_t *delay_slot)
 {
     if (tok->kind == TOK_ERROR)
@@ -334,7 +337,7 @@
         case TOK_DOTDOTDOT:    /* function ending with a '...' */
             {
                 intptr_t nbargs = *delay_slot++;
-                crx_type_t **argtypes = (crx_type_t **)delay_slot;
+                _crx_type_t **argtypes = (_crx_type_t **)delay_slot;
                 delay_slot += nbargs;
                 if (tok_kind == TOK_DOTDOTDOT)
                     t1 = tok->cb->get_ellipsis_function_type(tok->cb, t1,
@@ -350,9 +353,9 @@
     }
 }
 
-static crx_type_t *parse_complete(crxp_token_t *tok)
+static _crx_type_t *parse_complete(token_t *tok)
 {
-    crx_type_t *t1;
+    _crx_type_t *t1;
     int is_const = (tok->kind == TOK_CONST);
     if (is_const) {
         next_token(tok);
@@ -506,20 +509,27 @@
     return fetch_delay_slots(tok, t1, orig);
 }
 
-const char *creflect_decl_parser(crx_builder_t *cb, const char *input,
-                                 crx_type_t **result)
+_crx_type_t *creflect_decl_parser(_crx_builder_t *cb, const char *input,
+                                  const char **error_location)
 {
-    crxp_token_t token;
+    _crx_type_t *result;
+    token_t token;
     token.kind = TOK_START;
     token.cb = cb;
     token.p = input;
+    token.error_location = error_location;
     token.size = 0;
     token.delay_slots = token.all_delay_slots;
     next_token(&token);
-    *result = parse_complete(&token);
+    result = parse_complete(&token);
 
-    if (token.kind == TOK_END)
+    if (token.kind == TOK_END) {
+        if (error_location)
+            *error_location = NULL;
+        return result;
+    }
+    else {
+        parse_error(&token, "unexpected symbol");
         return NULL;
-    parse_error(&token, "unexpected symbol");
-    return token.p;
+    }
 }
diff --git a/creflect/creflect_cdecl.h b/creflect/creflect_cdecl.h
new file mode 100644
--- /dev/null
+++ b/creflect/creflect_cdecl.h
@@ -0,0 +1,11 @@
+#ifndef _CRX_CDECL_H_
+#define _CRX_CDECL_H_
+
+#include "creflect.h"
+
+
+_crx_type_t *creflect_decl_parser(_crx_builder_t *cb, const char *input,
+                                  const char **error_location);
+
+
+#endif  /* _CRX_CDECL_H_ */
diff --git a/creflect/creflect_cdecl_main.c b/creflect/creflect_cdecl_main.c
new file mode 100644
--- /dev/null
+++ b/creflect/creflect_cdecl_main.c
@@ -0,0 +1,30 @@
+#include 
+#include 
+#include "creflect_cdecl.h"
+#include "creflect_debug_print.h"
+
+int main(int argc, char *argv[])
+{
+    _crx_type_t *t1;
+    const char *error;
+
+    if (argc != 2) {
+        fprintf(stderr, "Usage: %s 'c_type_declaration'\n", argv[0]);
+        return 2;
+    }
+    t1 = creflect_decl_parser(&creflect_debug_builder, argv[1], &error);
+    if (error == NULL) {
+        assert(t1 != NULL);
+        printf("%s\n", creflect_type_text(t1));
+    }
+    else {
+        assert(t1 == NULL);
+        printf("%s\n", argv[1]);
+        while (error > argv[1]) {
+            printf(" ");
+            error--;
+        }
+        printf("^\n");
+    }
+    return 0;
+}
diff --git a/creflect/src/creflect_print.h b/creflect/creflect_debug_print.c
rename from creflect/src/creflect_print.h
rename to creflect/creflect_debug_print.c
--- a/creflect/src/creflect_print.h
+++ b/creflect/creflect_debug_print.c
@@ -2,47 +2,53 @@
 #include 
 #include 
 #include 
+#include "creflect_debug_print.h"
 
 
-struct crx_type_s {
+struct _crx_type_s {
     char text[1];
 };
 
 
-static crx_type_t *newtype(const char *a)
+const char *creflect_type_text(_crx_type_t *t1)
+{
+    return t1->text;
+}
+
+static _crx_type_t *newtype(const char *a)
 {
     size_t la = strlen(a);
-    crx_type_t *t = malloc(la + 1);
+    _crx_type_t *t = malloc(la + 1);
     strcpy(t->text, a);
     return t;
 }
 
-static crx_type_t *newtype2(const char *a, const char *b)
+static _crx_type_t *newtype2(const char *a, const char *b)
 {
     size_t la = strlen(a);
     size_t lb = strlen(b);
-    crx_type_t *t = malloc(la + lb + 1);
+    _crx_type_t *t = malloc(la + lb + 1);
     strcat(strcpy(t->text, a), b);
     return t;
 }
 
-static crx_type_t *tst_get_void_type(crx_builder_t *cb)
+static _crx_type_t *tst_get_void_type(_crx_builder_t *cb)
 {
     return newtype("void");
 }
 
-static crx_type_t *tst_get_char_type(crx_builder_t *cb)
+static _crx_type_t *tst_get_char_type(_crx_builder_t *cb)
 {
     return newtype("char");
 }
 
-static crx_type_t *tst_get_bool_type(crx_builder_t *cb)
+static _crx_type_t *tst_get_bool_type(_crx_builder_t *cb)
 {
     return newtype("_Bool");
 }
 
-static crx_type_t *tst_get_signed_type(crx_builder_t *cb, size_t sz,
-                                       const char *g)
+static _crx_type_t *tst_get_signed_type(_crx_builder_t *cb, size_t sz,
+                                        const char *g)
 {
     int skip = 0;
     if (sizeof(long) == sizeof(long long))
@@ -63,8 +69,8 @@
     abort();
 }
 
-static crx_type_t *tst_get_unsigned_type(crx_builder_t *cb, size_t sz,
-                                         const char *g)
+static _crx_type_t *tst_get_unsigned_type(_crx_builder_t *cb, size_t sz,
+                                          const char *g)
 {
     int skip = 0;
     if (sizeof(long) == sizeof(long long))
@@ -84,8 +90,8 @@
     abort();
 }
 
-static crx_type_t *tst_get_float_type(crx_builder_t *cb, size_t sz,
-                                      const char *g)
+static _crx_type_t *tst_get_float_type(_crx_builder_t *cb, size_t sz,
+                                       const char *g)
 {
     int skip = 0;
     if (sizeof(double) == sizeof(long double))
@@ -101,9 +107,9 @@
     abort();
 }
 
-static crx_type_t *tst_get_function_type(crx_builder_t *cb, crx_type_t *ret,
-                                         crx_type_t *args[], int nargs,
-                                         crx_trampoline1_fn trampl)
+static _crx_type_t *tst_get_function_type(_crx_builder_t *cb, _crx_type_t *ret,
+                                          _crx_type_t *args[], int nargs,
+                                          _crx_trampoline1_fn trampl)
 {
     int i;
     const char *prev = "FUNC( ";
@@ -115,66 +121,67 @@
     return newtype2(prev, " )");
 }
 
-static crx_type_t *tst_get_ellipsis_function_type(crx_builder_t *cb,
-                                                  crx_type_t *ret,
-                                                  crx_type_t *args[], int nargs)
+static _crx_type_t *tst_get_ellipsis_function_type(_crx_builder_t *cb,
+                                                   _crx_type_t *ret,
+                                                   _crx_type_t *args[],
+                                                   int nargs)
 {
-    crx_type_t *dotdotdot = newtype2("... -> ", ret->text);
+    _crx_type_t *dotdotdot = newtype2("... -> ", ret->text);
     return tst_get_function_type(cb, dotdotdot, args, nargs, 0);
 }
 
-static crx_type_t *tst_get_pointer_type(crx_builder_t *cb, crx_type_t *t)
+static _crx_type_t *tst_get_pointer_type(_crx_builder_t *cb, _crx_type_t *t)
 {
     return newtype2("PTR ", t->text);
 }
 
-static crx_type_t *tst_get_const_type(crx_builder_t *cb, crx_type_t *t)
+static _crx_type_t *tst_get_const_type(_crx_builder_t *cb, _crx_type_t *t)
 {
     return newtype2("CONST ", t->text);
 }
 
-static crx_type_t *tst_get_array_type(crx_builder_t *cb, crx_type_t *t,
-                                      size_t len)
+static _crx_type_t *tst_get_array_type(_crx_builder_t *cb, _crx_type_t *t,
+                                       size_t len)
 {
     char c[32];
     sprintf(c, "ARRAY[%zd] ", len);
     return newtype2(c, t->text);
 }
 
-static crx_type_t *tst_get_incomplete_array_type(crx_builder_t *cb,
-                                                 crx_type_t *t)
+static _crx_type_t *tst_get_incomplete_array_type(_crx_builder_t *cb,
+                                                  _crx_type_t *t)
 {
     return newtype2("ARRAY[] ", t->text);
 }
 
-static crx_type_t *tst_get_struct_type(crx_builder_t *cb, const char *name)
+static _crx_type_t *tst_get_struct_type(_crx_builder_t *cb, const char *name)
 {
     return newtype2("STRUCT ", name);
 }
 
-static crx_type_t *tst_get_union_type(crx_builder_t *cb, const char *name)
+static _crx_type_t *tst_get_union_type(_crx_builder_t *cb, const char *name)
 {
     return newtype2("UNION ", name);
 }
 
-static crx_type_t *tst_get_enum_type(crx_builder_t *cb, const char *name)
+static _crx_type_t *tst_get_enum_type(_crx_builder_t *cb, const char *name)
 {
     return newtype2("ENUM ", name);
 }
 
-static crx_type_t *tst_get_user_type(crx_builder_t *cb, const char *name)
+static _crx_type_t *tst_get_user_type(_crx_builder_t *cb, const char *name)
 {
     return newtype(name);
 }
 
-static crx_type_t *tst_get_unknown_type(crx_builder_t *cb, const char *name)
+static _crx_type_t *tst_get_unknown_type(_crx_builder_t *cb, const char *name)
 {
     return newtype2("UNKNOWN ", name);
 }
 
-static void tst_complete(crx_builder_t *cb, crx_type_t *t,
+static void tst_complete(_crx_builder_t *cb, _crx_type_t *t,
                          size_t sz, size_t align,
-                         crx_field_t fields[], int nfields)
+                         _crx_field_t fields[], int nfields)
 {
     int i;
     printf("%s:\n", t->text);
@@ -183,26 +190,26 @@
     }
 }
 
-static void tst_complete_enum(crx_builder_t *cb, crx_type_t *t,
-                              crx_type_t *inttype)
+static void tst_complete_enum(_crx_builder_t *cb, _crx_type_t *t,
+                              _crx_type_t *inttype)
 {
     abort();
 }
 
-static void tst_define_type(crx_builder_t *cb, const char *name, crx_type_t *t)
+static void tst_define_type(_crx_builder_t *cb, const char *name, _crx_type_t *t)
 {
     printf("TYPEDEF %s = %s\n", name, t->text);
 }
 
-static void tst_define_var(crx_builder_t *cb, const char *name, crx_type_t *t,
+static void tst_define_var(_crx_builder_t *cb, const char *name, _crx_type_t *t,
                            void *addr)
 {
     printf("VAR %s: %s\n", name, t->text);
 }
 
-static void tst_define_func(crx_builder_t *cb, const char *name,
-                            crx_type_t *ret, crx_type_t *args[], int nargs,
-                            crx_trampoline0_fn trampl, void *directcall)
+static void tst_define_func(_crx_builder_t *cb, const char *name,
+                            _crx_type_t *ret, _crx_type_t *args[], int nargs,
+                            _crx_trampoline0_fn trampl, void *directcall)
 {
     int i;
     printf("FUNC %s: ", name);
@@ -211,8 +218,8 @@
     printf("%s\n", ret->text);
 }
 
-static void tst_define_num_const(crx_builder_t *cb, const char *name,
-                                 crx_type_t *t, crx_num_const_t *value)
+static void tst_define_num_const(_crx_builder_t *cb, const char *name,
+                                 _crx_type_t *t, _crx_num_const_t *value)
 {
     printf("NUMCONST %s = %s ", name, t->text);
     if (strcmp(t->text, "int") == 0)
@@ -241,12 +248,12 @@
     }
 }
 
-static void tst_error(crx_builder_t *cb, const char *msg)
+static void tst_error(_crx_builder_t *cb, const char *msg)
 {
     printf("ERROR: %s\n", msg);
 }
 
-static crx_builder_t maincb = {
+_crx_builder_t creflect_debug_builder = {
     tst_get_void_type,
     tst_get_char_type,
     tst_get_bool_type,
diff --git a/creflect/creflect_debug_print.h b/creflect/creflect_debug_print.h
new file mode 100644
--- /dev/null
+++ b/creflect/creflect_debug_print.h
@@ -0,0 +1,11 @@
+#ifndef _CRX_DEBUG_PRINT_H_
+#define _CRX_DEBUG_PRINT_H_
+
+#include "creflect.h"
+
+
+extern _crx_builder_t creflect_debug_builder;
+const char *creflect_type_text(_crx_type_t *);
+
+
+#endif  /* _CRX_DEBUG_PRINT_H_ */
diff --git a/creflect/src/creflect_check.c b/creflect/creflect_libcheck.c
rename from creflect/src/creflect_check.c
rename to creflect/creflect_libcheck.c
diff --git a/creflect/driver.py b/creflect/driver.py
--- a/creflect/driver.py
+++ b/creflect/driver.py
@@ -84,7 +84,7 @@
 CALL_TEMPLATE1 = '''
 /***** CREFLECT main entry point *****/
 
-void %s(crx_builder_t *cb) {
+void %s(_crx_builder_t *cb) {
 %s}
 '''
 CALL_TEMPLATE2 = '''\
@@ -95,9 +95,9 @@
 
 DEBUG_TEMPLATE = '''\
 /***** CREFLECT debug code *****/
-#include "creflect_print.h"
+#include "creflect_debug_print.h"
 int main(void) {
-    %s(&maincb);
+    %s(&creflect_debug_builder);
     return 0;
 }
 '''
diff --git a/creflect/model.py b/creflect/model.py
--- a/creflect/model.py
+++ b/creflect/model.py
@@ -120,7 +120,7 @@
                 hint = self.name.split()
                 if hint[0] in ('signed', 'unsigned'):
                     hint = hint[1:]
-                expr = 'CRX_INT_TYPE(cb, %s, "%s")' % (star_p1, " ".join(hint))
+                expr = '_CRX_INT_TYPE(cb, %s, "%s")' % (star_p1, " ".join(hint))
             elif self.is_char_type():
                 errmsg = "numeric type '%s' is not a char" % (
                     inspect.get_comment_type(0, False),)
@@ -163,7 +163,7 @@
         return block.write_crx_type_var(expr)
 
     def write_num_const_decl(self, block, varname):
-        block.writedecl("crx_num_const_t v;")
+        block.writedecl("_crx_num_const_t v;")
         comment = "an integer"
         op = "<< 1"
         if self.is_char_type():
@@ -181,12 +181,12 @@
                     preference = 2
             else:
                 preference = 1
-            expr = "CRX_INT_CONST(cb, %s, &v, %d)" % (varname, preference)
+            expr = "_CRX_INT_CONST(cb, %s, &v, %d)" % (varname, preference)
         elif self.is_char_type():
-            block.writeline('CRX_CHAR_CONST(cb, %s, &v);' % varname)
+            block.writeline('_CRX_CHAR_CONST(cb, %s, &v);' % varname)
             expr = 'cb->get_char_type(cb)'
         elif self.is_float_type():
-            expr = "CRX_FLOAT_CONST(cb, %s, &v)" % (varname,)
+            expr = "_CRX_FLOAT_CONST(cb, %s, &v)" % (varname,)
         else:
             raise AssertionError
         t1 = block.write_crx_type_var(expr)
diff --git a/test/codegen/001.c b/test/codegen/001.c
--- a/test/codegen/001.c
+++ b/test/codegen/001.c
@@ -2,6 +2,6 @@
 
 # ____________________________________________________________
 
-void test001(crx_builder_t *cb)
+void test001(_crx_builder_t *cb)
 {
 }
diff --git a/test/codegen/002.c b/test/codegen/002.c
--- a/test/codegen/002.c
+++ b/test/codegen/002.c
@@ -2,9 +2,9 @@
 
 # ____________________________________________________________
 
-void test002(crx_builder_t *cb)
+void test002(_crx_builder_t *cb)
 {
-    crx_type_t *t1, *t2;
+    _crx_type_t *t1, *t2;
     {
         num_t *p1;
         char *p2;
diff --git a/test/codegen/002b.c b/test/codegen/002b.c
--- a/test/codegen/002b.c
+++ b/test/codegen/002b.c
@@ -2,9 +2,9 @@
 
 # ____________________________________________________________
 
-void test002b(crx_builder_t *cb)
+void test002b(_crx_builder_t *cb)
 {
-    crx_type_t *t1, *t2, *t3, *t4;
+    _crx_type_t *t1, *t2, *t3, *t4;
     {
         num_t *p1;
         char *p2;
diff --git a/test/codegen/002c.c b/test/codegen/002c.c
--- a/test/codegen/002c.c
+++ b/test/codegen/002c.c
@@ -2,9 +2,9 @@
 
 # ____________________________________________________________
 
-void test002c(crx_builder_t *cb)
+void test002c(_crx_builder_t *cb)
 {
-    crx_type_t *t1, *t2, *t3;
+    _crx_type_t *t1, *t2, *t3;
     {
         num_t *p1;
         char *p2;
diff --git a/test/codegen/003.c b/test/codegen/003.c
--- a/test/codegen/003.c
+++ b/test/codegen/003.c
@@ -2,16 +2,16 @@
 
 # ____________________________________________________________
 
-void test003(crx_builder_t *cb)
+void test003(_crx_builder_t *cb)
 {
-    crx_type_t *t1;
+    _crx_type_t *t1;
     {
         num_t *p1;
         char b[sizeof(*p1)];
         p1 = (void *)b;
         (void)(*p1 << 1);  /* check that 'num_t' is an integer type */
         *p1 = -1;  /* check that 'num_t' is not declared 'const' */
-        t1 = CRX_INT_TYPE(cb, *p1, "int");
+        t1 = _CRX_INT_TYPE(cb, *p1, "int");
         cb->define_type(cb, "num_t", t1);
 #expect TYPEDEF num_t = int
     }
diff --git a/test/codegen/003b.c b/test/codegen/003b.c
--- a/test/codegen/003b.c
+++ b/test/codegen/003b.c
@@ -2,16 +2,16 @@
 
 # ____________________________________________________________
 
-void test003b(crx_builder_t *cb)
+void test003b(_crx_builder_t *cb)
 {
-    crx_type_t *t1;
+    _crx_type_t *t1;
     {
         num_t *p1;
         char b[sizeof(*p1)];
         p1 = (void *)b;
         (void)(*p1 << 1);  /* check that 'num_t' is an integer type */
         *p1 = -1;  /* check that 'num_t' is not declared 'const' */
-        t1 = CRX_INT_TYPE(cb, *p1, "long");
+        t1 = _CRX_INT_TYPE(cb, *p1, "long");
         cb->define_type(cb, "num_t", t1);
 #expect TYPEDEF num_t = long
     }
diff --git a/test/codegen/003c.c b/test/codegen/003c.c
--- a/test/codegen/003c.c
+++ b/test/codegen/003c.c
@@ -2,16 +2,16 @@
 
 # ____________________________________________________________
 
-void test003c(crx_builder_t *cb)
+void test003c(_crx_builder_t *cb)
 {
-    crx_type_t *t1;
+    _crx_type_t *t1;
     {
         num_t *p1;
         char b[sizeof(*p1)];
         p1 = (void *)b;
         (void)(*p1 << 1);  /* check that 'num_t' is an integer type */
         *p1 = -1;  /* check that 'num_t' is not declared 'const' */
-        t1 = CRX_INT_TYPE(cb, *p1, "long long");
+        t1 = _CRX_INT_TYPE(cb, *p1, "long long");
         cb->define_type(cb, "num_t", t1);
 #expect TYPEDEF num_t = long long
     }
diff --git a/test/codegen/003d.c b/test/codegen/003d.c
--- a/test/codegen/003d.c
+++ b/test/codegen/003d.c
@@ -2,16 +2,16 @@
 
 # ____________________________________________________________
 
-void test003d(crx_builder_t *cb)
+void test003d(_crx_builder_t *cb)
 {
-    crx_type_t *t1;
+    _crx_type_t *t1;
     {
         num_t *p1;
         char b[sizeof(*p1)];
         p1 = (void *)b;
         (void)(*p1 << 1);  /* check that 'num_t' is an integer type */
         *p1 = -1;  /* check that 'num_t' is not declared 'const' */
-        t1 = CRX_INT_TYPE(cb, *p1, "char");
+        t1 = _CRX_INT_TYPE(cb, *p1, "char");
         cb->define_type(cb, "num_t", t1);
 #expect TYPEDEF num_t = signed char
     }
diff --git a/test/codegen/003e.c b/test/codegen/003e.c
--- a/test/codegen/003e.c
+++ b/test/codegen/003e.c
@@ -2,9 +2,9 @@
 
 # ____________________________________________________________
 
-void test003e(crx_builder_t *cb)
+void test003e(_crx_builder_t *cb)
 {
-    crx_type_t *t1;
+    _crx_type_t *t1;
     {
         num_t *p1;
         char b[sizeof(*p1)];
diff --git a/test/codegen/003f.c b/test/codegen/003f.c
--- a/test/codegen/003f.c
+++ b/test/codegen/003f.c
@@ -2,16 +2,16 @@
 
 # ____________________________________________________________
 
-void test003f(crx_builder_t *cb)
+void test003f(_crx_builder_t *cb)
 {
-    crx_type_t *t1;
+    _crx_type_t *t1;
     {
         num_t *p1;
         char b[sizeof(*p1)];
         p1 = (void *)b;
         (void)(*p1 << 1);  /* check that 'num_t' is an integer type */
         *p1 = -1;  /* check that 'num_t' is not declared 'const' */
-        t1 = CRX_INT_TYPE(cb, *p1, "long long");
+        t1 = _CRX_INT_TYPE(cb, *p1, "long long");
         cb->define_type(cb, "num_t", t1);
 #expect TYPEDEF num_t = unsigned long long
     }
diff --git a/test/codegen/003g.c b/test/codegen/003g.c
--- a/test/codegen/003g.c
+++ b/test/codegen/003g.c
@@ -2,9 +2,9 @@
 
 # ____________________________________________________________
 
-void test003g(crx_builder_t *cb)
+void test003g(_crx_builder_t *cb)
 {
-    crx_type_t *t1;
+    _crx_type_t *t1;
     {
         num_t *p1;
         char b[sizeof(*p1)];
diff --git a/test/codegen/003h.c b/test/codegen/003h.c
--- a/test/codegen/003h.c
+++ b/test/codegen/003h.c
@@ -6,16 +6,16 @@
 
 # ____________________________________________________________
 
-void test003h(crx_builder_t *cb)
+void test003h(_crx_builder_t *cb)
 {
-    crx_type_t *t1;
+    _crx_type_t *t1;
     {
         num_t *p1;
         char b[sizeof(*p1)];
         p1 = (void *)b;
         (void)(*p1 << 1);  /* check that 'num_t' is an integer type */
         *p1 = -1;  /* check that 'num_t' is not declared 'const' */
-        t1 = CRX_INT_TYPE(cb, *p1, "long long");
+        t1 = _CRX_INT_TYPE(cb, *p1, "long long");
         cb->define_type(cb, "num_t", t1);
 #expect TYPEDEF num_t = int
     }
diff --git a/test/codegen/003i.c b/test/codegen/003i.c
--- a/test/codegen/003i.c
+++ b/test/codegen/003i.c
@@ -2,16 +2,16 @@
 
 # ____________________________________________________________
 
-void test003i(crx_builder_t *cb)
+void test003i(_crx_builder_t *cb)
 {
-    crx_type_t *t1;
+    _crx_type_t *t1;
     {
         num_t *p1;
         char b[sizeof(*p1)];
         p1 = (void *)b;
         (void)(*p1 << 1);  /* check that 'num_t' is an integer type */
         *p1 = -1;  /* check that 'num_t' is not declared 'const' */
-        t1 = CRX_INT_TYPE(cb, *p1, "char");
+        t1 = _CRX_INT_TYPE(cb, *p1, "char");
         cb->define_type(cb, "num_t", t1);
 #expect TYPEDEF num_t = signed char
     }
diff --git a/test/codegen/004.c b/test/codegen/004.c
--- a/test/codegen/004.c
+++ b/test/codegen/004.c
@@ -2,9 +2,9 @@
 
 # ____________________________________________________________
 
-void test004(crx_builder_t *cb)
+void test004(_crx_builder_t *cb)
 {
-    crx_type_t *t1, *t2;
+    _crx_type_t *t1, *t2;
     {
         num_t *p1;
         char *p2;
@@ -13,7 +13,7 @@
         *p1 = (void *)b;    /* check that 'num_t' is a pointer type */
         (void)(**p1 << 1);  /* check that '*num_t' is an integer type */
         **p1 = -1;  /* check that '*num_t' is not declared 'const' */
-        t1 = CRX_INT_TYPE(cb, **p1, "int");
+        t1 = _CRX_INT_TYPE(cb, **p1, "int");
         t2 = cb->get_pointer_type(cb, t1);
         cb->define_type(cb, "num_t", t2);
 #expect TYPEDEF num_t = PTR int
diff --git a/test/codegen/004b.c b/test/codegen/004b.c
--- a/test/codegen/004b.c
+++ b/test/codegen/004b.c
@@ -2,9 +2,9 @@
 
 # ____________________________________________________________
 
-void test004b(crx_builder_t *cb)
+void test004b(_crx_builder_t *cb)
 {
-    crx_type_t *t1, *t2, *t3;
+    _crx_type_t *t1, *t2, *t3;
     {
         num_t *p1;
         char *p2;
@@ -15,7 +15,7 @@
         **p1 = (void *)b;    /* check that '*num_t' is a pointer type */
         (void)(***p1 << 1);  /* check that '**num_t' is an integer type */
         ***p1 = -1;  /* check that '**num_t' is not declared 'const' */
-        t1 = CRX_INT_TYPE(cb, ***p1, "int");
+        t1 = _CRX_INT_TYPE(cb, ***p1, "int");
         t2 = cb->get_pointer_type(cb, t1);
         t3 = cb->get_pointer_type(cb, t2);
         cb->define_type(cb, "num_t", t3);
diff --git a/test/codegen/005.c b/test/codegen/005.c
--- a/test/codegen/005.c
+++ b/test/codegen/005.c
@@ -2,9 +2,9 @@
 
 # ____________________________________________________________
 
-void test005(crx_builder_t *cb)
+void test005(_crx_builder_t *cb)
 {
-    crx_type_t *t1, *t2;
+    _crx_type_t *t1, *t2;
     {
         foo_t *p1;
         char b[sizeof(**p1)];  /* check that 'foo_t[]' is a valid type */
@@ -15,7 +15,7 @@
         }
         (void)(**p1 << 1);  /* check that 'foo_t[]' is an integer type */
         **p1 = -1;  /* check that 'foo_t[]' is not declared 'const' */
-        t1 = CRX_INT_TYPE(cb, **p1, "int");
+        t1 = _CRX_INT_TYPE(cb, **p1, "int");
         t2 = cb->get_array_type(cb, t1, sizeof(*p1) / sizeof(**p1));
         cb->define_type(cb, "foo_t", t2);
 #expect TYPEDEF foo_t = ARRAY[27] int
diff --git a/test/codegen/005b.c b/test/codegen/005b.c
--- a/test/codegen/005b.c
+++ b/test/codegen/005b.c
@@ -2,9 +2,9 @@
 
 # ____________________________________________________________
 
-void test005b(crx_builder_t *cb)
+void test005b(_crx_builder_t *cb)
 {
-    crx_type_t *t1, *t2, *t3;
+    _crx_type_t *t1, *t2, *t3;
     {
         foo_t *p1;
         char *p3;
@@ -17,7 +17,7 @@
         **p1 = (void *)b;    /* check that 'foo_t[]' is a pointer type */
         (void)(***p1 << 1);  /* check that '*foo_t[]' is an integer type */
         ***p1 = -1;  /* check that '*foo_t[]' is not declared 'const' */
-        t1 = CRX_INT_TYPE(cb, ***p1, "int");
+        t1 = _CRX_INT_TYPE(cb, ***p1, "int");
         t2 = cb->get_pointer_type(cb, t1);
         t3 = cb->get_array_type(cb, t2, sizeof(*p1) / sizeof(**p1));
         cb->define_type(cb, "foo_t", t3);
diff --git a/test/codegen/005c.c b/test/codegen/005c.c
--- a/test/codegen/005c.c
+++ b/test/codegen/005c.c
@@ -2,9 +2,9 @@
 
 # ____________________________________________________________
 
-void test005c(crx_builder_t *cb)
+void test005c(_crx_builder_t *cb)
 {
-    crx_type_t *t1, *t2, *t3;
+    _crx_type_t *t1, *t2, *t3;
     {
         foo_t *p1;
         char *p2;
@@ -17,7 +17,7 @@
         }
         (void)(***p1 << 1);  /* check that '(*foo_t)[]' is an integer type */
         ***p1 = -1;  /* check that '(*foo_t)[]' is not declared 'const' */
-        t1 = CRX_INT_TYPE(cb, ***p1, "int");
+        t1 = _CRX_INT_TYPE(cb, ***p1, "int");
         t2 = cb->get_array_type(cb, t1, sizeof(**p1) / sizeof(***p1));
         t3 = cb->get_pointer_type(cb, t2);
         cb->define_type(cb, "foo_t", t3);
diff --git a/test/codegen/005d.c b/test/codegen/005d.c
--- a/test/codegen/005d.c
+++ b/test/codegen/005d.c
@@ -2,9 +2,9 @@
 
 # ____________________________________________________________
 
-void test005d(crx_builder_t *cb)
+void test005d(_crx_builder_t *cb)
 {
-    crx_type_t *t1, *t2, *t3, *t4, *t5, *t6, *t7, *t8, *t9, *t10;
+    _crx_type_t *t1, *t2, *t3, *t4, *t5, *t6, *t7, *t8, *t9, *t10;
     {
         foo_t *p1;
         char *p4;
@@ -37,7 +37,7 @@
         *********p1 = (void *)b;    /* check that '*(***foo_t[][])[][]' is a pointer type */
         (void)(**********p1 << 1);  /* check that '**(***foo_t[][])[][]' is an integer type */
         **********p1 = -1;  /* check that '**(***foo_t[][])[][]' is not declared 'const' */
-        t1 = CRX_INT_TYPE(cb, **********p1, "int");
+        t1 = _CRX_INT_TYPE(cb, **********p1, "int");
         t2 = cb->get_pointer_type(cb, t1);
         t3 = cb->get_pointer_type(cb, t2);
         t4 = cb->get_array_type(cb, t3, sizeof(*******p1) / sizeof(********p1));
diff --git a/test/codegen/006.c b/test/codegen/006.c
--- a/test/codegen/006.c
+++ b/test/codegen/006.c
@@ -2,16 +2,16 @@
 
 # ____________________________________________________________
 
-void test006(crx_builder_t *cb)
+void test006(_crx_builder_t *cb)
 {
-    crx_type_t *t1, *t2;
+    _crx_type_t *t1, *t2;
     {
         num_t *p1;
         char b[sizeof(*p1)];
         memset(b, -1, sizeof(b));
         p1 = (void *)b;
         (void)(*p1 << 1);  /* check that 'num_t' is an integer type */
-        t1 = CRX_INT_TYPE(cb, *p1, "int");
+        t1 = _CRX_INT_TYPE(cb, *p1, "int");
         t2 = cb->get_const_type(cb, t1);
         cb->define_type(cb, "num_t", t2);
 #expect TYPEDEF num_t = CONST unsigned int
diff --git a/test/codegen/006b.c b/test/codegen/006b.c
--- a/test/codegen/006b.c
+++ b/test/codegen/006b.c
@@ -2,9 +2,9 @@
 
 # ____________________________________________________________
 
-void test006b(crx_builder_t *cb)
+void test006b(_crx_builder_t *cb)
 {
-    crx_type_t *t1, *t2, *t3;
+    _crx_type_t *t1, *t2, *t3;
     {
         num_t *p1;
         char *p2;
@@ -17,7 +17,7 @@
         }
         (void)(**p1 << 1);  /* check that '*num_t' is an integer type */
         **p1 = -1;  /* check that '*num_t' is not declared 'const' */
-        t1 = CRX_INT_TYPE(cb, **p1, "int");
+        t1 = _CRX_INT_TYPE(cb, **p1, "int");
         t2 = cb->get_pointer_type(cb, t1);
         t3 = cb->get_const_type(cb, t2);
         cb->define_type(cb, "num_t", t3);
diff --git a/test/codegen/006c.c b/test/codegen/006c.c
--- a/test/codegen/006c.c
+++ b/test/codegen/006c.c
@@ -2,9 +2,9 @@
 
 # ____________________________________________________________
 
-void test006c(crx_builder_t *cb)
+void test006c(_crx_builder_t *cb)
 {
-    crx_type_t *t1, *t2, *t3, *t4;
+    _crx_type_t *t1, *t2, *t3, *t4;
     {
         foo_t *p1;
         char *p2;
@@ -21,7 +21,7 @@
         }
         (void)(***p1 << 1);  /* check that '(*foo_t)[]' is an integer type */
         ***p1 = -1;  /* check that '(*foo_t)[]' is not declared 'const' */
-        t1 = CRX_INT_TYPE(cb, ***p1, "int");
+        t1 = _CRX_INT_TYPE(cb, ***p1, "int");
         t2 = cb->get_array_type(cb, t1, sizeof(**p1) / sizeof(***p1));
         t3 = cb->get_pointer_type(cb, t2);
         t4 = cb->get_const_type(cb, t3);
diff --git a/test/codegen/007.c b/test/codegen/007.c
--- a/test/codegen/007.c
+++ b/test/codegen/007.c
@@ -2,9 +2,9 @@
 
 # ____________________________________________________________
 
-void test007(crx_builder_t *cb)
+void test007(_crx_builder_t *cb)
 {
-    crx_type_t *t1;
+    _crx_type_t *t1;
     {
         t1 = cb->get_unknown_type(cb, "$num_t");
         cb->define_type(cb, "num_t", t1);
diff --git a/test/codegen/func-001.c b/test/codegen/func-001.c
--- a/test/codegen/func-001.c
+++ b/test/codegen/func-001.c
@@ -14,9 +14,9 @@
     return f();
 }
 
-void testfunc_001(crx_builder_t *cb)
+void testfunc_001(_crx_builder_t *cb)
 {
-    crx_type_t *t1;
+    _crx_type_t *t1;
     {
         t1 = cb->get_signed_type(cb, sizeof(int), "int");
         cb->define_func(cb, "f", t1, 0, 0, &testfunc_001__c_f, &testfunc_001__d_f);
diff --git a/test/codegen/func-001b.c b/test/codegen/func-001b.c
--- a/test/codegen/func-001b.c
+++ b/test/codegen/func-001b.c
@@ -34,9 +34,9 @@
     return h(a0, a1);
 }
 
-void testfunc_001b(crx_builder_t *cb)
+void testfunc_001b(_crx_builder_t *cb)
 {
-    crx_type_t *t1, *t2, *a3[1], *t4, *a5[2], *a6[2];
+    _crx_type_t *t1, *t2, *a3[1], *t4, *a5[2], *a6[2];
     {
         t1 = cb->get_unsigned_type(cb, sizeof(unsigned int), "unsigned int");
         t2 = cb->get_signed_type(cb, sizeof(long), "long");
diff --git a/test/codegen/func-001c.c b/test/codegen/func-001c.c
--- a/test/codegen/func-001c.c
+++ b/test/codegen/func-001c.c
@@ -14,9 +14,9 @@
     return f(a0);
 }
 
-void testfunc_001c(crx_builder_t *cb)
+void testfunc_001c(_crx_builder_t *cb)
 {
-    crx_type_t *t1, *t2, *a3[1];
+    _crx_type_t *t1, *t2, *a3[1];
     {
         t1 = cb->get_float_type(cb, sizeof(double), "double");
         t2 = cb->get_float_type(cb, sizeof(float), "float");
diff --git a/test/codegen/func-002.c b/test/codegen/func-002.c
--- a/test/codegen/func-002.c
+++ b/test/codegen/func-002.c
@@ -7,9 +7,9 @@
 
 # ____________________________________________________________
 
-void testfunc_002(crx_builder_t *cb)
+void testfunc_002(_crx_builder_t *cb)
 {
-    crx_type_t *t1, *a2[1], *t3;
+    _crx_type_t *t1, *a2[1], *t3;
     {
         int (*p1)(int, ...) = &f;  /* check that 'f' is a function with exactly the given signature */
         (void)p1;
diff --git a/test/codegen/func-003.c b/test/codegen/func-003.c
--- a/test/codegen/func-003.c
+++ b/test/codegen/func-003.c
@@ -7,9 +7,9 @@
     *(int *)result = f(*(long *)args[0], *(long *)args[1], *(int *)args[2]);
 }
 
-void testfunc_003(crx_builder_t *cb)
+void testfunc_003(_crx_builder_t *cb)
 {
-    crx_type_t *t1, *t2, *a3[3], *t4, *t5;
+    _crx_type_t *t1, *t2, *a3[3], *t4, *t5;
     {
         func_t *p1;
         char *p2;
diff --git a/test/codegen/func-003b.c b/test/codegen/func-003b.c
--- a/test/codegen/func-003b.c
+++ b/test/codegen/func-003b.c
@@ -2,9 +2,9 @@
 
 # ____________________________________________________________
 
-void testfunc_003b(crx_builder_t *cb)
+void testfunc_003b(_crx_builder_t *cb)
 {
-    crx_type_t *t1, *t2, *a3[1], *t4, *t5;
+    _crx_type_t *t1, *t2, *a3[1], *t4, *t5;
     {
         func_t *p1;
         char *p2;
diff --git a/test/codegen/func-004.c b/test/codegen/func-004.c
--- a/test/codegen/func-004.c
+++ b/test/codegen/func-004.c
@@ -7,9 +7,9 @@
     *(int *)result = f(*(long *)args[0], *(int *)args[1]);
 }
 
-void testfunc_004(crx_builder_t *cb)
+void testfunc_004(_crx_builder_t *cb)
 {
-    crx_type_t *t1, *t2, *a3[2], *t4, *t5, *t6;
+    _crx_type_t *t1, *t2, *a3[2], *t4, *t5, *t6;
     {
         func_t *p1;
         char *p2;
diff --git a/test/codegen/func-005.c b/test/codegen/func-005.c
--- a/test/codegen/func-005.c
+++ b/test/codegen/func-005.c
@@ -19,9 +19,9 @@
     *(long *)result = f(*(char *)args[0], *(short *)args[1]);
 }
 
-void testfunc_005(crx_builder_t *cb)
+void testfunc_005(_crx_builder_t *cb)
 {
-    crx_type_t *t1, *t2, *t3, *t4, *t5, *t6, *a7[2], *t8, *t9, *a10[2];
+    _crx_type_t *t1, *t2, *t3, *t4, *t5, *t6, *a7[2], *t8, *t9, *a10[2];
     {
         t1 = cb->get_void_type(cb);
         t2 = cb->get_signed_type(cb, sizeof(int), "int");
diff --git a/test/codegen/glob-001.c b/test/codegen/glob-001.c
--- a/test/codegen/glob-001.c
+++ b/test/codegen/glob-001.c
@@ -2,9 +2,9 @@
 
 # ____________________________________________________________
 
-void testglob_001(crx_builder_t *cb)
+void testglob_001(_crx_builder_t *cb)
 {
-    crx_type_t *t1, *t2;
+    _crx_type_t *t1, *t2;
     {
         void *p1 = someglob;  /* check that 'someglob' is a pointer */
         if (0) { someglob = p1; }  /* check that 'someglob' is a pointer variable, and not an array or a constant */
diff --git a/test/codegen/glob-002.c b/test/codegen/glob-002.c
--- a/test/codegen/glob-002.c
+++ b/test/codegen/glob-002.c
@@ -2,9 +2,9 @@
 
 # ____________________________________________________________
 
-void testglob_002(crx_builder_t *cb)
+void testglob_002(_crx_builder_t *cb)
 {
-    crx_type_t *t1, *t2;
+    _crx_type_t *t1, *t2;
     {
         char (*p1)[] = &someglob;  /* check that 'someglob' is of type 'char[]' */
         (void)p1;
diff --git a/test/codegen/glob-002b.c b/test/codegen/glob-002b.c
--- a/test/codegen/glob-002b.c
+++ b/test/codegen/glob-002b.c
@@ -4,9 +4,9 @@
 
 # ____________________________________________________________
 
-void testglob_002b(crx_builder_t *cb)
+void testglob_002b(_crx_builder_t *cb)
 {
-    crx_type_t *t1, *t2;
+    _crx_type_t *t1, *t2;
     {
         int (*p1)[] = &someglob;  /* check that 'someglob' is of type 'int[]' */
         (void)p1;
diff --git a/test/codegen/glob-002c.c b/test/codegen/glob-002c.c
--- a/test/codegen/glob-002c.c
+++ b/test/codegen/glob-002c.c
@@ -4,9 +4,9 @@
 
 # ____________________________________________________________
 
-void testglob_002c(crx_builder_t *cb)
+void testglob_002c(_crx_builder_t *cb)
 {
-    crx_type_t *t1;
+    _crx_type_t *t1;
     {
         int *p1 = &someglob;  /* check that 'someglob' is of type 'int' */
         (void)p1;
diff --git a/test/codegen/glob-002d.c b/test/codegen/glob-002d.c
--- a/test/codegen/glob-002d.c
+++ b/test/codegen/glob-002d.c
@@ -2,9 +2,9 @@
 
 # ____________________________________________________________
 
-void testglob_002d(crx_builder_t *cb)
+void testglob_002d(_crx_builder_t *cb)
 {
-    crx_type_t *t1, *t2;
+    _crx_type_t *t1, *t2;
     {
         int **p1 = &someglob;  /* check that 'someglob' is of type 'int *' */
         (void)p1;
diff --git a/test/codegen/glob-003.c b/test/codegen/glob-003.c
--- a/test/codegen/glob-003.c
+++ b/test/codegen/glob-003.c
@@ -2,13 +2,13 @@
 
 # ____________________________________________________________
 
-void testglob_003(crx_builder_t *cb)
+void testglob_003(_crx_builder_t *cb)
 {
-    crx_type_t *t1;
+    _crx_type_t *t1;
     {
-        crx_num_const_t v;
+        _crx_num_const_t v;
         (void)((ab) << 1);  /* check that 'ab' is an integer */
-        t1 = CRX_INT_CONST(cb, ab, &v, 1);
+        t1 = _CRX_INT_CONST(cb, ab, &v, 1);
         cb->define_num_const(cb, "ab", t1, &v);
 #expect NUMCONST ab = int -42
     }
diff --git a/test/codegen/glob-003b.c b/test/codegen/glob-003b.c
--- a/test/codegen/glob-003b.c
+++ b/test/codegen/glob-003b.c
@@ -6,13 +6,13 @@
 
 # ____________________________________________________________
 
-void testglob_003b(crx_builder_t *cb)
+void testglob_003b(_crx_builder_t *cb)
 {
-    crx_type_t *t1;
+    _crx_type_t *t1;
     {
-        crx_num_const_t v;
+        _crx_num_const_t v;
         (void)((ab) << 1);  /* check that 'ab' is an integer */
-        t1 = CRX_INT_CONST(cb, ab, &v, 1);
+        t1 = _CRX_INT_CONST(cb, ab, &v, 1);
         cb->define_num_const(cb, "ab", t1, &v);
 #expect NUMCONST ab = int 42
     }
diff --git a/test/codegen/glob-003c.c b/test/codegen/glob-003c.c
--- a/test/codegen/glob-003c.c
+++ b/test/codegen/glob-003c.c
@@ -2,13 +2,13 @@
 
 # ____________________________________________________________
 
-void testglob_003c(crx_builder_t *cb)
+void testglob_003c(_crx_builder_t *cb)
 {
-    crx_type_t *t1;
+    _crx_type_t *t1;
     {
-        crx_num_const_t v;
+        _crx_num_const_t v;
         (void)((ab) << 1);  /* check that 'ab' is an integer */
-        t1 = CRX_INT_CONST(cb, ab, &v, 1);
+        t1 = _CRX_INT_CONST(cb, ab, &v, 1);
         cb->define_num_const(cb, "ab", t1, &v);
 #expect NUMCONST ab = unsigned int 42
     }
diff --git a/test/codegen/glob-003d.c b/test/codegen/glob-003d.c
--- a/test/codegen/glob-003d.c
+++ b/test/codegen/glob-003d.c
@@ -2,13 +2,13 @@
 
 # ____________________________________________________________
 
-void testglob_003d(crx_builder_t *cb)
+void testglob_003d(_crx_builder_t *cb)
 {
-    crx_type_t *t1;
+    _crx_type_t *t1;
     {
-        crx_num_const_t v;
+        _crx_num_const_t v;
         (void)((ab) << 1);  /* check that 'ab' is an integer or char */
-        CRX_CHAR_CONST(cb, ab, &v);
+        _CRX_CHAR_CONST(cb, ab, &v);
         t1 = cb->get_char_type(cb);
         cb->define_num_const(cb, "ab", t1, &v);
 #expect NUMCONST ab = char '*'
diff --git a/test/codegen/glob-003e.c b/test/codegen/glob-003e.c
--- a/test/codegen/glob-003e.c
+++ b/test/codegen/glob-003e.c
@@ -2,13 +2,13 @@
 
 # ____________________________________________________________
 
-void testglob_003e(crx_builder_t *cb)
+void testglob_003e(_crx_builder_t *cb)
 {
-    crx_type_t *t1;
+    _crx_type_t *t1;
     {
-        crx_num_const_t v;
+        _crx_num_const_t v;
         (void)((ab) << 1);  /* check that 'ab' is an integer */
-        t1 = CRX_INT_CONST(cb, ab, &v, 2);
+        t1 = _CRX_INT_CONST(cb, ab, &v, 2);
         cb->define_num_const(cb, "ab", t1, &v);
 #expect NUMCONST ab = long -42
     }
diff --git a/test/codegen/glob-003f.c b/test/codegen/glob-003f.c
--- a/test/codegen/glob-003f.c
+++ b/test/codegen/glob-003f.c
@@ -2,13 +2,13 @@
 
 # ____________________________________________________________
 
-void testglob_003f(crx_builder_t *cb)
+void testglob_003f(_crx_builder_t *cb)
 {
-    crx_type_t *t1;
+    _crx_type_t *t1;
     {
-        crx_num_const_t v;
+        _crx_num_const_t v;
         (void)((ab) << 1);  /* check that 'ab' is an integer */
-        t1 = CRX_INT_CONST(cb, ab, &v, 3);
+        t1 = _CRX_INT_CONST(cb, ab, &v, 3);
         cb->define_num_const(cb, "ab", t1, &v);
 #expect NUMCONST ab = long long -42
     }
diff --git a/test/codegen/glob-003g.c b/test/codegen/glob-003g.c
--- a/test/codegen/glob-003g.c
+++ b/test/codegen/glob-003g.c
@@ -2,13 +2,13 @@
 
 # ____________________________________________________________
 
-void testglob_003g(crx_builder_t *cb)
+void testglob_003g(_crx_builder_t *cb)
 {
-    crx_type_t *t1;
+    _crx_type_t *t1;
     {
-        crx_num_const_t v;
+        _crx_num_const_t v;
         (void)((ab) << 1);  /* check that 'ab' is an integer */
-        t1 = CRX_INT_CONST(cb, ab, &v, 1);
+        t1 = _CRX_INT_CONST(cb, ab, &v, 1);
         cb->define_num_const(cb, "ab", t1, &v);
 #expect NUMCONST ab = int -42
     }
diff --git a/test/codegen/glob-003h.c b/test/codegen/glob-003h.c
--- a/test/codegen/glob-003h.c
+++ b/test/codegen/glob-003h.c
@@ -3,20 +3,20 @@
 
 # ____________________________________________________________
 
-void testglob_003h(crx_builder_t *cb)
+void testglob_003h(_crx_builder_t *cb)
 {
-    crx_type_t *t1, *t2;
+    _crx_type_t *t1, *t2;
     {
-        crx_num_const_t v;
+        _crx_num_const_t v;
         (void)((ab) / 2.0);  /* check that 'ab' is a number */
-        t1 = CRX_FLOAT_CONST(cb, ab, &v);
+        t1 = _CRX_FLOAT_CONST(cb, ab, &v);
         cb->define_num_const(cb, "ab", t1, &v);
 #expect NUMCONST ab = float 42.000000
     }
     {
-        crx_num_const_t v;
+        _crx_num_const_t v;
         (void)((abd) / 2.0);  /* check that 'abd' is a number */
-        t2 = CRX_FLOAT_CONST(cb, abd, &v);
+        t2 = _CRX_FLOAT_CONST(cb, abd, &v);
         cb->define_num_const(cb, "abd", t2, &v);
 #expect NUMCONST abd = double 42.000000
     }
diff --git a/test/codegen/glob-003i.c b/test/codegen/glob-003i.c
--- a/test/codegen/glob-003i.c
+++ b/test/codegen/glob-003i.c
@@ -2,13 +2,13 @@
 
 # ____________________________________________________________
 
-void testglob_003i(crx_builder_t *cb)
+void testglob_003i(_crx_builder_t *cb)
 {
-    crx_type_t *t1;
+    _crx_type_t *t1;
     {
-        crx_num_const_t v;
+        _crx_num_const_t v;
         (void)((abl) / 2.0);  /* check that 'abl' is a number */
-        t1 = CRX_FLOAT_CONST(cb, abl, &v);
+        t1 = _CRX_FLOAT_CONST(cb, abl, &v);
         cb->define_num_const(cb, "abl", t1, &v);
 #expect NUMCONST abl = long double 42.000000
     }
diff --git a/test/codegen/glob-003j.c b/test/codegen/glob-003j.c
--- a/test/codegen/glob-003j.c
+++ b/test/codegen/glob-003j.c
@@ -2,13 +2,13 @@
 
 # ____________________________________________________________
 
-void testglob_003j(crx_builder_t *cb)
+void testglob_003j(_crx_builder_t *cb)
 {
-    crx_type_t *t1;
+    _crx_type_t *t1;
     {
-        crx_num_const_t v;
+        _crx_num_const_t v;
         (void)((ab) << 1);  /* check that 'ab' is an integer */
-        t1 = CRX_INT_CONST(cb, ab, &v, 1);
+        t1 = _CRX_INT_CONST(cb, ab, &v, 1);
         cb->define_num_const(cb, "ab", t1, &v);
 #expect NUMCONST ab = int 42
     }
diff --git a/test/codegen/glob-004.c b/test/codegen/glob-004.c
--- a/test/codegen/glob-004.c
+++ b/test/codegen/glob-004.c
@@ -7,9 +7,9 @@
     *(int *)result = f(*(long *)args[0], *(long *)args[1]);
 }
 
-void testglob_004(crx_builder_t *cb)
+void testglob_004(_crx_builder_t *cb)
 {
-    crx_type_t *t1, *t2, *a3[2], *t4, *t5;
+    _crx_type_t *t1, *t2, *a3[2], *t4, *t5;
     {
         int (**p1)(long, long) = &someglob;  /* check that 'someglob' is of type 'int (*)(long, long)' */
         (void)p1;
diff --git a/test/codegen/glob-005.c b/test/codegen/glob-005.c
--- a/test/codegen/glob-005.c
+++ b/test/codegen/glob-005.c
@@ -4,9 +4,9 @@
 
 # ____________________________________________________________
 
-void testglob_005(crx_builder_t *cb)
+void testglob_005(_crx_builder_t *cb)
 {
-    crx_type_t *t1, *t2, *t3;
+    _crx_type_t *t1, *t2, *t3;
     {
         mytype_t *p1;
         char *p2;
diff --git a/test/codegen/macro-001.c b/test/codegen/macro-001.c
--- a/test/codegen/macro-001.c
+++ b/test/codegen/macro-001.c
@@ -3,20 +3,20 @@
 
 # ____________________________________________________________
 
-void testmacro_001(crx_builder_t *cb)
+void testmacro_001(_crx_builder_t *cb)
 {
-    crx_type_t *t1, *t2;
+    _crx_type_t *t1, *t2;
     {
-        crx_num_const_t v;
+        _crx_num_const_t v;
         (void)((FOO) << 1);  /* check that 'FOO' is an integer */
-        t1 = CRX_INT_CONST(cb, FOO, &v, 1);
+        t1 = _CRX_INT_CONST(cb, FOO, &v, 1);
         cb->define_num_const(cb, "FOO", t1, &v);
 #expect NUMCONST FOO = int 42
     }
     {
-        crx_num_const_t v;
+        _crx_num_const_t v;
         (void)((BAR) << 1);  /* check that 'BAR' is an integer */
-        t2 = CRX_INT_CONST(cb, BAR, &v, 1);
+        t2 = _CRX_INT_CONST(cb, BAR, &v, 1);
         cb->define_num_const(cb, "BAR", t2, &v);
 #expect NUMCONST BAR = unsigned int 42
     }
diff --git a/test/codegen/struct-001.c b/test/codegen/struct-001.c
--- a/test/codegen/struct-001.c
+++ b/test/codegen/struct-001.c
@@ -5,10 +5,10 @@
 
 # ____________________________________________________________
 
-void teststruct_001(crx_builder_t *cb)
+void teststruct_001(_crx_builder_t *cb)
 {
-    crx_type_t *t1, *t2, *t3;
-    crx_field_t d1[2];
+    _crx_type_t *t1, *t2, *t3;
+    _crx_field_t d1[2];
     t1 = cb->get_struct_type(cb, "foo_s");
     {
         struct foo_s *p1;
@@ -17,7 +17,7 @@
         p1 = (void *)(((char *)b) - o);
         (void)(p1->aa << 1);  /* check that 'struct foo_s::aa' is an integer type */
         p1->aa = -1;  /* check that 'struct foo_s::aa' is not declared 'const' */
-        t2 = CRX_INT_TYPE(cb, p1->aa, "int");
+        t2 = _CRX_INT_TYPE(cb, p1->aa, "int");
         d1[0].name = "aa";
         d1[0].type = t2;
         d1[0].offset = o;
@@ -31,7 +31,7 @@
         p1 = (void *)(((char *)b) - o);
         (void)(p1->bb << 1);  /* check that 'struct foo_s::bb' is an integer type */
         p1->bb = -1;  /* check that 'struct foo_s::bb' is not declared 'const' */
-        t3 = CRX_INT_TYPE(cb, p1->bb, "int");
+        t3 = _CRX_INT_TYPE(cb, p1->bb, "int");
         d1[1].name = "bb";
         d1[1].type = t3;
         d1[1].offset = o;
diff --git a/test/codegen/struct-001b.c b/test/codegen/struct-001b.c
--- a/test/codegen/struct-001b.c
+++ b/test/codegen/struct-001b.c
@@ -4,10 +4,10 @@
 
 # ____________________________________________________________
 
-void teststruct_001b(crx_builder_t *cb)
+void teststruct_001b(_crx_builder_t *cb)
 {
-    crx_type_t *t1, *t2, *t3;
-    crx_field_t d1[1];
+    _crx_type_t *t1, *t2, *t3;
+    _crx_field_t d1[1];
     t1 = cb->get_struct_type(cb, "foo_s");
     {
         struct foo_s *p1;
@@ -16,7 +16,7 @@
         memset(b, -1, sizeof(b));
         p1 = (void *)(((char *)b) - o);
         (void)(p1->aa << 1);  /* check that 'struct foo_s::aa' is an integer type */
-        t2 = CRX_INT_TYPE(cb, p1->aa, "int");
+        t2 = _CRX_INT_TYPE(cb, p1->aa, "int");
         t3 = cb->get_const_type(cb, t2);
         d1[0].name = "aa";
         d1[0].type = t3;
diff --git a/test/codegen/struct-002.c b/test/codegen/struct-002.c
--- a/test/codegen/struct-002.c
+++ b/test/codegen/struct-002.c
@@ -4,10 +4,10 @@
 
 # ____________________________________________________________
 
-void teststruct_002(crx_builder_t *cb)
+void teststruct_002(_crx_builder_t *cb)
 {
-    crx_type_t *t1, *t2, *t3;
-    crx_field_t d1[1];
+    _crx_type_t *t1, *t2, *t3;
+    _crx_field_t d1[1];
     t1 = cb->get_struct_type(cb, "foo_s");
     {
         struct foo_s *p1;
diff --git a/test/codegen/struct-003.c b/test/codegen/struct-003.c
--- a/test/codegen/struct-003.c
+++ b/test/codegen/struct-003.c
@@ -4,10 +4,10 @@
 
 # ____________________________________________________________
 
-void teststruct_003(crx_builder_t *cb)
+void teststruct_003(_crx_builder_t *cb)
 {
-    crx_type_t *t1, *t2, *t3;
-    crx_field_t d1[1];
+    _crx_type_t *t1, *t2, *t3;
+    _crx_field_t d1[1];
     t1 = cb->get_struct_type(cb, "foo_s");
     {
         struct foo_s *p1;
@@ -18,7 +18,7 @@
         p1->aa = (void *)b;    /* check that 'struct foo_s::aa' is a pointer type */
         (void)(*p1->aa << 1);  /* check that '*struct foo_s::aa' is an integer type */
         *p1->aa = -1;  /* check that '*struct foo_s::aa' is not declared 'const' */
-        t2 = CRX_INT_TYPE(cb, *p1->aa, "int");
+        t2 = _CRX_INT_TYPE(cb, *p1->aa, "int");
         t3 = cb->get_pointer_type(cb, t2);
         d1[0].name = "aa";
         d1[0].type = t3;
diff --git a/test/codegen/struct-004.c b/test/codegen/struct-004.c
--- a/test/codegen/struct-004.c
+++ b/test/codegen/struct-004.c
@@ -4,10 +4,10 @@
 
 # ____________________________________________________________
 
-void teststruct_004(crx_builder_t *cb)
+void teststruct_004(_crx_builder_t *cb)
 {
-    crx_type_t *t1, *t2, *t3;
-    crx_field_t d1[1];
+    _crx_type_t *t1, *t2, *t3;
+    _crx_field_t d1[1];
     t1 = cb->get_struct_type(cb, "foo_s");
     {
         struct foo_s *p1;
@@ -20,7 +20,7 @@
         }
         (void)(*p1->aa << 1);  /* check that 'struct foo_s::aa[]' is an integer type */
         *p1->aa = -1;  /* check that 'struct foo_s::aa[]' is not declared 'const' */
-        t2 = CRX_INT_TYPE(cb, *p1->aa, "int");
+        t2 = _CRX_INT_TYPE(cb, *p1->aa, "int");
         t3 = cb->get_array_type(cb, t2, sizeof(p1->aa) / sizeof(*p1->aa));
         d1[0].name = "aa";
         d1[0].type = t3;
diff --git a/test/codegen/struct-005.c b/test/codegen/struct-005.c
--- a/test/codegen/struct-005.c
+++ b/test/codegen/struct-005.c
@@ -2,10 +2,10 @@
 
 # ____________________________________________________________
 
-void teststruct_005(crx_builder_t *cb)
+void teststruct_005(_crx_builder_t *cb)
 {
-    crx_type_t *t1, *t2;
-    crx_field_t d1[1];
+    _crx_type_t *t1, *t2;
+    _crx_field_t d1[1];
     t1 = cb->get_struct_type(cb, "$foo_t");
     {
         foo_t *p1;
@@ -14,7 +14,7 @@
         p1 = (void *)(((char *)b) - o);
         (void)(p1->aa << 1);  /* check that 'foo_t::aa' is an integer type */
         p1->aa = -1;  /* check that 'foo_t::aa' is not declared 'const' */
-        t2 = CRX_INT_TYPE(cb, p1->aa, "int");
+        t2 = _CRX_INT_TYPE(cb, p1->aa, "int");
         d1[0].name = "aa";
         d1[0].type = t2;
         d1[0].offset = o;
diff --git a/test/codegen/struct-005b.c b/test/codegen/struct-005b.c
--- a/test/codegen/struct-005b.c
+++ b/test/codegen/struct-005b.c
@@ -2,10 +2,10 @@
 
 # ____________________________________________________________
 
-void teststruct_005b(crx_builder_t *cb)
+void teststruct_005b(_crx_builder_t *cb)
 {
-    crx_type_t *t1, *t2, *t3;
-    crx_field_t d1[1];
+    _crx_type_t *t1, *t2, *t3;
+    _crx_field_t d1[1];
     t1 = cb->get_struct_type(cb, "$$foo_p");
     {
         foo_p p1;
@@ -14,7 +14,7 @@
         p1 = (void *)(((char *)b) - o);
         (void)(p1->aa << 1);  /* check that '*foo_p::aa' is an integer type */
         p1->aa = -1;  /* check that '*foo_p::aa' is not declared 'const' */
-        t2 = CRX_INT_TYPE(cb, p1->aa, "int");
+        t2 = _CRX_INT_TYPE(cb, p1->aa, "int");
         d1[0].name = "aa";
         d1[0].type = t2;
         d1[0].offset = o;
diff --git a/test/codegen/struct-006.c b/test/codegen/struct-006.c
--- a/test/codegen/struct-006.c
+++ b/test/codegen/struct-006.c
@@ -2,9 +2,9 @@
 
 # ____________________________________________________________
 
-void teststruct_006(crx_builder_t *cb)
+void teststruct_006(_crx_builder_t *cb)
 {
-    crx_type_t *t1;
+    _crx_type_t *t1;
     t1 = cb->get_struct_type(cb, "$foo_t");
     cb->complete(cb, t1, sizeof(foo_t),
                  ((char *)&((struct{char a; foo_t b;} *)0)->b) - (char *)0,
diff --git a/test/test_c_decl_parser.py b/test/test_c_decl_parser.py
--- a/test/test_c_decl_parser.py
+++ b/test/test_c_decl_parser.py
@@ -2,37 +2,14 @@
 from .udir import udir
 
 
-TESTER = r"""
-#include "c_decl_parser.c"
-#include "creflect_print.h"
-
-int main(int argc, char *argv[])
-{
-    crx_type_t *t1;
-    const char *p = creflect_decl_parser(&maincb, argv[1], &t1);
-    if (p == NULL) {
-        printf("%s\n", t1->text);
-    }
-    else {
-        printf("%s\n", argv[1]);
-        while (p > argv[1]) {
-            printf(" ");
-            p--;
-        }
-        printf("^\n");
-    }
-    return 0;
-}
-"""
-
-
 def setup_module(mod):
+    creflect_dir = os.path.join(os.path.dirname(__file__), '..', 'creflect')
     executable = str(udir.join('c_decl_parser_test'))
-    f = open(executable + '.c', 'w')
-    f.write(TESTER)
-    f.close()
-    err = os.system("gcc -g -Werror '%s.c' -o '%s' -I../creflect/src" % (
-        executable, executable))
+    err = os.system("gcc -g -Werror -o '%s'"
+                    " '%s/creflect_cdecl.c'"
+                    " '%s/creflect_debug_print.c'"
+                    " '%s/creflect_cdecl_main.c'" % (
+                        executable, creflect_dir, creflect_dir, creflect_dir))
     assert not err
     mod.executable = executable
 
diff --git a/test/test_cgcompile.py b/test/test_cgcompile.py
--- a/test/test_cgcompile.py
+++ b/test/test_cgcompile.py
@@ -19,27 +19,29 @@
     infile = str(udir.join('cg-' + filename))
     outfile = str(udir.join('cg-' + basename))
     assert infile != outfile
+    creflect_dir = os.path.join(path, '..', '..', 'creflect')
     f = open(infile, 'w')
-    f.write('#include \n')
-    f.write('#include "%s/../../creflect/src/creflect.h"\n' % path)
-    f.write('\n')
+    f.write('''
+#include "creflect.h"
+#include "creflect_debug_print.h"
+''')
     for line in inputlines:
         if not (line.startswith('# ') or line.startswith('#expect')):
             if line.startswith('typedef ...'):
                 line = 'typedef struct _missing ' + line[11:]
             f.write(line)
-    f.write('\n')
-    f.write('#include "%s/../../creflect/src/creflect_print.h"\n' % path)
     f.write('''
 int main(void)
 {
-    test%s(&maincb);
+    test%s(&creflect_debug_builder);
     return 0;
 }
 ''' % (basename.replace('-','_'),))
     f.close()
     #
-    err = os.system("gcc -g -Werror -Wall '%s' -o '%s'" % (infile, outfile))

From noreply at buildbot.pypy.org  Sat Nov 29 18:14:36 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Sat, 29 Nov 2014 18:14:36 +0100 (CET)
Subject: [pypy-commit] creflect default: more fixes
Message-ID: <20141129171436.85BA71D370F@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r122:b08f974d6137
Date: 2014-11-29 18:13 +0100
http://bitbucket.org/cffi/creflect/changeset/b08f974d6137/

Log:	more fixes

diff --git a/creflect/driver.py b/creflect/driver.py
--- a/creflect/driver.py
+++ b/creflect/driver.py
@@ -68,7 +68,7 @@
         funcname = '_creflect_block%d' % blocknum
         output = expand_cdef(''.join(csource), funcname, first_lineno)
         outputf.write(output)
-        outputf.write('#define _CREFLECT_BLOCK%(blocknum)d\n' % {
+        outputf.write('#define _CRX_BLOCK%(blocknum)d\n' % {
             'blocknum': blocknum})
         if include_text_outside_creflect:
             outputf.write("\n/***** CREFLECT block end, "
@@ -88,7 +88,7 @@
 %s}
 '''
 CALL_TEMPLATE2 = '''\
-#ifdef _CREFLECT_BLOCK%(blocknum)d
+#ifdef _CRX_BLOCK%(blocknum)d
     %(funcname)s(cb);
 #endif
 '''
diff --git a/test/test_driver.py b/test/test_driver.py
--- a/test/test_driver.py
+++ b/test/test_driver.py
@@ -11,9 +11,10 @@
     f.close()
     #
     from creflect.driver import __file__
-    err = os.system("gcc -g -Werror -Wall -o '%s' -I../creflect"
-                    " '%s.c' ../creflect/creflect_debug_print.c" % (
-                        executable, executable))
+    creflect_dir = os.path.dirname(__file__)
+    err = os.system("cd '%s' && gcc -g -Werror -Wall -o '%s' -I."
+                    " '%s.c' creflect_debug_print.c" % (
+                        creflect_dir, executable, executable))
     assert err == 0
     #
     g = os.popen("'%s'" % (executable,), 'r')

From noreply at buildbot.pypy.org  Sat Nov 29 21:32:01 2014
From: noreply at buildbot.pypy.org (mattip)
Date: Sat, 29 Nov 2014 21:32:01 +0100 (CET)
Subject: [pypy-commit] pypy ufuncapi: handle the difference between the
 GenericUfuncApi and the GeneralizedUfuncApi
Message-ID: <20141129203201.1D5CC1D2705@cobra.cs.uni-duesseldorf.de>

Author: mattip 
Branch: ufuncapi
Changeset: r74760:8651d4080984
Date: 2014-11-29 22:28 +0200
http://bitbucket.org/pypy/pypy/changeset/8651d4080984/

Log:	handle the difference between the GenericUfuncApi and the
	GeneralizedUfuncApi

diff --git a/pypy/module/cpyext/ndarrayobject.py b/pypy/module/cpyext/ndarrayobject.py
--- a/pypy/module/cpyext/ndarrayobject.py
+++ b/pypy/module/cpyext/ndarrayobject.py
@@ -265,6 +265,13 @@
               rffi.CCHARP], PyObject)
 def PyUFunc_FromFuncAndDataAndSignature(space, funcs, data, types, ntypes,
                     nin, nout, identity, name, doc, check_return, signature):
+    w_signature = rffi.charp2str(signature)
+    return do_ufunc(space, funcs, data, types, ntypes, nin, nout, identity, name, doc,
+             check_return, w_signature)
+            
+
+def do_ufunc(space, funcs, data, types, ntypes, nin, nout, identity, name, doc,
+             check_return, w_signature):
     funcs_w = [None] * ntypes
     dtypes_w = [None] * ntypes * (nin + nout)
     for i in range(ntypes):
@@ -273,10 +280,17 @@
         dtypes_w[i] = get_dtype_cache(space).dtypes_by_num[ord(types[i])]
     w_funcs = space.newlist(funcs_w)
     w_dtypes = space.newlist(dtypes_w)
-    w_signature = rffi.charp2str(signature)
     w_doc = rffi.charp2str(doc)
     w_name = rffi.charp2str(name)
     w_identity = space.wrap(identity)
     ufunc_generic = ufuncs.frompyfunc(space, w_funcs, nin, nout, w_dtypes,
                  w_signature, w_identity, w_name, w_doc, stack_inputs=True)
     return ufunc_generic
+
+ at cpython_api([rffi.CArrayPtr(rffi.CArrayPtr(gufunctype)), rffi.VOIDP, rffi.CCHARP, Py_ssize_t, Py_ssize_t,
+              Py_ssize_t, Py_ssize_t, rffi.CCHARP, rffi.CCHARP, Py_ssize_t], PyObject)
+def PyUFunc_FromFuncAndData(space, funcs, data, types, ntypes,
+                    nin, nout, identity, name, doc, check_return):
+    w_signature = ""
+    return do_ufunc(space, funcs, data, types, ntypes, nin, nout, identity,
+                    name, doc, check_return, w_signature)
diff --git a/pypy/module/cpyext/test/test_ndarrayobject.py b/pypy/module/cpyext/test/test_ndarrayobject.py
--- a/pypy/module/cpyext/test/test_ndarrayobject.py
+++ b/pypy/module/cpyext/test/test_ndarrayobject.py
@@ -322,9 +322,9 @@
                 char types[] = { NPY_DOUBLE,NPY_DOUBLE, NPY_INT, NPY_INT };
                 void *array_data[] = {NULL, NULL};
                 PyObject * retval;
-                retval = PyUFunc_FromFuncAndDataAndSignature(funcs,
+                retval = PyUFunc_FromFuncAndData(funcs,
                                     array_data, types, 2, 1, 1, PyUFunc_None,
-                                    "times2", "times2_docstring", 0, "()->()");
+                                    "times2", "times2_docstring", 0);
                 return retval;
                 """
                 ),
@@ -336,13 +336,23 @@
                 PyObject * retval;
                 retval = PyUFunc_FromFuncAndDataAndSignature(funcs,
                                     array_data, types, 2, 1, 1, PyUFunc_None,
-                                    "times2", "times2_docstring", 0, "(m)->(m)");
+                                    "times2", "times2_docstring", 0, "()->()");
                 return retval;
+                """),
+                ("create_float_ufunc_3x3", "METH_NOARGS",
                 """
-                ),
+                PyUFuncGenericFunction funcs[] = {&float_func_with_sig_3x3};
+                char types[] = { NPY_FLOAT,NPY_FLOAT};
+                void *array_data[] = {NULL, NULL};
+                return PyUFunc_FromFuncAndDataAndSignature(funcs,
+                                    array_data, types, 1, 1, 1, PyUFunc_None,
+                                    "float_3x3", 
+                                    "a ufunc that tests a more complicated signature", 
+                                    0, "(m,m)->(m,m)");
+                """),
                 ], prologue='''
                 #include "numpy/ndarraytypes.h"
-                /*#include */
+                /*#include  generated by numpy setup.py*/
                 typedef void (*PyUFuncGenericFunction)
                             (char **args,
                              npy_intp *dimensions,
@@ -390,11 +400,34 @@
                         in += in_step;
                         out += out_step;
                     };
-                }; ''')
+                };
+                void float_func_with_sig_3x3(char ** args, npy_intp * dimensions,
+                              npy_intp* steps, void* data)
+                {
+                    int target_dims[] = {1, 3};
+                    int target_steps[] = {0, 0, 12, 4, 12, 4};
+                    int res = 0;
+                    int i;
+                    for (i=0; i 1:
+            _stride = [min(_stride[0], _stride[1])]
         _backstride = [(shape[0] - 1) * _stride[0]]
     return SliceIter(old_iter.array, old_iter.size / shape[fastest],
                 new_shape, new_strides, new_backstrides,
diff --git a/pypy/module/micronumpy/test/test_ufuncs.py b/pypy/module/micronumpy/test/test_ufuncs.py
--- a/pypy/module/micronumpy/test/test_ufuncs.py
+++ b/pypy/module/micronumpy/test/test_ufuncs.py
@@ -150,8 +150,9 @@
         ufunc = frompyfunc([int_times2, double_times2], 1, 1,
                             signature='()->()',
                             dtypes=[dtype(int), dtype(int),
-                            dtype(float), dtype(float)
-                            ]
+                                    dtype(float), dtype(float)
+                                    ],
+                            stack_inputs=True,
                     )
         ai = arange(10, dtype=int)
         ai2 = ufunc(ai)
@@ -167,13 +168,23 @@
             out_array[:] = in_array * 2
         from numpy import frompyfunc, dtype, arange
         ufunc = frompyfunc([times_2], 1, 1,
+                            signature='(m,n)->(n,m)',
+                            dtypes=[dtype(int), dtype(int)],
+                            stack_inputs=True,
+                          )
+        ai = arange(18, dtype=int).reshape(2,3,3)
+        ai3 = ufunc(ai[0,:,:])
+        ai2 = ufunc(ai)
+        assert (ai2 == ai * 2).all()
+        ufunc = frompyfunc([times_2], 1, 1,
                             signature='(m,m)->(m,m)',
                             dtypes=[dtype(int), dtype(int)],
                             stack_inputs=True,
                           )
         ai = arange(18, dtype=int).reshape(2,3,3)
         exc = raises(ValueError, ufunc, ai[:,:,0])
-        assert "mismatch in its core dimension 1" in exc.value.message
+        assert "Operand 0 has a mismatch in its core dimension 1" in exc.value.message
+        ai3 = ufunc(ai[0,:,:])
         ai2 = ufunc(ai)
         assert (ai2 == ai * 2).all()
 
diff --git a/pypy/module/micronumpy/ufuncs.py b/pypy/module/micronumpy/ufuncs.py
--- a/pypy/module/micronumpy/ufuncs.py
+++ b/pypy/module/micronumpy/ufuncs.py
@@ -581,8 +581,13 @@
         func = self.funcs[index]
         if not self.core_enabled:
             # func is going to do all the work, it must accept W_NDimArray args
-            arglist = space.newlist(list(inargs + outargs))
-            space.call_args(func, Arguments.frompacked(space, arglist))
+            if self.stack_inputs:
+                arglist = space.newlist(list(inargs + outargs))
+                space.call_args(func, Arguments.frompacked(space, arglist))
+            else:
+                arglist = space.newlist(inargs)
+                outargs = space.call_args(func, Arguments.frompacked(space, arglist))
+                return outargs
             if len(outargs) < 2:
                 return outargs[0]
             return space.newtuple(outargs)
@@ -603,12 +608,9 @@
         for i in range(self.nin, self.nargs):
             iter_ndim += self.core_num_dims[i];
         # Validate the core dimensions of all the operands,
-        # and collect all of the labeled core dimension sizes
-        # into the array 'inner_dimensions[1:]'. Initialize them to
-        # 1, for example in the case where the operand broadcasts
-        # to a core dimension, it won't be visited.
         inner_dimensions = [1] * (self.core_num_dim_ix + 2)
         idim = 0
+        core_start_dim = 0
         for i in range(self.nin):
             curarg = inargs[i]
             assert isinstance(curarg, W_NDimArray)
@@ -706,8 +708,33 @@
         w_itershape = space.newlist([space.wrap(i) for i in iter_shape]) 
         w_op_axes = space.w_None
 
-
         if self.stack_inputs:
+            #print '\nsignature', sig
+            #print [(d, getattr(self,d)) for d in dir(self) if 'core' in d or 'broad' in d]
+            #print [(d, locals()[d]) for d in locals() if 'core' in d or 'broad' in d]
+            #print 'shapes',[d.get_shape() for d in inargs + outargs]
+            #print 'steps',[d.implementation.strides for d in inargs + outargs]
+            if isinstance(func, W_GenericUFuncCaller):
+                # Use GeneralizeUfunc interface with signature
+                # Unlike numpy, we will not broadcast dims before
+                # the core_ndims rather we use nditer iteration
+                # so dims[0] == 1
+                dims = [1] + inner_dimensions[1:]
+                steps = []
+                allargs = inargs + outargs
+                assert core_start_dim >= 0
+                for i in range(len(allargs)):
+                    steps.append(0)
+                for i in range(len(allargs)):
+                    _arg = allargs[i]
+                    assert isinstance(_arg, W_NDimArray)
+                    steps += _arg.implementation.strides[core_start_dim:]
+                #print 'set_dims_and_steps with dims, steps',dims,steps
+                func.set_dims_and_steps(space, dims, steps)
+            else:
+                # it is a function, ready to be called by the iterator,
+                # from frompyfunc
+                pass
             # mimic NpyIter_AdvancedNew with a nditer
             nd_it = W_NDIter(space, space.newlist(inargs + outargs), w_flags,
                           w_op_flags, w_op_dtypes, w_casting, w_op_axes,
@@ -808,14 +835,6 @@
             assert isinstance(outargs[i], W_NDimArray)
         return outargs
 
-    def prep_call(self, space, index, inargs, outargs):
-        # Use the index and signature to determine
-        # dims and steps for function call
-        return self.funcs[index], [inargs[0].get_shape()[0]], \
-                 [inargs[0].implementation.get_strides()[0],
-                  outargs[0].implementation.get_strides()[0]]
-
-
 W_Ufunc.typedef = TypeDef("numpy.ufunc",
     __call__ = interp2app(W_Ufunc.descr_call),
     __repr__ = interp2app(W_Ufunc.descr_repr),
@@ -1176,7 +1195,7 @@
     stack_inputs*: boolean, whether the function is of the form
             out = func(*in)  False
             or
-            func(*in_out)    True (forces use of a nditer with 'external_loop')
+            func(*[in + out])    True 
 
     only one of out_dtype or signature may be specified
 
@@ -1243,8 +1262,8 @@
             'identity must be None or an int')
 
     if len(signature) == 0:
-        # cpython compatability, func is of the form (i),(i)->(i)
-        signature = ','.join(['(i)'] * nin) + '->' + ','.join(['(i)'] * nout)
+        # cpython compatability, func is of the form (),()->()
+        signature = ','.join(['()'] * nin) + '->' + ','.join(['()'] * nout)
     else:
         #stack_inputs = True
         pass
@@ -1257,18 +1276,21 @@
         w_ret.w_doc = space.wrap(doc)
     return w_ret
 
-# Instantiated in cpyext/ndarrayobject
+# Instantiated in cpyext/ndarrayobject. It is here since ufunc calls
+# set_dims_and_steps, otherwise ufunc, ndarrayobject would have circular
+# imports
 npy_intpp = rffi.LONGP
 LONG_SIZE = LONG_BIT / 8
 CCHARP_SIZE = _get_bitsize('P') / 8
 
 class W_GenericUFuncCaller(W_Root):
-    _attrs_ = ['func', 'data', 'dims', 'steps']
+    _attrs_ = ['func', 'data', 'dims', 'steps', 'dims_steps_set']
     def __init__(self, func, data):
         self.func = func
         self.data = data
         self.dims = alloc_raw_storage(0, track_allocation=False)
         self.steps = alloc_raw_storage(0, track_allocation=False)
+        self.dims_steps_set = False
 
     def __del__(self):
         free_raw_storage(self.dims, track_allocation=False)
@@ -1276,10 +1298,14 @@
 
     def descr_call(self, space, __args__):
         args_w, kwds_w = __args__.unpack()
-        # Can be called two ways, as an inner-loop function with boxes, 
-        # or as an outer-looop functio with ndarrays
+        # Can be called two ways, as a GenericUfunc or a GeneralizedUfunc.
+        # The difference is in the meaning of dims and steps,
+        # a GenericUfunc is a scalar function that flatiters over the array(s).
+        # a GeneralizedUfunc will iterate over dims[0], but will use dims[1...]
+        # and steps[1, ...] to call a function on ndarray(s).
+        # set up via a call to set_dims_and_steps()
         dataps = alloc_raw_storage(CCHARP_SIZE * len(args_w), track_allocation=False)
-        if isinstance(args_w[0], W_NDimArray):
+        if self.dims_steps_set is False:
             self.dims = alloc_raw_storage(LONG_SIZE * len(args_w), track_allocation=False)
             self.steps = alloc_raw_storage(LONG_SIZE * len(args_w), track_allocation=False)
             for i in range(len(args_w)):
@@ -1294,13 +1320,10 @@
                 raw_storage_setitem(self.dims, LONG_SIZE * i, rffi.cast(rffi.LONG, arg_i.get_size()))
                 raw_storage_setitem(self.steps, LONG_SIZE * i, rffi.cast(rffi.LONG, arg_i.get_dtype().elsize))
         else:
-            if self.dims is None or self.steps is None:
-                raise OperationError(space.w_RuntimeError,
-                     space.wrap("call set_dims_and_steps first"))
             for i in range(len(args_w)):
                 arg_i = args_w[i]
-                # raw_storage_setitem(dataps, CCHARP_SIZE * i,
-                #       rffi.cast(rffi.CCHARP, arg_i.storage))
+                raw_storage_setitem(dataps, CCHARP_SIZE * i,
+                        rffi.cast(rffi.CCHARP, arg_i.implementation.get_storage_as_int(space)))
         try:
             arg1 = rffi.cast(rffi.CArrayPtr(rffi.CCHARP), dataps)
             arg2 = rffi.cast(npy_intpp, self.dims)
@@ -1309,29 +1332,24 @@
         finally:
             free_raw_storage(dataps, track_allocation=False)
 
+    def set_dims_and_steps(self, space, dims, steps):
+        if not isinstance(dims, list) or not isinstance(steps, list):
+            raise oefmt(space.w_RuntimeError,
+                 "set_dims_and_steps called inappropriately")
+        if self.dims_steps_set:
+            raise oefmt(space.w_RuntimeError,
+                 "set_dims_and_steps called inappropriately")
+        self.dims = alloc_raw_storage(LONG_SIZE * len(dims), track_allocation=False)
+        self.steps = alloc_raw_storage(LONG_SIZE * len(steps), track_allocation=False)
+        for i in range(len(dims)):
+            raw_storage_setitem(self.dims, LONG_SIZE * i, rffi.cast(rffi.LONG, dims[i]))
+        for i in range(len(steps)):
+            raw_storage_setitem(self.steps, LONG_SIZE * i, rffi.cast(rffi.LONG, steps[i]))
+        self.dims_steps_set = True
+
 W_GenericUFuncCaller.typedef = TypeDef("hiddenclass",
     __call__ = interp2app(W_GenericUFuncCaller.descr_call),
 )
 
-def set_dims_and_steps(obj, space, dims, steps):
-        if not isinstance(obj, W_GenericUFuncCaller):
-            raise OperationError(space.w_RuntimeError,
-                 space.wrap("set_dims_and_steps called inappropriately"))
-        if not isinstance(dims, list) or not isinstance(steps, list):
-            raise OperationError(space.w_RuntimeError,
-                 space.wrap("set_dims_and_steps called inappropriately"))
-        if len(dims) != len(step):
-            raise OperationError(space.w_RuntimeError,
-                 space.wrap("set_dims_and_steps called inappropriately"))
-        if self.dims is not None or self.steps is not None:
-            raise OperationError(space.w_RuntimeError,
-                 space.wrap("set_dims_and_steps called inappropriately"))
-        self.dims = alloc_raw_storage(LONG_SIZE * len(dims), track_allocation=False)
-        self.steps = alloc_raw_storage(LONG_SIZE * len(dims), track_allocation=False)
-        for d in dims:
-            raw_storage_setitem(self.dims, LONG_SIZE * i, rffi.cast(rffi.LONG, d))
-        for d in steps:
-            raw_storage_setitem(self.steps, LONG_SIZE * i, rffi.cast(rffi.LONG, d))
-
 GenericUfunc = lltype.FuncType([rffi.CArrayPtr(rffi.CCHARP), npy_intpp, npy_intpp,
                                       rffi.VOIDP], lltype.Void)

From noreply at buildbot.pypy.org  Sat Nov 29 22:00:35 2014
From: noreply at buildbot.pypy.org (mattip)
Date: Sat, 29 Nov 2014 22:00:35 +0100 (CET)
Subject: [pypy-commit] pypy ufuncapi: assert yet again that args_w contains
 W_NDimArray items for translation
Message-ID: <20141129210035.5A9F01D282F@cobra.cs.uni-duesseldorf.de>

Author: mattip 
Branch: ufuncapi
Changeset: r74761:ad40184861fe
Date: 2014-11-29 23:00 +0200
http://bitbucket.org/pypy/pypy/changeset/ad40184861fe/

Log:	assert yet again that args_w contains W_NDimArray items for
	translation

diff --git a/pypy/module/micronumpy/ufuncs.py b/pypy/module/micronumpy/ufuncs.py
--- a/pypy/module/micronumpy/ufuncs.py
+++ b/pypy/module/micronumpy/ufuncs.py
@@ -1322,6 +1322,7 @@
         else:
             for i in range(len(args_w)):
                 arg_i = args_w[i]
+                assert isinstance(arg_i, W_NDimArray)
                 raw_storage_setitem(dataps, CCHARP_SIZE * i,
                         rffi.cast(rffi.CCHARP, arg_i.implementation.get_storage_as_int(space)))
         try:

From noreply at buildbot.pypy.org  Sat Nov 29 22:32:45 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Sat, 29 Nov 2014 22:32:45 +0100 (CET)
Subject: [pypy-commit] creflect default: Make the second argument to
 'creflect' optional (defaults to the first
Message-ID: <20141129213245.BCE9A1D263B@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r123:62e314b00876
Date: 2014-11-29 18:24 +0100
http://bitbucket.org/cffi/creflect/changeset/62e314b00876/

Log:	Make the second argument to 'creflect' optional (defaults to the
	first argument with extension '.c').

diff --git a/creflect/cmdline.py b/creflect/cmdline.py
--- a/creflect/cmdline.py
+++ b/creflect/cmdline.py
@@ -1,7 +1,7 @@
 """Usage:
 
-    creflect  [options]  input.crx  output.c
-    creflect  [options]  -i  input.h  output.c
+    creflect  [options]  input.crx  [output.c]
+    creflect  [options]  -i  input.h  [output.c]
 
 Read the input file and expand the CREFLECT sections in it
 into regular C code.  Write the result to 'output.c'.
@@ -62,9 +62,13 @@
             print 'CReflect %s' % __version__
             return 0
     #
-    if len(args) != 2:
-        return error("expected exactly 2 arguments, got %d" % (len(args),))
-    inputfile, outputfile = args
+    if len(args) == 1:
+        inputfile, = args
+        outputfile = os.path.splitext(inputfile)[0] + '.c'
+    elif len(args) == 2:
+        inputfile, outputfile = args
+    else:
+        return error("expected 1 or 2 arguments, got %d" % (len(args),))
     if inputfile == outputfile:
         return error("not overwriting the file %r" % (inputfile,))
     #

From noreply at buildbot.pypy.org  Sat Nov 29 22:32:47 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Sat, 29 Nov 2014 22:32:47 +0100 (CET)
Subject: [pypy-commit] creflect default: Move 'test' into 'creflect' and
 start the 'zeffir' package. ('zeffir'
Message-ID: <20141129213247.233CE1D263B@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r124:1e96fd0d81ff
Date: 2014-11-29 19:20 +0100
http://bitbucket.org/cffi/creflect/changeset/1e96fd0d81ff/

Log:	Move 'test' into 'creflect' and start the 'zeffir' package.
	('zeffir' is the name for what I called cffi 1.0, maybe
	temporarily.)

diff --git a/creflect/creflect_cdecl.c b/creflect/creflect_cdecl.c
--- a/creflect/creflect_cdecl.c
+++ b/creflect/creflect_cdecl.c
@@ -56,7 +56,7 @@
 
 static int is_ident_first(char x)
 {
-    return ('A' <= x && x <= 'Z' || 'a' <= x && x <= 'z' || x == '_');
+    return (('A' <= x && x <= 'Z') || ('a' <= x && x <= 'z') || x == '_');
 }
 
 static int is_digit(char x)
diff --git a/creflect/creflect_libcheck.c b/creflect/creflect_libcheck.c
--- a/creflect/creflect_libcheck.c
+++ b/creflect/creflect_libcheck.c
@@ -6,7 +6,7 @@
 #include 
 
 #include "creflect.h"
-#include "creflect_print.h"
+#include "creflect_debug_print.h"
 
 
 void creflect_dump_information(const char *filename, const char *creflect_main)
@@ -23,12 +23,26 @@
     if (lib == NULL)
         goto err;
 
-    void (*crxmain)(crx_builder_t *);
-    crxmain = (void(*)(crx_builder_t *))dlsym(lib, creflect_main);
+    int (*crxmain)(_crx_builder_t *);
+    crxmain = (int(*)(_crx_builder_t *))dlsym(lib, creflect_main);
     if (crxmain == NULL)
         goto err;
 
-    crxmain(&maincb);
+    int result = crxmain(NULL);
+    if ((result & ~0xFF) != 0x54822e00) {
+        fprintf(stderr, "error: %s.%s() is not a creflect entry point\n",
+                filename, creflect_main);
+        dlclose(lib);
+        return;
+    }
+    if ((result & 0xFF) != 0x01) {
+        fprintf(stderr, "error: %s is creflect version %d, expected %d\n",
+                filename, result & 0xFF, 0x01);
+        dlclose(lib);
+        return;
+    }
+
+    crxmain(&creflect_debug_builder);
 
     if (dlclose(lib) != 0) {
         lib = NULL;
diff --git a/creflect/driver.py b/creflect/driver.py
--- a/creflect/driver.py
+++ b/creflect/driver.py
@@ -84,12 +84,15 @@
 CALL_TEMPLATE1 = '''
 /***** CREFLECT main entry point *****/
 
-void %s(_crx_builder_t *cb) {
+int %s(_crx_builder_t *cb) {
+    if (cb) {
 %s}
+    return 0x54822e01;
+}
 '''
 CALL_TEMPLATE2 = '''\
 #ifdef _CRX_BLOCK%(blocknum)d
-    %(funcname)s(cb);
+        %(funcname)s(cb);
 #endif
 '''
 
diff --git a/test/__init__.py b/creflect/test/__init__.py
rename from test/__init__.py
rename to creflect/test/__init__.py
diff --git a/test/codegen/001.c b/creflect/test/codegen/001.c
rename from test/codegen/001.c
rename to creflect/test/codegen/001.c
diff --git a/test/codegen/002.c b/creflect/test/codegen/002.c
rename from test/codegen/002.c
rename to creflect/test/codegen/002.c
diff --git a/test/codegen/002b.c b/creflect/test/codegen/002b.c
rename from test/codegen/002b.c
rename to creflect/test/codegen/002b.c
diff --git a/test/codegen/002c.c b/creflect/test/codegen/002c.c
rename from test/codegen/002c.c
rename to creflect/test/codegen/002c.c
diff --git a/test/codegen/003.c b/creflect/test/codegen/003.c
rename from test/codegen/003.c
rename to creflect/test/codegen/003.c
diff --git a/test/codegen/003b.c b/creflect/test/codegen/003b.c
rename from test/codegen/003b.c
rename to creflect/test/codegen/003b.c
diff --git a/test/codegen/003c.c b/creflect/test/codegen/003c.c
rename from test/codegen/003c.c
rename to creflect/test/codegen/003c.c
diff --git a/test/codegen/003d.c b/creflect/test/codegen/003d.c
rename from test/codegen/003d.c
rename to creflect/test/codegen/003d.c
diff --git a/test/codegen/003e.c b/creflect/test/codegen/003e.c
rename from test/codegen/003e.c
rename to creflect/test/codegen/003e.c
diff --git a/test/codegen/003f.c b/creflect/test/codegen/003f.c
rename from test/codegen/003f.c
rename to creflect/test/codegen/003f.c
diff --git a/test/codegen/003g.c b/creflect/test/codegen/003g.c
rename from test/codegen/003g.c
rename to creflect/test/codegen/003g.c
diff --git a/test/codegen/003h.c b/creflect/test/codegen/003h.c
rename from test/codegen/003h.c
rename to creflect/test/codegen/003h.c
diff --git a/test/codegen/003i.c b/creflect/test/codegen/003i.c
rename from test/codegen/003i.c
rename to creflect/test/codegen/003i.c
diff --git a/test/codegen/004.c b/creflect/test/codegen/004.c
rename from test/codegen/004.c
rename to creflect/test/codegen/004.c
diff --git a/test/codegen/004b.c b/creflect/test/codegen/004b.c
rename from test/codegen/004b.c
rename to creflect/test/codegen/004b.c
diff --git a/test/codegen/005.c b/creflect/test/codegen/005.c
rename from test/codegen/005.c
rename to creflect/test/codegen/005.c
diff --git a/test/codegen/005b.c b/creflect/test/codegen/005b.c
rename from test/codegen/005b.c
rename to creflect/test/codegen/005b.c
diff --git a/test/codegen/005c.c b/creflect/test/codegen/005c.c
rename from test/codegen/005c.c
rename to creflect/test/codegen/005c.c
diff --git a/test/codegen/005d.c b/creflect/test/codegen/005d.c
rename from test/codegen/005d.c
rename to creflect/test/codegen/005d.c
diff --git a/test/codegen/006.c b/creflect/test/codegen/006.c
rename from test/codegen/006.c
rename to creflect/test/codegen/006.c
diff --git a/test/codegen/006b.c b/creflect/test/codegen/006b.c
rename from test/codegen/006b.c
rename to creflect/test/codegen/006b.c
diff --git a/test/codegen/006c.c b/creflect/test/codegen/006c.c
rename from test/codegen/006c.c
rename to creflect/test/codegen/006c.c
diff --git a/test/codegen/007.c b/creflect/test/codegen/007.c
rename from test/codegen/007.c
rename to creflect/test/codegen/007.c
diff --git a/test/codegen/func-001.c b/creflect/test/codegen/func-001.c
rename from test/codegen/func-001.c
rename to creflect/test/codegen/func-001.c
diff --git a/test/codegen/func-001b.c b/creflect/test/codegen/func-001b.c
rename from test/codegen/func-001b.c
rename to creflect/test/codegen/func-001b.c
diff --git a/test/codegen/func-001c.c b/creflect/test/codegen/func-001c.c
rename from test/codegen/func-001c.c
rename to creflect/test/codegen/func-001c.c
diff --git a/test/codegen/func-002.c b/creflect/test/codegen/func-002.c
rename from test/codegen/func-002.c
rename to creflect/test/codegen/func-002.c
diff --git a/test/codegen/func-003.c b/creflect/test/codegen/func-003.c
rename from test/codegen/func-003.c
rename to creflect/test/codegen/func-003.c
diff --git a/test/codegen/func-003b.c b/creflect/test/codegen/func-003b.c
rename from test/codegen/func-003b.c
rename to creflect/test/codegen/func-003b.c
diff --git a/test/codegen/func-004.c b/creflect/test/codegen/func-004.c
rename from test/codegen/func-004.c
rename to creflect/test/codegen/func-004.c
diff --git a/test/codegen/func-005.c b/creflect/test/codegen/func-005.c
rename from test/codegen/func-005.c
rename to creflect/test/codegen/func-005.c
diff --git a/test/codegen/glob-001.c b/creflect/test/codegen/glob-001.c
rename from test/codegen/glob-001.c
rename to creflect/test/codegen/glob-001.c
diff --git a/test/codegen/glob-002.c b/creflect/test/codegen/glob-002.c
rename from test/codegen/glob-002.c
rename to creflect/test/codegen/glob-002.c
diff --git a/test/codegen/glob-002b.c b/creflect/test/codegen/glob-002b.c
rename from test/codegen/glob-002b.c
rename to creflect/test/codegen/glob-002b.c
diff --git a/test/codegen/glob-002c.c b/creflect/test/codegen/glob-002c.c
rename from test/codegen/glob-002c.c
rename to creflect/test/codegen/glob-002c.c
diff --git a/test/codegen/glob-002d.c b/creflect/test/codegen/glob-002d.c
rename from test/codegen/glob-002d.c
rename to creflect/test/codegen/glob-002d.c
diff --git a/test/codegen/glob-003.c b/creflect/test/codegen/glob-003.c
rename from test/codegen/glob-003.c
rename to creflect/test/codegen/glob-003.c
diff --git a/test/codegen/glob-003b.c b/creflect/test/codegen/glob-003b.c
rename from test/codegen/glob-003b.c
rename to creflect/test/codegen/glob-003b.c
diff --git a/test/codegen/glob-003c.c b/creflect/test/codegen/glob-003c.c
rename from test/codegen/glob-003c.c
rename to creflect/test/codegen/glob-003c.c
diff --git a/test/codegen/glob-003d.c b/creflect/test/codegen/glob-003d.c
rename from test/codegen/glob-003d.c
rename to creflect/test/codegen/glob-003d.c
diff --git a/test/codegen/glob-003e.c b/creflect/test/codegen/glob-003e.c
rename from test/codegen/glob-003e.c
rename to creflect/test/codegen/glob-003e.c
diff --git a/test/codegen/glob-003f.c b/creflect/test/codegen/glob-003f.c
rename from test/codegen/glob-003f.c
rename to creflect/test/codegen/glob-003f.c
diff --git a/test/codegen/glob-003g.c b/creflect/test/codegen/glob-003g.c
rename from test/codegen/glob-003g.c
rename to creflect/test/codegen/glob-003g.c
diff --git a/test/codegen/glob-003h.c b/creflect/test/codegen/glob-003h.c
rename from test/codegen/glob-003h.c
rename to creflect/test/codegen/glob-003h.c
diff --git a/test/codegen/glob-003i.c b/creflect/test/codegen/glob-003i.c
rename from test/codegen/glob-003i.c
rename to creflect/test/codegen/glob-003i.c
diff --git a/test/codegen/glob-003j.c b/creflect/test/codegen/glob-003j.c
rename from test/codegen/glob-003j.c
rename to creflect/test/codegen/glob-003j.c
diff --git a/test/codegen/glob-004.c b/creflect/test/codegen/glob-004.c
rename from test/codegen/glob-004.c
rename to creflect/test/codegen/glob-004.c
diff --git a/test/codegen/glob-005.c b/creflect/test/codegen/glob-005.c
rename from test/codegen/glob-005.c
rename to creflect/test/codegen/glob-005.c
diff --git a/test/codegen/macro-001.c b/creflect/test/codegen/macro-001.c
rename from test/codegen/macro-001.c
rename to creflect/test/codegen/macro-001.c
diff --git a/test/codegen/struct-001.c b/creflect/test/codegen/struct-001.c
rename from test/codegen/struct-001.c
rename to creflect/test/codegen/struct-001.c
diff --git a/test/codegen/struct-001b.c b/creflect/test/codegen/struct-001b.c
rename from test/codegen/struct-001b.c
rename to creflect/test/codegen/struct-001b.c
diff --git a/test/codegen/struct-002.c b/creflect/test/codegen/struct-002.c
rename from test/codegen/struct-002.c
rename to creflect/test/codegen/struct-002.c
diff --git a/test/codegen/struct-003.c b/creflect/test/codegen/struct-003.c
rename from test/codegen/struct-003.c
rename to creflect/test/codegen/struct-003.c
diff --git a/test/codegen/struct-004.c b/creflect/test/codegen/struct-004.c
rename from test/codegen/struct-004.c
rename to creflect/test/codegen/struct-004.c
diff --git a/test/codegen/struct-005.c b/creflect/test/codegen/struct-005.c
rename from test/codegen/struct-005.c
rename to creflect/test/codegen/struct-005.c
diff --git a/test/codegen/struct-005b.c b/creflect/test/codegen/struct-005b.c
rename from test/codegen/struct-005b.c
rename to creflect/test/codegen/struct-005b.c
diff --git a/test/codegen/struct-006.c b/creflect/test/codegen/struct-006.c
rename from test/codegen/struct-006.c
rename to creflect/test/codegen/struct-006.c
diff --git a/test/test_c_decl_parser.py b/creflect/test/test_c_decl_parser.py
rename from test/test_c_decl_parser.py
rename to creflect/test/test_c_decl_parser.py
--- a/test/test_c_decl_parser.py
+++ b/creflect/test/test_c_decl_parser.py
@@ -3,7 +3,7 @@
 
 
 def setup_module(mod):
-    creflect_dir = os.path.join(os.path.dirname(__file__), '..', 'creflect')
+    creflect_dir = os.path.join(os.path.dirname(__file__), '..')
     executable = str(udir.join('c_decl_parser_test'))
     err = os.system("gcc -g -Werror -o '%s'"
                     " '%s/creflect_cdecl.c'"
diff --git a/test/test_cgcompile.py b/creflect/test/test_cgcompile.py
rename from test/test_cgcompile.py
rename to creflect/test/test_cgcompile.py
--- a/test/test_cgcompile.py
+++ b/creflect/test/test_cgcompile.py
@@ -19,7 +19,7 @@
     infile = str(udir.join('cg-' + filename))
     outfile = str(udir.join('cg-' + basename))
     assert infile != outfile
-    creflect_dir = os.path.join(path, '..', '..', 'creflect')
+    creflect_dir = os.path.join(path, '..', '..')
     f = open(infile, 'w')
     f.write('''
 #include "creflect.h"
diff --git a/test/test_codegen.py b/creflect/test/test_codegen.py
rename from test/test_codegen.py
rename to creflect/test/test_codegen.py
diff --git a/test/test_cparser.py b/creflect/test/test_cparser.py
rename from test/test_cparser.py
rename to creflect/test/test_cparser.py
diff --git a/test/test_driver.py b/creflect/test/test_driver.py
rename from test/test_driver.py
rename to creflect/test/test_driver.py
diff --git a/test/udir.py b/creflect/test/udir.py
rename from test/udir.py
rename to creflect/test/udir.py
diff --git a/zeffir/zeffir.c b/zeffir/zeffir.c
new file mode 100644
--- /dev/null
+++ b/zeffir/zeffir.c
@@ -0,0 +1,44 @@
+#include 
+#include "../creflect/creflect.h"
+#include "../creflect/creflect_cdecl.h"
+
+
+/************************************************************/
+
+typedef struct {
+    Py_OBJECT_HEAD
+    PyObject *libname;
+} PyFFIObject;
+
+/************************************************************/
+
+static PyObject *b_open(PyObject *self, PyObject *args, PyObject *kwds)
+{
+    char *keywords[] = {"libname", "relative_to", NULL};
+    char *libname, *relative_to;
+
+    if (!PyArg_ParseTupleAndKeywords(args, kwds, "ss", keywords,
+                                     &libname, &relative_to))
+        return NULL;
+
+    PyFFIObject *ffi = PyObject_New(PyFFIObject, &FFI_Type);
+    if (ffi == NULL)
+        return NULL;
+
+    return Py_BuildValue("OO", ffi, Py_None);
+}
+
+
+static PyMethodDef ZeffirMethods[] = {
+    {"open",   (PyCFunction)b_open, METH_KEYWORDS},
+    {NULL,     NULL}    /* Sentinel */
+};
+
+PyMODINIT_FUNC
+initzeffir(void)
+{
+    PyObject *m;
+
+    m = Py_InitModule("zeffir", ZeffirMethods);
+    (void)m;
+}

From noreply at buildbot.pypy.org  Sat Nov 29 22:32:48 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Sat, 29 Nov 2014 22:32:48 +0100 (CET)
Subject: [pypy-commit] creflect default: in-progress
Message-ID: <20141129213248.348931D263B@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r125:10955dc30513
Date: 2014-11-29 22:12 +0100
http://bitbucket.org/cffi/creflect/changeset/10955dc30513/

Log:	in-progress

diff --git a/zeffir/test/basic.crx b/zeffir/test/basic.crx
new file mode 100644
--- /dev/null
+++ b/zeffir/test/basic.crx
@@ -0,0 +1,9 @@
+#define forty_two  42
+
+
+
+// CREFLECT: start
+
+static const int forty_two;
+
+// CREFLECT: end
diff --git a/zeffir/test/test_basic.py b/zeffir/test/test_basic.py
new file mode 100644
--- /dev/null
+++ b/zeffir/test/test_basic.py
@@ -0,0 +1,34 @@
+import os, sys
+from udir import udir, include_dir
+
+def setup_module(mod):
+    zeffir_dir = os.path.join(os.path.dirname(__file__), '..')
+    zeffir_lib_path = str(udir.join('zeffir.so'))
+    err = os.system("cd '%s' && "
+            "gcc -g -I'%s' -Wall -Werror -fPIC -shared zeffir.c -o '%s'" %
+            (zeffir_dir, include_dir, zeffir_lib_path))
+    assert not err
+    #
+    sys.path.insert(0, str(udir))
+    global zeffir
+    import zeffir
+    #
+    basic_c_path = str(udir.join('basic.c'))
+    basic_lib_path = str(udir.join('libbasic.so'))
+    err = os.system("cd '%s'/test && "
+            "PYTHONPATH=../.. ../../bin/creflect basic.crx '%s' && "
+            "gcc -g -I../../creflect -fPIC -shared '%s' -o '%s'" %
+            (zeffir_dir, basic_c_path, basic_c_path, basic_lib_path))
+    assert not err
+    #
+    global ffi, lib
+    ffi, lib = zeffir.open('basic', relative_to=zeffir_lib_path)
+
+
+def test_ffi_type():
+    assert type(ffi).__module__ == 'zeffir'
+    assert type(ffi).__name__ == 'FFI'
+
+def test_forty_two():
+    assert lib.forty_two == 42
+    assert type(lib.forty_two) is int
diff --git a/zeffir/test/udir.py b/zeffir/test/udir.py
new file mode 100644
--- /dev/null
+++ b/zeffir/test/udir.py
@@ -0,0 +1,5 @@
+import py
+import sysconfig
+
+udir = py.path.local.make_numbered_dir(prefix = 'zeffir-')
+include_dir = sysconfig.get_config_var('INCLUDEPY')
diff --git a/zeffir/zeffir.c b/zeffir/zeffir.c
--- a/zeffir/zeffir.c
+++ b/zeffir/zeffir.c
@@ -6,10 +6,48 @@
 /************************************************************/
 
 typedef struct {
-    Py_OBJECT_HEAD
-    PyObject *libname;
+    PyObject_HEAD
+    char *libname;
 } PyFFIObject;
 
+static void ffi_dealloc(PyFFIObject *ffi)
+{
+    free(ffi->libname);
+    PyObject_Del(ffi);
+}
+
+static PyTypeObject FFI_Type = {
+    PyVarObject_HEAD_INIT(NULL, 0)
+    "zeffir.FFI",
+    sizeof(PyFFIObject),
+    0,
+    (destructor)ffi_dealloc,                    /* tp_dealloc */
+    0,                                          /* tp_print */
+    0,                                          /* tp_getattr */
+    0,                                          /* tp_setattr */
+    0,                                          /* tp_compare */
+    0,                                          /* tp_repr */
+    0,                                          /* tp_as_number */
+    0,                                          /* tp_as_sequence */
+    0,                                          /* tp_as_mapping */
+    0,                                          /* tp_hash */
+    0,                                          /* tp_call */
+    0,                                          /* tp_str */
+    PyObject_GenericGetAttr,                    /* tp_getattro */
+    0,                                          /* tp_setattro */
+    0,                                          /* tp_as_buffer */
+    Py_TPFLAGS_DEFAULT,                         /* tp_flags */
+    0,                                          /* tp_doc */
+    0,                                          /* tp_traverse */
+    0,                                          /* tp_clear */
+    0,                                          /* tp_richcompare */
+    0,                                          /* tp_weaklistoffset */
+    0,                                          /* tp_iter */
+    0,                                          /* tp_iternext */
+    0,                                          /* tp_methods */
+    0,                                          /* tp_members */
+};
+
 /************************************************************/
 
 static PyObject *b_open(PyObject *self, PyObject *args, PyObject *kwds)
@@ -24,8 +62,11 @@
     PyFFIObject *ffi = PyObject_New(PyFFIObject, &FFI_Type);
     if (ffi == NULL)
         return NULL;
+    ffi->libname = strdup(libname);
 
-    return Py_BuildValue("OO", ffi, Py_None);
+    PyObject *result = Py_BuildValue("OO", ffi, Py_None);
+    Py_DECREF(ffi);
+    return result;
 }
 
 
@@ -41,4 +82,6 @@
 
     m = Py_InitModule("zeffir", ZeffirMethods);
     (void)m;
+
+    PyType_Ready(&FFI_Type);
 }

From noreply at buildbot.pypy.org  Sat Nov 29 22:32:49 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Sat, 29 Nov 2014 22:32:49 +0100 (CET)
Subject: [pypy-commit] creflect default: tweak tweak tweak
Message-ID: <20141129213249.3F90B1D263B@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r126:80866ed7c824
Date: 2014-11-29 22:33 +0100
http://bitbucket.org/cffi/creflect/changeset/80866ed7c824/

Log:	tweak tweak tweak

diff --git a/zeffir/test/test_basic.py b/zeffir/test/test_basic.py
--- a/zeffir/test/test_basic.py
+++ b/zeffir/test/test_basic.py
@@ -2,6 +2,7 @@
 from udir import udir, include_dir
 
 def setup_module(mod):
+    global zeffir, zeffir_lib_path
     zeffir_dir = os.path.join(os.path.dirname(__file__), '..')
     zeffir_lib_path = str(udir.join('zeffir.so'))
     err = os.system("cd '%s' && "
@@ -10,7 +11,6 @@
     assert not err
     #
     sys.path.insert(0, str(udir))
-    global zeffir
     import zeffir
     #
     basic_c_path = str(udir.join('basic.c'))
@@ -20,15 +20,16 @@
             "gcc -g -I../../creflect -fPIC -shared '%s' -o '%s'" %
             (zeffir_dir, basic_c_path, basic_c_path, basic_lib_path))
     assert not err
-    #
-    global ffi, lib
-    ffi, lib = zeffir.open('basic', relative_to=zeffir_lib_path)
 
 
 def test_ffi_type():
+    ffi, lib = zeffir.open('basic', relative_to=zeffir_lib_path)
     assert type(ffi).__module__ == 'zeffir'
     assert type(ffi).__name__ == 'FFI'
+    assert repr(ffi) == "" % (
+        os.path.dirname(zeffir_lib_path),)
 
 def test_forty_two():
+    ffi, lib = zeffir.open('basic', relative_to=zeffir_lib_path)
     assert lib.forty_two == 42
     assert type(lib.forty_two) is int
diff --git a/zeffir/zeffir.c b/zeffir/zeffir.c
--- a/zeffir/zeffir.c
+++ b/zeffir/zeffir.c
@@ -1,4 +1,5 @@
 #include 
+#include 
 #include "../creflect/creflect.h"
 #include "../creflect/creflect_cdecl.h"
 
@@ -7,15 +8,23 @@
 
 typedef struct {
     PyObject_HEAD
+    void *dl_lib;
     char *libname;
 } PyFFIObject;
 
 static void ffi_dealloc(PyFFIObject *ffi)
 {
+    if (ffi->dl_lib != NULL)
+        dlclose(ffi->dl_lib);
     free(ffi->libname);
     PyObject_Del(ffi);
 }
 
+static PyObject *ffi_repr(PyFFIObject *ffi)
+{
+    return PyString_FromFormat("", ffi->libname);
+}
+
 static PyTypeObject FFI_Type = {
     PyVarObject_HEAD_INIT(NULL, 0)
     "zeffir.FFI",
@@ -26,7 +35,7 @@
     0,                                          /* tp_getattr */
     0,                                          /* tp_setattr */
     0,                                          /* tp_compare */
-    0,                                          /* tp_repr */
+    (reprfunc)ffi_repr,                         /* tp_repr */
     0,                                          /* tp_as_number */
     0,                                          /* tp_as_sequence */
     0,                                          /* tp_as_mapping */
@@ -54,19 +63,51 @@
 {
     char *keywords[] = {"libname", "relative_to", NULL};
     char *libname, *relative_to;
+    char *path;
 
     if (!PyArg_ParseTupleAndKeywords(args, kwds, "ss", keywords,
                                      &libname, &relative_to))
         return NULL;
 
+    char *last_slash = strrchr(relative_to, '/');
+    if (last_slash == NULL) {
+        relative_to = "./";
+        last_slash = relative_to + 1;
+    }
+    size_t n = last_slash + 1 - relative_to;
+    path = malloc(n + strlen(libname) + 16);
+    if (path == NULL)
+        return PyErr_NoMemory();
+    memcpy(path, relative_to, n);
+    strcpy(path + n, "lib");
+    strcat(path + n, libname);
+    strcat(path + n, ".so");
+
+    dlerror();   /* clear errors */
+    void *dl_lib = dlopen(path, RTLD_LAZY);
+    if (dl_lib == NULL) {
+        char *error = dlerror();
+        if (error == NULL)
+            error = "failed to open";
+        PyErr_Format(PyExc_OSError, "%s: %s", path, error);
+        goto error;
+    }
+
     PyFFIObject *ffi = PyObject_New(PyFFIObject, &FFI_Type);
     if (ffi == NULL)
-        return NULL;
-    ffi->libname = strdup(libname);
+        goto error;
+    ffi->dl_lib = dl_lib;
+    ffi->libname = path;
 
     PyObject *result = Py_BuildValue("OO", ffi, Py_None);
     Py_DECREF(ffi);
     return result;
+
+ error:
+    if (dl_lib != NULL)
+        dlclose(dl_lib);
+    free(path);
+    return NULL;
 }
 
 

From noreply at buildbot.pypy.org  Sat Nov 29 22:57:01 2014
From: noreply at buildbot.pypy.org (mattip)
Date: Sat, 29 Nov 2014 22:57:01 +0100 (CET)
Subject: [pypy-commit] pypy ufuncapi: each call resets the steps, dims
Message-ID: <20141129215701.0607A1D283E@cobra.cs.uni-duesseldorf.de>

Author: mattip 
Branch: ufuncapi
Changeset: r74762:de3415b4e9f4
Date: 2014-11-29 23:48 +0200
http://bitbucket.org/pypy/pypy/changeset/de3415b4e9f4/

Log:	each call resets the steps, dims

diff --git a/pypy/module/micronumpy/ufuncs.py b/pypy/module/micronumpy/ufuncs.py
--- a/pypy/module/micronumpy/ufuncs.py
+++ b/pypy/module/micronumpy/ufuncs.py
@@ -1338,8 +1338,8 @@
             raise oefmt(space.w_RuntimeError,
                  "set_dims_and_steps called inappropriately")
         if self.dims_steps_set:
-            raise oefmt(space.w_RuntimeError,
-                 "set_dims_and_steps called inappropriately")
+            free_raw_storage(self.dims, track_allocation=False)
+            free_raw_storage(self.steps, track_allocation=False)
         self.dims = alloc_raw_storage(LONG_SIZE * len(dims), track_allocation=False)
         self.steps = alloc_raw_storage(LONG_SIZE * len(steps), track_allocation=False)
         for i in range(len(dims)):

From noreply at buildbot.pypy.org  Sun Nov 30 01:31:19 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Sun, 30 Nov 2014 01:31:19 +0100 (CET)
Subject: [pypy-commit] creflect default: The minimum (hum) to make the
 "assert lib.forty_two == 42" pass
Message-ID: <20141130003119.1E1EF1D263B@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r127:7464b4e4b27d
Date: 2014-11-30 01:31 +0100
http://bitbucket.org/cffi/creflect/changeset/7464b4e4b27d/

Log:	The minimum (hum) to make the "assert lib.forty_two == 42" pass

diff --git a/zeffir/zeffir.c b/zeffir/zeffir.c
--- a/zeffir/zeffir.c
+++ b/zeffir/zeffir.c
@@ -1,9 +1,14 @@
-#include 
+#include "Python.h"
+#include "structmember.h"
 #include 
 #include "../creflect/creflect.h"
 #include "../creflect/creflect_cdecl.h"
 
 
+#define MAGIC_CREFLECT_TAG  0x54822e00
+#define CREFLECT_VERSION    0x01
+
+
 /************************************************************/
 
 typedef struct {
@@ -59,6 +64,281 @@
 
 /************************************************************/
 
+typedef struct {
+    PyObject_HEAD
+    PyObject *dict;
+    PyFFIObject *ffi;
+} PyLibObject;
+
+static PyMemberDef lib_members[] = {
+    {"__dict__", T_OBJECT, offsetof(PyLibObject, dict), READONLY},
+    {NULL}
+};
+
+static void lib_dealloc(PyLibObject *lib)
+{
+    Py_DECREF(lib->dict);
+    Py_DECREF(lib->ffi);
+    PyObject_Del(lib);
+}
+
+static PyObject *lib_repr(PyLibObject *lib)
+{
+    return PyString_FromFormat("",
+                               lib->ffi->libname);
+}
+
+static PyTypeObject Lib_Type = {
+    PyVarObject_HEAD_INIT(NULL, 0)
+    "zeffir.Lib",
+    sizeof(PyLibObject),
+    0,
+    (destructor)lib_dealloc,                    /* tp_dealloc */
+    0,                                          /* tp_print */
+    0,                                          /* tp_getattr */
+    0,                                          /* tp_setattr */
+    0,                                          /* tp_compare */
+    (reprfunc)lib_repr,                         /* tp_repr */
+    0,                                          /* tp_as_number */
+    0,                                          /* tp_as_sequence */
+    0,                                          /* tp_as_mapping */
+    0,                                          /* tp_hash */
+    0,                                          /* tp_call */
+    0,                                          /* tp_str */
+    PyObject_GenericGetAttr,                    /* tp_getattro */
+    0,                                          /* tp_setattro */
+    0,                                          /* tp_as_buffer */
+    Py_TPFLAGS_DEFAULT,                         /* tp_flags */
+    0,                                          /* tp_doc */
+    0,                                          /* tp_traverse */
+    0,                                          /* tp_clear */
+    0,                                          /* tp_richcompare */
+    0,                                          /* tp_weaklistoffset */
+    0,                                          /* tp_iter */
+    0,                                          /* tp_iternext */
+    0,                                          /* tp_methods */
+    lib_members,                                /* tp_members */
+    0,                                          /* tp_getset */
+    0,                                          /* tp_base */
+    0,                                          /* tp_dict */
+    0,                                          /* tp_descr_get */
+    0,                                          /* tp_descr_set */
+    offsetof(PyLibObject, dict),                /* tp_dictoffset */
+};
+
+/************************************************************/
+
+typedef struct {
+    _crx_builder_t cb;
+    PyLibObject *lib;
+    PyFFIObject *ffi;
+    PyObject *dict;
+} zeffir_builder_t;
+
+static _crx_type_t *zef_get_void_type(_crx_builder_t *cb)
+{
+    abort();
+}
+
+static _crx_type_t *zef_get_char_type(_crx_builder_t *cb)
+{
+    abort();
+}
+
+static _crx_type_t *zef_get_bool_type(_crx_builder_t *cb)
+{
+    abort();
+}
+
+static _crx_type_t *zef_get_signed_type(_crx_builder_t *cb, size_t sz,
+                                        const char *g)
+{
+    return NULL;  // XXX
+}
+
+static _crx_type_t *zef_get_unsigned_type(_crx_builder_t *cb, size_t sz,
+                                          const char *g)
+{
+    abort();
+}
+
+static _crx_type_t *zef_get_float_type(_crx_builder_t *cb, size_t sz,
+                                       const char *g)
+{
+    abort();
+}
+
+static _crx_type_t *zef_get_function_type(_crx_builder_t *cb, _crx_type_t *ret,
+                                          _crx_type_t *args[], int nargs,
+                                          _crx_trampoline1_fn trampl)
+{
+    abort();
+}
+
+static _crx_type_t *zef_get_ellipsis_function_type(_crx_builder_t *cb,
+                                                   _crx_type_t *ret,
+                                                   _crx_type_t *args[],
+                                                   int nargs)
+{
+    abort();
+}
+
+static _crx_type_t *zef_get_pointer_type(_crx_builder_t *cb, _crx_type_t *t)
+{
+    abort();
+}
+
+static _crx_type_t *zef_get_const_type(_crx_builder_t *cb, _crx_type_t *t)
+{
+    abort();
+}
+
+static _crx_type_t *zef_get_array_type(_crx_builder_t *cb, _crx_type_t *t,
+                                       size_t len)
+{
+    abort();
+}
+
+static _crx_type_t *zef_get_incomplete_array_type(_crx_builder_t *cb,
+                                                  _crx_type_t *t)
+{
+    abort();
+}
+
+static _crx_type_t *zef_get_struct_type(_crx_builder_t *cb, const char *name)
+{
+    abort();
+}
+
+static _crx_type_t *zef_get_union_type(_crx_builder_t *cb, const char *name)
+{
+    abort();
+}
+
+static _crx_type_t *zef_get_enum_type(_crx_builder_t *cb, const char *name)
+{
+    abort();
+}
+
+static _crx_type_t *zef_get_user_type(_crx_builder_t *cb, const char *name)
+{
+    abort();
+}
+
+static _crx_type_t *zef_get_unknown_type(_crx_builder_t *cb, const char *name)
+{
+    abort();
+}
+
+static void zef_complete(_crx_builder_t *cb, _crx_type_t *t,
+                         size_t sz, size_t align,
+                         _crx_field_t fields[], int nfields)
+{
+    abort();
+}
+
+static void zef_complete_enum(_crx_builder_t *cb, _crx_type_t *t,
+                              _crx_type_t *inttype)
+{
+    abort();
+}
+
+static void zef_define_type(_crx_builder_t *cb, const char *name,
+                            _crx_type_t *t)
+{
+    abort();
+}
+
+static void zef_define_var(_crx_builder_t *cb, const char *name,
+                           _crx_type_t *t, void *addr)
+{
+    abort();
+}
+
+static void zef_define_func(_crx_builder_t *cb, const char *name,
+                            _crx_type_t *ret, _crx_type_t *args[], int nargs,
+                            _crx_trampoline0_fn trampl, void *directcall)
+{
+    abort();
+}
+
+static void zef_define_num_const(_crx_builder_t *cb, const char *name,
+                                 _crx_type_t *t, _crx_num_const_t *value)
+{
+    PyObject *dict = ((zeffir_builder_t *)cb)->dict;
+
+    PyObject *x = PyInt_FromLong(value->as_int);
+    if (x == NULL)
+        return;
+
+    PyDict_SetItemString(dict, name, x);
+    Py_DECREF(x);
+}
+
+static void zef_error(_crx_builder_t *cb, const char *msg)
+{
+    abort();
+}
+
+static int load_creflect_main(PyLibObject *lib)
+{
+    const char *creflect_main = "_creflect_main";   /* XXX fixed for now */
+    int (*crxmain)(_crx_builder_t *);
+    crxmain = (int(*)(_crx_builder_t *))dlsym(lib->ffi->dl_lib, creflect_main);
+    if (crxmain == NULL) {
+        PyErr_Format(PyExc_OSError, "%s: symbol '%s' not found",
+                     lib->ffi->libname, creflect_main);
+        return -1;
+    }
+
+    int result = crxmain(NULL);
+    if ((result & ~0xFF) != MAGIC_CREFLECT_TAG) {
+        PyErr_Format(PyExc_OSError, "%s: %s() is not a creflect entry point",
+                     lib->ffi->libname, creflect_main);
+        return -1;
+    }
+    if ((result & 0xFF) != CREFLECT_VERSION) {
+        PyErr_Format(PyExc_OSError, "%s was made with creflect version %d, "
+                     "expected %d", lib->ffi->libname, result & 0xFF,
+                     CREFLECT_VERSION);
+        return -1;
+    }
+
+    zeffir_builder_t builder = {
+        {
+            zef_get_void_type,
+            zef_get_char_type,
+            zef_get_bool_type,
+            zef_get_signed_type,
+            zef_get_unsigned_type,
+            zef_get_float_type,
+            zef_get_function_type,
+            zef_get_ellipsis_function_type,
+            zef_get_pointer_type,
+            zef_get_const_type,
+            zef_get_array_type,
+            zef_get_incomplete_array_type,
+            zef_get_struct_type,
+            zef_get_union_type,
+            zef_get_enum_type,
+            zef_get_user_type,
+            zef_get_unknown_type,
+            zef_complete,
+            zef_complete_enum,
+            zef_define_type,
+            zef_define_var,
+            zef_define_func,
+            zef_define_num_const,
+            zef_error,
+        },
+        lib,        /* lib */
+        lib->ffi,   /* ffi */
+        lib->dict,  /* dict */
+    };
+    crxmain(&builder.cb);
+    return PyErr_Occurred() ? -1 : 0;
+}
+
 static PyObject *b_open(PyObject *self, PyObject *args, PyObject *kwds)
 {
     char *keywords[] = {"libname", "relative_to", NULL};
@@ -83,6 +363,10 @@
     strcat(path + n, libname);
     strcat(path + n, ".so");
 
+    PyFFIObject *ffi = NULL;
+    PyLibObject *lib = NULL;
+    PyObject *dict = NULL;
+
     dlerror();   /* clear errors */
     void *dl_lib = dlopen(path, RTLD_LAZY);
     if (dl_lib == NULL) {
@@ -93,17 +377,33 @@
         goto error;
     }
 
-    PyFFIObject *ffi = PyObject_New(PyFFIObject, &FFI_Type);
+    ffi = PyObject_New(PyFFIObject, &FFI_Type);
     if (ffi == NULL)
         goto error;
-    ffi->dl_lib = dl_lib;
-    ffi->libname = path;
+    ffi->dl_lib = dl_lib;  dl_lib = NULL;
+    ffi->libname = path;   path = NULL;
 
-    PyObject *result = Py_BuildValue("OO", ffi, Py_None);
-    Py_DECREF(ffi);
+    dict = PyDict_New();
+    if (dict == NULL)
+        goto error;
+
+    lib = PyObject_New(PyLibObject, &Lib_Type);
+    if (lib == NULL)
+        goto error;
+    lib->ffi = ffi;    ffi = NULL;
+    lib->dict = dict;  dict = NULL;
+
+    if (load_creflect_main(lib) < 0)
+        goto error;
+
+    PyObject *result = Py_BuildValue("OO", lib->ffi, lib);
+    Py_DECREF(lib);
     return result;
 
  error:
+    Py_XDECREF(dict);
+    Py_XDECREF(lib);
+    Py_XDECREF(ffi);
     if (dl_lib != NULL)
         dlclose(dl_lib);
     free(path);
@@ -125,4 +425,5 @@
     (void)m;
 
     PyType_Ready(&FFI_Type);
+    PyType_Ready(&Lib_Type);
 }

From noreply at buildbot.pypy.org  Sun Nov 30 01:38:57 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Sun, 30 Nov 2014 01:38:57 +0100 (CET)
Subject: [pypy-commit] buildbot default: Disable the 'ppc-jit-backend'
 nightly builder that keeps checking the
Message-ID: <20141130003857.EB9B41D2837@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r923:1b3287625da6
Date: 2014-11-30 01:39 +0100
http://bitbucket.org/pypy/buildbot/changeset/1b3287625da6/

Log:	Disable the 'ppc-jit-backend' nightly builder that keeps checking
	the same January 2013 revision over and over and over

diff --git a/bot2/pypybuildbot/master.py b/bot2/pypybuildbot/master.py
--- a/bot2/pypybuildbot/master.py
+++ b/bot2/pypybuildbot/master.py
@@ -240,9 +240,11 @@
             APPLVLLINUX64,             # on allegro64, uses 1 core
             ], branch="py3k", hour=2, minute=0),
 
-        Nightly("nighly-ppc", [
-            JITONLYLINUXPPC64,         # on gcc1
-            ], branch='ppc-jit-backend', hour=1, minute=0),
+        # this one has faithfully run every night even though the latest
+        # change to that branch was in January 2013.  Re-enable one day.
+        #Nightly("nighly-ppc", [
+        #    JITONLYLINUXPPC64,         # on gcc1
+        #    ], branch='ppc-jit-backend', hour=1, minute=0),
 
         CustomForceScheduler('Force Scheduler',
             builderNames=[

From noreply at buildbot.pypy.org  Sun Nov 30 11:15:21 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Sun, 30 Nov 2014 11:15:21 +0100 (CET)
Subject: [pypy-commit] creflect default: Split zeffir.c into pieces,
 to avoid ending up like _cffi_backend.c
Message-ID: <20141130101521.AE3A71C0FFF@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r128:25439df4dc03
Date: 2014-11-30 11:15 +0100
http://bitbucket.org/cffi/creflect/changeset/25439df4dc03/

Log:	Split zeffir.c into pieces, to avoid ending up like _cffi_backend.c

diff --git a/zeffir/test/test_basic.py b/zeffir/test/test_basic.py
--- a/zeffir/test/test_basic.py
+++ b/zeffir/test/test_basic.py
@@ -33,3 +33,7 @@
     ffi, lib = zeffir.open('basic', relative_to=zeffir_lib_path)
     assert lib.forty_two == 42
     assert type(lib.forty_two) is int
+
+def test_dir():
+    ffi, lib = zeffir.open('basic', relative_to=zeffir_lib_path)
+    assert dir(lib) == ['forty_two']
diff --git a/zeffir/zef_builder.c b/zeffir/zef_builder.c
new file mode 100644
--- /dev/null
+++ b/zeffir/zef_builder.c
@@ -0,0 +1,248 @@
+
+#define MAGIC_CREFLECT_TAG  0x54822e00
+#define CREFLECT_VERSION    0x01
+
+typedef struct {
+    _crx_builder_t cb;
+    PyLibObject *lib;
+    PyFFIObject *ffi;
+    PyObject *dict;
+} zeffir_builder_t;
+
+static PyObject *zef_cache_primitive_signed;
+
+static _crx_type_t *zef_get_void_type(_crx_builder_t *cb)
+{
+    abort();
+}
+
+static _crx_type_t *zef_get_char_type(_crx_builder_t *cb)
+{
+    abort();
+}
+
+static _crx_type_t *zef_get_bool_type(_crx_builder_t *cb)
+{
+    abort();
+}
+
+static _crx_type_t *zef_get_signed_type(_crx_builder_t *cb, size_t sz,
+                                        const char *name)
+{
+    PyObject *x = PyDict_GetItemString(zef_cache_primitive_signed, name);
+    CTypeDescrObject *td = (CTypeDescrObject *)x;
+
+    if (td == NULL) {
+        int err;
+        size_t name_length = strlen(name);
+
+        td = ctypedescr_new(name_length + 1);
+        if (td == NULL)
+            return NULL;
+
+        memcpy(td->ct_name, name, name_length + 1);
+        td->ct_name_position = name_length;
+        td->ct_size = sz;
+        //td->ct_length = ptypes->align;
+        //td->ct_extra = ffitype;
+        td->ct_flags = CT_PRIMITIVE_SIGNED;
+        if (td->ct_size <= sizeof(long))
+            td->ct_flags |= CT_PRIMITIVE_FITS_LONG;
+
+        x = (PyObject *)td;
+        err = PyDict_SetItemString(zef_cache_primitive_signed, name, x);
+        Py_DECREF(x);
+        if (err < 0)
+            return NULL;
+    }
+    return td;
+}
+
+static _crx_type_t *zef_get_unsigned_type(_crx_builder_t *cb, size_t sz,
+                                          const char *g)
+{
+    abort();
+}
+
+static _crx_type_t *zef_get_float_type(_crx_builder_t *cb, size_t sz,
+                                       const char *g)
+{
+    abort();
+}
+
+static _crx_type_t *zef_get_function_type(_crx_builder_t *cb, _crx_type_t *ret,
+                                          _crx_type_t *args[], int nargs,
+                                          _crx_trampoline1_fn trampl)
+{
+    abort();
+}
+
+static _crx_type_t *zef_get_ellipsis_function_type(_crx_builder_t *cb,
+                                                   _crx_type_t *ret,
+                                                   _crx_type_t *args[],
+                                                   int nargs)
+{
+    abort();
+}
+
+static _crx_type_t *zef_get_pointer_type(_crx_builder_t *cb, _crx_type_t *t)
+{
+    abort();
+}
+
+static _crx_type_t *zef_get_const_type(_crx_builder_t *cb, _crx_type_t *t)
+{
+    abort();
+}
+
+static _crx_type_t *zef_get_array_type(_crx_builder_t *cb, _crx_type_t *t,
+                                       size_t len)
+{
+    abort();
+}
+
+static _crx_type_t *zef_get_incomplete_array_type(_crx_builder_t *cb,
+                                                  _crx_type_t *t)
+{
+    abort();
+}
+
+static _crx_type_t *zef_get_struct_type(_crx_builder_t *cb, const char *name)
+{
+    abort();
+}
+
+static _crx_type_t *zef_get_union_type(_crx_builder_t *cb, const char *name)
+{
+    abort();
+}
+
+static _crx_type_t *zef_get_enum_type(_crx_builder_t *cb, const char *name)
+{
+    abort();
+}
+
+static _crx_type_t *zef_get_user_type(_crx_builder_t *cb, const char *name)
+{
+    abort();
+}
+
+static _crx_type_t *zef_get_unknown_type(_crx_builder_t *cb, const char *name)
+{
+    abort();
+}
+
+static void zef_complete(_crx_builder_t *cb, _crx_type_t *t,
+                         size_t sz, size_t align,
+                         _crx_field_t fields[], int nfields)
+{
+    abort();
+}
+
+static void zef_complete_enum(_crx_builder_t *cb, _crx_type_t *t,
+                              _crx_type_t *inttype)
+{
+    abort();
+}
+
+static void zef_define_type(_crx_builder_t *cb, const char *name,
+                            _crx_type_t *t)
+{
+    abort();
+}
+
+static void zef_define_var(_crx_builder_t *cb, const char *name,
+                           _crx_type_t *t, void *addr)
+{
+    abort();
+}
+
+static void zef_define_func(_crx_builder_t *cb, const char *name,
+                            _crx_type_t *ret, _crx_type_t *args[], int nargs,
+                            _crx_trampoline0_fn trampl, void *directcall)
+{
+    abort();
+}
+
+static void zef_define_num_const(_crx_builder_t *cb, const char *name,
+                                 _crx_type_t *t, _crx_num_const_t *value)
+{
+    PyObject *dict = ((zeffir_builder_t *)cb)->dict;
+
+    PyObject *x = PyInt_FromLong(value->as_int);
+    if (x == NULL)
+        return;
+
+    PyDict_SetItemString(dict, name, x);
+    Py_DECREF(x);
+}
+
+static void zef_error(_crx_builder_t *cb, const char *msg)
+{
+    abort();
+}
+
+static int load_creflect_main(PyLibObject *lib)
+{
+    const char *creflect_main = "_creflect_main";   /* XXX fixed for now */
+    int (*crxmain)(_crx_builder_t *);
+    crxmain = (int(*)(_crx_builder_t *))dlsym(lib->ffi->dl_lib, creflect_main);
+    if (crxmain == NULL) {
+        PyErr_Format(PyExc_OSError, "%s: symbol '%s' not found",
+                     lib->ffi->libname, creflect_main);
+        return -1;
+    }
+
+    int result = crxmain(NULL);
+    if ((result & ~0xFF) != MAGIC_CREFLECT_TAG) {
+        PyErr_Format(PyExc_OSError, "%s: %s() is not a creflect entry point",
+                     lib->ffi->libname, creflect_main);
+        return -1;
+    }
+    if ((result & 0xFF) != CREFLECT_VERSION) {
+        PyErr_Format(PyExc_OSError, "%s was made with creflect version %d, "
+                     "expected %d", lib->ffi->libname, result & 0xFF,
+                     CREFLECT_VERSION);
+        return -1;
+    }
+
+    zeffir_builder_t builder = {
+        {
+            zef_get_void_type,
+            zef_get_char_type,
+            zef_get_bool_type,
+            zef_get_signed_type,
+            zef_get_unsigned_type,
+            zef_get_float_type,
+            zef_get_function_type,
+            zef_get_ellipsis_function_type,
+            zef_get_pointer_type,
+            zef_get_const_type,
+            zef_get_array_type,
+            zef_get_incomplete_array_type,
+            zef_get_struct_type,
+            zef_get_union_type,
+            zef_get_enum_type,
+            zef_get_user_type,
+            zef_get_unknown_type,
+            zef_complete,
+            zef_complete_enum,
+            zef_define_type,
+            zef_define_var,
+            zef_define_func,
+            zef_define_num_const,
+            zef_error,
+        },
+        lib,        /* lib */
+        lib->ffi,   /* ffi */
+        lib->dict,  /* dict */
+    };
+    crxmain(&builder.cb);
+    return PyErr_Occurred() ? -1 : 0;
+}
+
+static void init_caches(void)
+{
+    if ((zef_cache_primitive_signed = PyDict_New()) == NULL)
+        return;
+}
diff --git a/zeffir/zef_ctype.c b/zeffir/zef_ctype.c
new file mode 100644
--- /dev/null
+++ b/zeffir/zef_ctype.c
@@ -0,0 +1,141 @@
+
+/* base type flag: exactly one of the following: */
+#define CT_PRIMITIVE_SIGNED   1    /* signed integer */
+#define CT_PRIMITIVE_UNSIGNED 2    /* unsigned integer, bool */
+#define CT_PRIMITIVE_CHAR     4    /* char */
+#define CT_PRIMITIVE_FLOAT    8    /* float, double, long double */
+#define CT_POINTER           16    /* pointer, excluding ptr-to-func */
+#define CT_ARRAY             32    /* array */
+#define CT_STRUCT            64    /* struct */
+#define CT_UNION            128    /* union */
+#define CT_FUNCTIONPTR      256    /* pointer to function */
+#define CT_VOID             512    /* void */
+
+/* other flags that may also be set in addition to the base flag: */
+#define CT_CAST_ANYTHING         1024    /* 'char *' and 'void *' only */
+#define CT_PRIMITIVE_FITS_LONG   2048
+#define CT_IS_OPAQUE             4096
+#define CT_IS_ENUM               8192
+#define CT_IS_PTR_TO_OWNED      16384
+#define CT_IS_LONGDOUBLE        65536
+#define CT_IS_BOOL             131072
+#define CT_IS_VOID_PTR         524288
+#define CT_WITH_VAR_ARRAY     1048576
+#define CT_PRIMITIVE_ANY  (CT_PRIMITIVE_SIGNED |        \
+                           CT_PRIMITIVE_UNSIGNED |      \
+                           CT_PRIMITIVE_CHAR |          \
+                           CT_PRIMITIVE_FLOAT)
+
+struct _crx_type_s {
+    PyObject_VAR_HEAD
+
+    CTypeDescrObject *ct_itemdescr;    /* ptrs and arrays: the item type */
+    PyObject *ct_stuff;                /* structs: dict of the fields
+                                          arrays: ctypedescr of the ptr type
+                                          function: tuple(abi, ctres, ctargs..)
+                                          enum: pair {"name":x},{x:"name"}
+                                          ptrs: lazily, ctypedescr of array */
+    //void *ct_extra;                    /* structs: first field (not a ref!)
+    //                                      function types: cif_description
+    //                                      primitives: prebuilt "cif" object */
+
+    PyObject *ct_weakreflist;    /* weakref support */
+
+    Py_ssize_t ct_size;     /* size of instances, or -1 if unknown */
+    Py_ssize_t ct_length;   /* length of arrays, or -1 if unknown */
+    int ct_flags;           /* CT_xxx flags */
+
+    int ct_name_position;   /* index in ct_name of where to put a var name */
+    char ct_name[1];        /* string, e.g. "int *" for pointers to ints */
+};
+
+static PyTypeObject CTypeDescr_Type;
+//static PyTypeObject CField_Type;
+//static PyTypeObject CData_Type;
+//static PyTypeObject CDataOwning_Type;
+//static PyTypeObject CDataOwningGC_Type;
+
+#define CTypeDescr_Check(ob)  (Py_TYPE(ob) == &CTypeDescr_Type)
+#define CData_Check(ob)       (Py_TYPE(ob) == &CData_Type ||            \
+                               Py_TYPE(ob) == &CDataOwning_Type ||      \
+                               Py_TYPE(ob) == &CDataOwningGC_Type)
+#define CDataOwn_Check(ob)    (Py_TYPE(ob) == &CDataOwning_Type ||      \
+                               Py_TYPE(ob) == &CDataOwningGC_Type)
+
+/************************************************************/
+
+static CTypeDescrObject *ctypedescr_new(size_t name_size)
+{
+    CTypeDescrObject *ct = PyObject_GC_NewVar(CTypeDescrObject,
+                                              &CTypeDescr_Type,
+                                              name_size);
+    if (ct == NULL)
+        return NULL;
+
+    ct->ct_itemdescr = NULL;
+    ct->ct_stuff = NULL;
+    ct->ct_weakreflist = NULL;
+    ct->ct_size = -1;
+    ct->ct_length = -1;
+    PyObject_GC_Track(ct);
+    return ct;
+}
+
+static void ctypedescr_dealloc(CTypeDescrObject *ct)
+{
+    PyObject_GC_UnTrack(ct);
+    if (ct->ct_weakreflist != NULL)
+        PyObject_ClearWeakRefs((PyObject *) ct);
+    Py_XDECREF(ct->ct_itemdescr);
+    Py_XDECREF(ct->ct_stuff);
+    //if (ct->ct_flags & CT_FUNCTIONPTR)
+    //    PyObject_Free(ct->ct_extra);
+    Py_TYPE(ct)->tp_free((PyObject *)ct);
+}
+
+static int ctypedescr_traverse(CTypeDescrObject *ct, visitproc visit, void *arg)
+{
+    Py_VISIT(ct->ct_itemdescr);
+    Py_VISIT(ct->ct_stuff);
+    return 0;
+}
+
+static int ctypedescr_clear(CTypeDescrObject *ct)
+{
+    Py_CLEAR(ct->ct_itemdescr);
+    Py_CLEAR(ct->ct_stuff);
+    return 0;
+}
+
+static PyTypeObject CTypeDescr_Type = {
+    PyVarObject_HEAD_INIT(NULL, 0)
+    "_cffi_backend.CTypeDescr",
+    offsetof(CTypeDescrObject, ct_name),
+    sizeof(char),
+    (destructor)ctypedescr_dealloc,             /* tp_dealloc */
+    0,                                          /* tp_print */
+    0,                                          /* tp_getattr */
+    0,                                          /* tp_setattr */
+    0,                                          /* tp_compare */
+    0,//(reprfunc)ctypedescr_repr,                  /* tp_repr */
+    0,                                          /* tp_as_number */
+    0,                                          /* tp_as_sequence */
+    0,                                          /* tp_as_mapping */
+    0,                                          /* tp_hash */
+    0,                                          /* tp_call */
+    0,                                          /* tp_str */
+    PyObject_GenericGetAttr,                    /* tp_getattro */
+    0,                                          /* tp_setattro */
+    0,                                          /* tp_as_buffer */
+    Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC,    /* tp_flags */
+    0,                                          /* tp_doc */
+    (traverseproc)ctypedescr_traverse,          /* tp_traverse */
+    (inquiry)ctypedescr_clear,                  /* tp_clear */
+    0,                                          /* tp_richcompare */
+    offsetof(CTypeDescrObject, ct_weakreflist), /* tp_weaklistoffset */
+    0,                                          /* tp_iter */
+    0,                                          /* tp_iternext */
+    0,//ctypedescr_methods,                         /* tp_methods */
+    0,                                          /* tp_members */
+    0,//ctypedescr_getsets,                         /* tp_getset */
+};
diff --git a/zeffir/zef_ffi_obj.c b/zeffir/zef_ffi_obj.c
new file mode 100644
--- /dev/null
+++ b/zeffir/zef_ffi_obj.c
@@ -0,0 +1,51 @@
+
+struct PyFFIObject_s {
+    PyObject_HEAD
+    void *dl_lib;
+    char *libname;
+};
+
+static void ffi_dealloc(PyFFIObject *ffi)
+{
+    if (ffi->dl_lib != NULL)
+        dlclose(ffi->dl_lib);
+    free(ffi->libname);
+    PyObject_Del(ffi);
+}
+
+static PyObject *ffi_repr(PyFFIObject *ffi)
+{
+    return PyString_FromFormat("", ffi->libname);
+}
+
+static PyTypeObject FFI_Type = {
+    PyVarObject_HEAD_INIT(NULL, 0)
+    "zeffir.FFI",
+    sizeof(PyFFIObject),
+    0,
+    (destructor)ffi_dealloc,                    /* tp_dealloc */
+    0,                                          /* tp_print */
+    0,                                          /* tp_getattr */
+    0,                                          /* tp_setattr */
+    0,                                          /* tp_compare */
+    (reprfunc)ffi_repr,                         /* tp_repr */
+    0,                                          /* tp_as_number */
+    0,                                          /* tp_as_sequence */
+    0,                                          /* tp_as_mapping */
+    0,                                          /* tp_hash */
+    0,                                          /* tp_call */
+    0,                                          /* tp_str */
+    PyObject_GenericGetAttr,                    /* tp_getattro */
+    0,                                          /* tp_setattro */
+    0,                                          /* tp_as_buffer */
+    Py_TPFLAGS_DEFAULT,                         /* tp_flags */
+    0,                                          /* tp_doc */
+    0,                                          /* tp_traverse */
+    0,                                          /* tp_clear */
+    0,                                          /* tp_richcompare */
+    0,                                          /* tp_weaklistoffset */
+    0,                                          /* tp_iter */
+    0,                                          /* tp_iternext */
+    0,                                          /* tp_methods */
+    0,                                          /* tp_members */
+};
diff --git a/zeffir/zef_funcs.c b/zeffir/zef_funcs.c
new file mode 100644
--- /dev/null
+++ b/zeffir/zef_funcs.c
@@ -0,0 +1,77 @@
+
+static PyObject *b_open(PyObject *self, PyObject *args, PyObject *kwds)
+{
+    char *keywords[] = {"libname", "relative_to", NULL};
+    char *libname, *relative_to;
+    char *path;
+
+    if (!PyArg_ParseTupleAndKeywords(args, kwds, "ss", keywords,
+                                     &libname, &relative_to))
+        return NULL;
+
+    char *last_slash = strrchr(relative_to, '/');
+    if (last_slash == NULL) {
+        relative_to = "./";
+        last_slash = relative_to + 1;
+    }
+    size_t n = last_slash + 1 - relative_to;
+    path = malloc(n + strlen(libname) + 16);
+    if (path == NULL)
+        return PyErr_NoMemory();
+    memcpy(path, relative_to, n);
+    strcpy(path + n, "lib");
+    strcat(path + n, libname);
+    strcat(path + n, ".so");
+
+    PyFFIObject *ffi = NULL;
+    PyLibObject *lib = NULL;
+    PyObject *dict = NULL;
+
+    dlerror();   /* clear errors */
+    void *dl_lib = dlopen(path, RTLD_LAZY);
+    if (dl_lib == NULL) {
+        char *error = dlerror();
+        if (error == NULL)
+            error = "failed to open";
+        PyErr_Format(PyExc_OSError, "%s: %s", path, error);
+        goto error;
+    }
+
+    ffi = PyObject_New(PyFFIObject, &FFI_Type);
+    if (ffi == NULL)
+        goto error;
+    ffi->dl_lib = dl_lib;  dl_lib = NULL;
+    ffi->libname = path;   path = NULL;
+
+    dict = PyDict_New();
+    if (dict == NULL)
+        goto error;
+
+    lib = PyObject_New(PyLibObject, &Lib_Type);
+    if (lib == NULL)
+        goto error;
+    lib->ffi = ffi;    ffi = NULL;
+    lib->dict = dict;  dict = NULL;
+
+    if (load_creflect_main(lib) < 0)
+        goto error;
+
+    PyObject *result = Py_BuildValue("OO", lib->ffi, lib);
+    Py_DECREF(lib);
+    return result;
+
+ error:
+    Py_XDECREF(dict);
+    Py_XDECREF(lib);
+    Py_XDECREF(ffi);
+    if (dl_lib != NULL)
+        dlclose(dl_lib);
+    free(path);
+    return NULL;
+}
+
+
+static PyMethodDef ZeffirMethods[] = {
+    {"open",   (PyCFunction)b_open, METH_KEYWORDS},
+    {NULL,     NULL}    /* Sentinel */
+};
diff --git a/zeffir/zef_lib_obj.c b/zeffir/zef_lib_obj.c
new file mode 100644
--- /dev/null
+++ b/zeffir/zef_lib_obj.c
@@ -0,0 +1,72 @@
+
+struct PyLibObject_s {
+    PyObject_HEAD
+    PyObject *dict;
+    PyFFIObject *ffi;
+};
+
+static void lib_dealloc(PyLibObject *lib)
+{
+    Py_DECREF(lib->dict);
+    Py_DECREF(lib->ffi);
+    PyObject_Del(lib);
+}
+
+static PyObject *lib_repr(PyLibObject *lib)
+{
+    return PyString_FromFormat("",
+                               lib->ffi->libname);
+}
+
+static PyObject *lib_dir(PyObject *lib, PyObject *noarg)
+{
+    return PyDict_Keys(((PyLibObject *)lib)->dict);
+}
+
+static PyMethodDef lib_methods[] = {
+    {"__dir__",   lib_dir,  METH_NOARGS},
+    {NULL,        NULL}           /* sentinel */
+};
+
+static PyMemberDef lib_members[] = {
+    {"__dict__", T_OBJECT, offsetof(PyLibObject, dict), READONLY},
+    {NULL}
+};
+
+static PyTypeObject Lib_Type = {
+    PyVarObject_HEAD_INIT(NULL, 0)
+    "zeffir.Lib",
+    sizeof(PyLibObject),
+    0,
+    (destructor)lib_dealloc,                    /* tp_dealloc */
+    0,                                          /* tp_print */
+    0,                                          /* tp_getattr */
+    0,                                          /* tp_setattr */
+    0,                                          /* tp_compare */
+    (reprfunc)lib_repr,                         /* tp_repr */
+    0,                                          /* tp_as_number */
+    0,                                          /* tp_as_sequence */
+    0,                                          /* tp_as_mapping */
+    0,                                          /* tp_hash */
+    0,                                          /* tp_call */
+    0,                                          /* tp_str */
+    PyObject_GenericGetAttr,                    /* tp_getattro */
+    0,                                          /* tp_setattro */
+    0,                                          /* tp_as_buffer */
+    Py_TPFLAGS_DEFAULT,                         /* tp_flags */
+    0,                                          /* tp_doc */
+    0,                                          /* tp_traverse */
+    0,                                          /* tp_clear */
+    0,                                          /* tp_richcompare */
+    0,                                          /* tp_weaklistoffset */
+    0,                                          /* tp_iter */
+    0,                                          /* tp_iternext */
+    lib_methods,                                /* tp_methods */
+    lib_members,                                /* tp_members */
+    0,                                          /* tp_getset */
+    0,                                          /* tp_base */
+    0,                                          /* tp_dict */
+    0,                                          /* tp_descr_get */
+    0,                                          /* tp_descr_set */
+    offsetof(PyLibObject, dict),                /* tp_dictoffset */
+};
diff --git a/zeffir/zeffir.c b/zeffir/zeffir.c
--- a/zeffir/zeffir.c
+++ b/zeffir/zeffir.c
@@ -5,417 +5,23 @@
 #include "../creflect/creflect_cdecl.h"
 
 
-#define MAGIC_CREFLECT_TAG  0x54822e00
-#define CREFLECT_VERSION    0x01
+typedef struct _crx_type_s CTypeDescrObject;
+typedef struct PyLibObject_s PyLibObject;
+typedef struct PyFFIObject_s PyFFIObject;
 
 
 /************************************************************/
+/* Works by including all other .c files.                   */
+/* Allows all function and global symbols to remain static. */
 
-typedef struct {
-    PyObject_HEAD
-    void *dl_lib;
-    char *libname;
-} PyFFIObject;
-
-static void ffi_dealloc(PyFFIObject *ffi)
-{
-    if (ffi->dl_lib != NULL)
-        dlclose(ffi->dl_lib);
-    free(ffi->libname);
-    PyObject_Del(ffi);
-}
-
-static PyObject *ffi_repr(PyFFIObject *ffi)
-{
-    return PyString_FromFormat("", ffi->libname);
-}
-
-static PyTypeObject FFI_Type = {
-    PyVarObject_HEAD_INIT(NULL, 0)
-    "zeffir.FFI",
-    sizeof(PyFFIObject),
-    0,
-    (destructor)ffi_dealloc,                    /* tp_dealloc */
-    0,                                          /* tp_print */
-    0,                                          /* tp_getattr */
-    0,                                          /* tp_setattr */
-    0,                                          /* tp_compare */
-    (reprfunc)ffi_repr,                         /* tp_repr */
-    0,                                          /* tp_as_number */
-    0,                                          /* tp_as_sequence */
-    0,                                          /* tp_as_mapping */
-    0,                                          /* tp_hash */
-    0,                                          /* tp_call */
-    0,                                          /* tp_str */
-    PyObject_GenericGetAttr,                    /* tp_getattro */
-    0,                                          /* tp_setattro */
-    0,                                          /* tp_as_buffer */
-    Py_TPFLAGS_DEFAULT,                         /* tp_flags */
-    0,                                          /* tp_doc */
-    0,                                          /* tp_traverse */
-    0,                                          /* tp_clear */
-    0,                                          /* tp_richcompare */
-    0,                                          /* tp_weaklistoffset */
-    0,                                          /* tp_iter */
-    0,                                          /* tp_iternext */
-    0,                                          /* tp_methods */
-    0,                                          /* tp_members */
-};
+#include "zef_ctype.c"
+#include "zef_ffi_obj.c"
+#include "zef_lib_obj.c"
+#include "zef_builder.c"
+#include "zef_funcs.c"
 
 /************************************************************/
 
-typedef struct {
-    PyObject_HEAD
-    PyObject *dict;
-    PyFFIObject *ffi;
-} PyLibObject;
-
-static PyMemberDef lib_members[] = {
-    {"__dict__", T_OBJECT, offsetof(PyLibObject, dict), READONLY},
-    {NULL}
-};
-
-static void lib_dealloc(PyLibObject *lib)
-{
-    Py_DECREF(lib->dict);
-    Py_DECREF(lib->ffi);
-    PyObject_Del(lib);
-}
-
-static PyObject *lib_repr(PyLibObject *lib)
-{
-    return PyString_FromFormat("",
-                               lib->ffi->libname);
-}
-
-static PyTypeObject Lib_Type = {
-    PyVarObject_HEAD_INIT(NULL, 0)
-    "zeffir.Lib",
-    sizeof(PyLibObject),
-    0,
-    (destructor)lib_dealloc,                    /* tp_dealloc */
-    0,                                          /* tp_print */
-    0,                                          /* tp_getattr */
-    0,                                          /* tp_setattr */
-    0,                                          /* tp_compare */
-    (reprfunc)lib_repr,                         /* tp_repr */
-    0,                                          /* tp_as_number */
-    0,                                          /* tp_as_sequence */
-    0,                                          /* tp_as_mapping */
-    0,                                          /* tp_hash */
-    0,                                          /* tp_call */
-    0,                                          /* tp_str */
-    PyObject_GenericGetAttr,                    /* tp_getattro */
-    0,                                          /* tp_setattro */
-    0,                                          /* tp_as_buffer */
-    Py_TPFLAGS_DEFAULT,                         /* tp_flags */
-    0,                                          /* tp_doc */
-    0,                                          /* tp_traverse */
-    0,                                          /* tp_clear */
-    0,                                          /* tp_richcompare */
-    0,                                          /* tp_weaklistoffset */
-    0,                                          /* tp_iter */
-    0,                                          /* tp_iternext */
-    0,                                          /* tp_methods */
-    lib_members,                                /* tp_members */
-    0,                                          /* tp_getset */
-    0,                                          /* tp_base */
-    0,                                          /* tp_dict */
-    0,                                          /* tp_descr_get */
-    0,                                          /* tp_descr_set */
-    offsetof(PyLibObject, dict),                /* tp_dictoffset */
-};
-
-/************************************************************/
-
-typedef struct {
-    _crx_builder_t cb;
-    PyLibObject *lib;
-    PyFFIObject *ffi;
-    PyObject *dict;
-} zeffir_builder_t;
-
-static _crx_type_t *zef_get_void_type(_crx_builder_t *cb)
-{
-    abort();
-}
-
-static _crx_type_t *zef_get_char_type(_crx_builder_t *cb)
-{
-    abort();
-}
-
-static _crx_type_t *zef_get_bool_type(_crx_builder_t *cb)
-{
-    abort();
-}
-
-static _crx_type_t *zef_get_signed_type(_crx_builder_t *cb, size_t sz,
-                                        const char *g)
-{
-    return NULL;  // XXX
-}
-
-static _crx_type_t *zef_get_unsigned_type(_crx_builder_t *cb, size_t sz,
-                                          const char *g)
-{
-    abort();
-}
-
-static _crx_type_t *zef_get_float_type(_crx_builder_t *cb, size_t sz,
-                                       const char *g)
-{
-    abort();
-}
-
-static _crx_type_t *zef_get_function_type(_crx_builder_t *cb, _crx_type_t *ret,
-                                          _crx_type_t *args[], int nargs,
-                                          _crx_trampoline1_fn trampl)
-{
-    abort();
-}
-
-static _crx_type_t *zef_get_ellipsis_function_type(_crx_builder_t *cb,
-                                                   _crx_type_t *ret,
-                                                   _crx_type_t *args[],
-                                                   int nargs)
-{
-    abort();
-}
-
-static _crx_type_t *zef_get_pointer_type(_crx_builder_t *cb, _crx_type_t *t)
-{
-    abort();
-}
-
-static _crx_type_t *zef_get_const_type(_crx_builder_t *cb, _crx_type_t *t)
-{
-    abort();
-}
-
-static _crx_type_t *zef_get_array_type(_crx_builder_t *cb, _crx_type_t *t,
-                                       size_t len)
-{
-    abort();
-}
-
-static _crx_type_t *zef_get_incomplete_array_type(_crx_builder_t *cb,
-                                                  _crx_type_t *t)
-{
-    abort();
-}
-
-static _crx_type_t *zef_get_struct_type(_crx_builder_t *cb, const char *name)
-{
-    abort();
-}
-
-static _crx_type_t *zef_get_union_type(_crx_builder_t *cb, const char *name)
-{
-    abort();
-}
-
-static _crx_type_t *zef_get_enum_type(_crx_builder_t *cb, const char *name)
-{
-    abort();
-}
-
-static _crx_type_t *zef_get_user_type(_crx_builder_t *cb, const char *name)
-{
-    abort();
-}
-
-static _crx_type_t *zef_get_unknown_type(_crx_builder_t *cb, const char *name)
-{
-    abort();
-}
-
-static void zef_complete(_crx_builder_t *cb, _crx_type_t *t,
-                         size_t sz, size_t align,
-                         _crx_field_t fields[], int nfields)
-{
-    abort();
-}
-
-static void zef_complete_enum(_crx_builder_t *cb, _crx_type_t *t,
-                              _crx_type_t *inttype)
-{
-    abort();
-}
-
-static void zef_define_type(_crx_builder_t *cb, const char *name,
-                            _crx_type_t *t)
-{
-    abort();
-}
-
-static void zef_define_var(_crx_builder_t *cb, const char *name,
-                           _crx_type_t *t, void *addr)
-{
-    abort();
-}
-
-static void zef_define_func(_crx_builder_t *cb, const char *name,
-                            _crx_type_t *ret, _crx_type_t *args[], int nargs,
-                            _crx_trampoline0_fn trampl, void *directcall)
-{
-    abort();
-}
-
-static void zef_define_num_const(_crx_builder_t *cb, const char *name,
-                                 _crx_type_t *t, _crx_num_const_t *value)
-{
-    PyObject *dict = ((zeffir_builder_t *)cb)->dict;
-
-    PyObject *x = PyInt_FromLong(value->as_int);
-    if (x == NULL)
-        return;
-
-    PyDict_SetItemString(dict, name, x);
-    Py_DECREF(x);
-}
-
-static void zef_error(_crx_builder_t *cb, const char *msg)
-{
-    abort();
-}
-
-static int load_creflect_main(PyLibObject *lib)
-{
-    const char *creflect_main = "_creflect_main";   /* XXX fixed for now */
-    int (*crxmain)(_crx_builder_t *);
-    crxmain = (int(*)(_crx_builder_t *))dlsym(lib->ffi->dl_lib, creflect_main);
-    if (crxmain == NULL) {
-        PyErr_Format(PyExc_OSError, "%s: symbol '%s' not found",
-                     lib->ffi->libname, creflect_main);
-        return -1;
-    }
-
-    int result = crxmain(NULL);
-    if ((result & ~0xFF) != MAGIC_CREFLECT_TAG) {
-        PyErr_Format(PyExc_OSError, "%s: %s() is not a creflect entry point",
-                     lib->ffi->libname, creflect_main);
-        return -1;
-    }
-    if ((result & 0xFF) != CREFLECT_VERSION) {
-        PyErr_Format(PyExc_OSError, "%s was made with creflect version %d, "
-                     "expected %d", lib->ffi->libname, result & 0xFF,
-                     CREFLECT_VERSION);
-        return -1;
-    }
-
-    zeffir_builder_t builder = {
-        {
-            zef_get_void_type,
-            zef_get_char_type,
-            zef_get_bool_type,
-            zef_get_signed_type,
-            zef_get_unsigned_type,
-            zef_get_float_type,
-            zef_get_function_type,
-            zef_get_ellipsis_function_type,
-            zef_get_pointer_type,
-            zef_get_const_type,
-            zef_get_array_type,
-            zef_get_incomplete_array_type,
-            zef_get_struct_type,
-            zef_get_union_type,
-            zef_get_enum_type,
-            zef_get_user_type,
-            zef_get_unknown_type,
-            zef_complete,
-            zef_complete_enum,
-            zef_define_type,
-            zef_define_var,
-            zef_define_func,
-            zef_define_num_const,
-            zef_error,
-        },
-        lib,        /* lib */
-        lib->ffi,   /* ffi */
-        lib->dict,  /* dict */
-    };
-    crxmain(&builder.cb);
-    return PyErr_Occurred() ? -1 : 0;
-}
-
-static PyObject *b_open(PyObject *self, PyObject *args, PyObject *kwds)
-{
-    char *keywords[] = {"libname", "relative_to", NULL};
-    char *libname, *relative_to;
-    char *path;
-
-    if (!PyArg_ParseTupleAndKeywords(args, kwds, "ss", keywords,
-                                     &libname, &relative_to))
-        return NULL;
-
-    char *last_slash = strrchr(relative_to, '/');
-    if (last_slash == NULL) {
-        relative_to = "./";
-        last_slash = relative_to + 1;
-    }
-    size_t n = last_slash + 1 - relative_to;
-    path = malloc(n + strlen(libname) + 16);
-    if (path == NULL)
-        return PyErr_NoMemory();
-    memcpy(path, relative_to, n);
-    strcpy(path + n, "lib");
-    strcat(path + n, libname);
-    strcat(path + n, ".so");
-
-    PyFFIObject *ffi = NULL;
-    PyLibObject *lib = NULL;
-    PyObject *dict = NULL;
-
-    dlerror();   /* clear errors */
-    void *dl_lib = dlopen(path, RTLD_LAZY);
-    if (dl_lib == NULL) {
-        char *error = dlerror();
-        if (error == NULL)
-            error = "failed to open";
-        PyErr_Format(PyExc_OSError, "%s: %s", path, error);
-        goto error;
-    }
-
-    ffi = PyObject_New(PyFFIObject, &FFI_Type);
-    if (ffi == NULL)
-        goto error;
-    ffi->dl_lib = dl_lib;  dl_lib = NULL;
-    ffi->libname = path;   path = NULL;
-
-    dict = PyDict_New();
-    if (dict == NULL)
-        goto error;
-
-    lib = PyObject_New(PyLibObject, &Lib_Type);
-    if (lib == NULL)
-        goto error;
-    lib->ffi = ffi;    ffi = NULL;
-    lib->dict = dict;  dict = NULL;
-
-    if (load_creflect_main(lib) < 0)
-        goto error;
-
-    PyObject *result = Py_BuildValue("OO", lib->ffi, lib);
-    Py_DECREF(lib);
-    return result;
-
- error:
-    Py_XDECREF(dict);
-    Py_XDECREF(lib);
-    Py_XDECREF(ffi);
-    if (dl_lib != NULL)
-        dlclose(dl_lib);
-    free(path);
-    return NULL;
-}
-
-
-static PyMethodDef ZeffirMethods[] = {
-    {"open",   (PyCFunction)b_open, METH_KEYWORDS},
-    {NULL,     NULL}    /* Sentinel */
-};
-
 PyMODINIT_FUNC
 initzeffir(void)
 {
@@ -424,6 +30,12 @@
     m = Py_InitModule("zeffir", ZeffirMethods);
     (void)m;
 
-    PyType_Ready(&FFI_Type);
-    PyType_Ready(&Lib_Type);
+    init_caches();
+    if (PyErr_Occurred())
+        return;
+
+    if (PyType_Ready(&FFI_Type) < 0)
+        return;
+    if (PyType_Ready(&Lib_Type) < 0)
+        return;
 }

From noreply at buildbot.pypy.org  Sun Nov 30 17:41:15 2014
From: noreply at buildbot.pypy.org (fijal)
Date: Sun, 30 Nov 2014 17:41:15 +0100 (CET)
Subject: [pypy-commit] pypy default: an attempt to improve nditer
	performance in simple cases
Message-ID: <20141130164115.2090F1C0296@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: 
Changeset: r74763:9a68fb120de2
Date: 2014-11-30 18:40 +0200
http://bitbucket.org/pypy/pypy/changeset/9a68fb120de2/

Log:	an attempt to improve nditer performance in simple cases

diff --git a/pypy/module/micronumpy/concrete.py b/pypy/module/micronumpy/concrete.py
--- a/pypy/module/micronumpy/concrete.py
+++ b/pypy/module/micronumpy/concrete.py
@@ -47,6 +47,7 @@
     def setitem(self, index, value):
         self.dtype.itemtype.store(self, index, 0, value)
 
+    @jit.unroll_safe
     def setslice(self, space, arr):
         if len(arr.get_shape()) > 0 and len(self.get_shape()) == 0:
             raise oefmt(space.w_ValueError,
diff --git a/pypy/module/micronumpy/loop.py b/pypy/module/micronumpy/loop.py
--- a/pypy/module/micronumpy/loop.py
+++ b/pypy/module/micronumpy/loop.py
@@ -88,6 +88,21 @@
                                 reds = 'auto')
 
 def setslice(space, shape, target, source):
+    if not shape:
+        # XXX - simplify
+        target_iter, target_state = target.create_iter(shape)
+        source_iter, source_state = source.create_iter(shape)
+        dtype = target.dtype
+        val = source_iter.getitem(source_state)
+        if dtype.is_str_or_unicode():
+            val = dtype.coerce(space, val)
+        else:
+            val = val.convert_to(space, dtype)
+        target_iter.setitem(target_state, val)
+        return target        
+    return _setslice(space, shape, target, source)
+
+def _setslice(space, shape, target, source):
     # note that unlike everything else, target and source here are
     # array implementations, not arrays
     target_iter, target_state = target.create_iter(shape)
diff --git a/pypy/module/micronumpy/nditer.py b/pypy/module/micronumpy/nditer.py
--- a/pypy/module/micronumpy/nditer.py
+++ b/pypy/module/micronumpy/nditer.py
@@ -1,3 +1,4 @@
+from rpython.rlib import jit
 from pypy.interpreter.baseobjspace import W_Root
 from pypy.interpreter.typedef import TypeDef, GetSetProperty
 from pypy.interpreter.gateway import interp2app, unwrap_spec, WrappedDefault
@@ -264,8 +265,8 @@
         self.index = [0] * len(shape)
         self.backward = backward
 
+    @jit.unroll_safe
     def next(self):
-        # TODO It's probably possible to refactor all the "next" method from each iterator
         for i in range(len(self.shape) - 1, -1, -1):
             if self.index[i] < self.shape[i] - 1:
                 self.index[i] += 1

From noreply at buildbot.pypy.org  Sun Nov 30 17:57:09 2014
From: noreply at buildbot.pypy.org (fijal)
Date: Sun, 30 Nov 2014 17:57:09 +0100 (CET)
Subject: [pypy-commit] pypy default: another attempt at improving performance
Message-ID: <20141130165709.1BC711D279F@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: 
Changeset: r74764:291081ced5e0
Date: 2014-11-30 18:57 +0200
http://bitbucket.org/pypy/pypy/changeset/291081ced5e0/

Log:	another attempt at improving performance

diff --git a/pypy/module/micronumpy/iterators.py b/pypy/module/micronumpy/iterators.py
--- a/pypy/module/micronumpy/iterators.py
+++ b/pypy/module/micronumpy/iterators.py
@@ -154,7 +154,7 @@
         index = state.index
         if self.track_index:
             index += 1
-        indices = state.indices
+        indices = [state.indices[i] for i in range(len(state.indices))]
         offset = state.offset
         if self.contiguous:
             offset += self.array.dtype.elsize

From noreply at buildbot.pypy.org  Sun Nov 30 18:00:01 2014
From: noreply at buildbot.pypy.org (fijal)
Date: Sun, 30 Nov 2014 18:00:01 +0100 (CET)
Subject: [pypy-commit] pypy default: fix, thanks alex
Message-ID: <20141130170001.3185D1D2865@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: 
Changeset: r74765:e1cfe5a7c951
Date: 2014-11-30 18:59 +0200
http://bitbucket.org/pypy/pypy/changeset/e1cfe5a7c951/

Log:	fix, thanks alex

diff --git a/pypy/module/micronumpy/iterators.py b/pypy/module/micronumpy/iterators.py
--- a/pypy/module/micronumpy/iterators.py
+++ b/pypy/module/micronumpy/iterators.py
@@ -154,7 +154,7 @@
         index = state.index
         if self.track_index:
             index += 1
-        indices = [state.indices[i] for i in range(len(state.indices))]
+        indices = state.indices[:]
         offset = state.offset
         if self.contiguous:
             offset += self.array.dtype.elsize

From noreply at buildbot.pypy.org  Sun Nov 30 21:56:38 2014
From: noreply at buildbot.pypy.org (mattip)
Date: Sun, 30 Nov 2014 21:56:38 +0100 (CET)
Subject: [pypy-commit] pypy ufuncapi: more tests,
 start to fix type_resolver and alloc_outargs
Message-ID: <20141130205638.ABBB51C0FFF@cobra.cs.uni-duesseldorf.de>

Author: mattip 
Branch: ufuncapi
Changeset: r74766:c2354e730694
Date: 2014-11-30 22:55 +0200
http://bitbucket.org/pypy/pypy/changeset/c2354e730694/

Log:	more tests, start to fix type_resolver and alloc_outargs

diff --git a/pypy/module/micronumpy/test/test_ufuncs.py b/pypy/module/micronumpy/test/test_ufuncs.py
--- a/pypy/module/micronumpy/test/test_ufuncs.py
+++ b/pypy/module/micronumpy/test/test_ufuncs.py
@@ -1,7 +1,10 @@
 from pypy.module.micronumpy.test.test_base import BaseNumpyAppTest
 from pypy.module.micronumpy.ufuncs import (find_binop_result_dtype,
-        find_unaryop_result_dtype)
+        find_unaryop_result_dtype, W_UfuncGeneric)
+from pypy.module.micronumpy.support import _parse_signature
 from pypy.module.micronumpy.descriptor import get_dtype_cache
+from pypy.module.micronumpy.base import W_NDimArray
+from pypy.module.micronumpy.concrete import VoidBoxStorage
 from pypy.interpreter.gateway import interp2app
 from pypy.conftest import option
 
@@ -83,6 +86,42 @@
         # promote bools, happens with sign ufunc
         assert find_unaryop_result_dtype(space, bool_dtype, promote_bools=True) is int8_dtype
 
+
+class TestGenericUfuncOperation(object):
+    def test_signature_parser(self, space):
+        class Ufunc(object):
+            def __init__(self, nin, nout):
+                self.nin = nin
+                self.nout = nout
+                self.nargs = nin + nout
+                self.core_enabled = True
+                self.core_num_dim_ix = 0 
+                self.core_num_dims = [0] * self.nargs  
+                self.core_offsets = [0] * self.nargs
+                self.core_dim_ixs = [] 
+
+        u = Ufunc(2, 1)
+        _parse_signature(space, u, '(m,n), (n,r)->(m,r)')
+        assert u.core_dim_ixs == [0, 1, 1, 2, 0, 2]
+        assert u.core_num_dims == [2, 2, 2]
+        assert u.core_offsets == [0, 2, 4]
+
+    def test_type_resolver(self, space):
+        c128_dtype = get_dtype_cache(space).w_complex128dtype
+        f64_dtype = get_dtype_cache(space).w_float64dtype
+        u32_dtype = get_dtype_cache(space).w_uint32dtype
+        b_dtype = get_dtype_cache(space).w_booldtype
+
+        ufunc = W_UfuncGeneric(space, [None, None], 'eigenvals', None, 1, 1,
+                     [f64_dtype, c128_dtype, c128_dtype, c128_dtype],
+                     '')
+        f64 = W_NDimArray(VoidBoxStorage(0, f64_dtype))
+        c128 = W_NDimArray(VoidBoxStorage(0, c128_dtype))
+        index, dtypes = ufunc.type_resolver(space, [f64], [c128], 'd->D')
+        assert index == 0
+        assert dtypes == [f64_dtype, c128_dtype]
+
+
 class AppTestUfuncs(BaseNumpyAppTest):
     def test_constants(self):
         import numpy as np
diff --git a/pypy/module/micronumpy/ufuncs.py b/pypy/module/micronumpy/ufuncs.py
--- a/pypy/module/micronumpy/ufuncs.py
+++ b/pypy/module/micronumpy/ufuncs.py
@@ -571,8 +571,8 @@
                 outargs[i] = out
         if sig is None:
             sig = space.wrap(self.signature)
-        index = self.type_resolver(space, inargs, outargs, sig)
-        outargs = self.alloc_outargs(space, index, inargs, outargs)
+        index, dtypes = self.type_resolver(space, inargs, outargs, sig)
+        outargs = self.alloc_outargs(space, index, inargs, outargs, dtypes)
         for i in range(len(outargs)):
             assert isinstance(outargs[i], W_NDimArray)
         outargs0 = outargs[0]
@@ -805,23 +805,40 @@
             kwargs_w.pop(kw)
         return w_subok, w_out, sig, casting, extobj
 
-    def type_resolver(self, space, inargs, outargs, sig):
+    def type_resolver(self, space, inargs, outargs, type_tup):
         # Find a match for the inargs.dtype in self.dtypes, like
         # linear_search_type_resolver in numpy ufunc_type_resolutions.c
+        # type_tup can be '', a tuple of dtypes, or a string
+        # of the form d,t -> D where the letters are dtype specs
         inargs0 = inargs[0]
         assert isinstance(inargs0, W_NDimArray)
+        nop = len(inargs) + len(outargs)
+        dtypes = []
+        if isinstance(type_tup, str):
+            if len(type_tup) == 1:
+                dtypes = [get_dtype_cache(space).dtypes_by_name[type_tup]] * self.nargs
+            elif len(type_tup) == self.nargs + 2:
+                for i in range(len(inargs)):
+                    dtypes.append(get_dtype_cache(space).dtype_by_name[type_tup[i]])
+                #skip the '->' in the signature
+                for i in range(len(self.nout)):
+                    dtypes.append(get_dtype_cache(space).dtype_by_name[type_tup[i+2]])
+            else:
+                raise oefmt(space.w_TypeError, "a type-string for %s," \
+                    "requires 1 typecode or %d typecode(s) before and %d" \
+                    " after the -> sign", self.name, self.nin, self.nout)
         for i in range(0, len(self.dtypes), self.nargs):
             if inargs0.get_dtype() == self.dtypes[i]:
                 break
         else:
             if len(self.funcs) < 2:
-                return 0
+                return 0, dtypes
             raise oefmt(space.w_TypeError,
                          'input dtype %s did not match any known dtypes',
                                               str(inargs0.get_dtype()))
-        return i / self.nargs
+        return i / self.nargs, dtypes
 
-    def alloc_outargs(self, space, index, inargs, outargs):
+    def alloc_outargs(self, space, index, inargs, outargs, dtypes):
         # Any None outarg should be allocated here
         inargs0 = inargs[0]
         assert isinstance(inargs0, W_NDimArray)

From noreply at buildbot.pypy.org  Sun Nov 30 22:29:41 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Sun, 30 Nov 2014 22:29:41 +0100 (CET)
Subject: [pypy-commit] creflect default: in-progress: refactoring: like C,
 we distinguish now "type qualifiers"
Message-ID: <20141130212941.C3FD51D2865@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r129:7673f248edbc
Date: 2014-11-30 22:29 +0100
http://bitbucket.org/cffi/creflect/changeset/7673f248edbc/

Log:	in-progress: refactoring: like C, we distinguish now "type
	qualifiers" as something different from types

diff --git a/creflect/codegen.py b/creflect/codegen.py
--- a/creflect/codegen.py
+++ b/creflect/codegen.py
@@ -7,11 +7,13 @@
         if parent is None:
             self.crx_type_vars = []
             self.crx_type_cache = {}
+            self.crx_qualtype_vars = []
             self.crx_field_vars = []
             self.crx_top_level = self
         else:
             self.crx_type_vars = parent.crx_type_vars
             self.crx_type_cache = parent.crx_type_cache
+            self.crx_qualtype_vars = parent.crx_qualtype_vars
             self.crx_field_vars = parent.crx_field_vars
             self.crx_top_level = parent.crx_top_level
 
@@ -40,13 +42,12 @@
         self.writeline('%s = %s;' % (t1, expr))
         return t1
 
-    def add_array_crx_types(self, array_size):
-        if array_size > 0:
-            a1 = self._get_next_name('a')
-            self.crx_type_vars.append('*%s[%d]' % (a1, array_size))
-            return a1
-        else:
+    def write_crx_qualtype_var(self, array_size):
+        if array_size == 0:
             return '0'
+        a1 = 'a%d' % (len(self.crx_qualtype_vars) + 1)
+        self.crx_qualtype_vars.append('%s[%d]' % (a1, array_size))
+        return a1
 
     def write_crx_field_var(self, nfields):
         d1 = 'd%d' % (len(self.crx_field_vars) + 1)
@@ -67,6 +68,8 @@
     if funcblock.crx_type_vars:
         funcblock.writedecl("_crx_type_t %s;" % (
             ', '.join(funcblock.crx_type_vars),))
+    for a1 in funcblock.crx_qualtype_vars:
+        funcblock.writedecl("_crx_qual_type %s;" % a1)
     for d1 in funcblock.crx_field_vars:
         funcblock.writedecl("_crx_field_t %s;" % d1)
 
diff --git a/creflect/commontypes.py b/creflect/commontypes.py
--- a/creflect/commontypes.py
+++ b/creflect/commontypes.py
@@ -19,7 +19,7 @@
     '_Bool':              'i',
     }
 
-def resolve_common_type(names, const):
+def resolve_common_type(names):
     # reduce synonyms to a single chosen combination
     names = list(names)
     if names != ['signed', 'char']:    # keep this unmodified
@@ -44,8 +44,5 @@
         names = newnames + names
     ident = ' '.join(names)
     if ident == 'void':
-        if const:
-            return model.const_void_type
-        else:
-            return model.void_type
-    return model.PrimitiveType(ident, const)
+        return model.void_type
+    return model.PrimitiveType(ident)
diff --git a/creflect/cparser.py b/creflect/cparser.py
--- a/creflect/cparser.py
+++ b/creflect/cparser.py
@@ -68,7 +68,7 @@
     def __init__(self, cdefblock, first_lineno=1):
         self.cdefblock = cdefblock
         self.first_lineno = first_lineno
-        self._struct_union_enum_decl = {}
+        self.struct_union_enum_decl = {}
 
     def prepare_source(self):
         csource = remove_comments(self.cdefblock)
@@ -104,130 +104,125 @@
         assert ast.ext[0].name == '__crx_unknown_t'
         for decl in ast.ext[1:]:
             if isinstance(decl, pycparser.c_ast.Decl):
-                self._parse_decl(decl)
+                self.parse_decl(decl)
             elif isinstance(decl, pycparser.c_ast.Typedef):
-                self._parse_typedef(decl)
+                self.parse_typedef(decl)
             else:
-                self._parse_error("unrecognized construct", decl)
+                self.parse_error("unrecognized construct", decl)
         return self.declarations
 
-    def _parse_error(self, msg, decl):
+    def parse_error(self, msg, decl):
         raise CDefError(msg, self, decl.coord.line - 1)
 
-    def _parse_decl(self, decl):
+    def parse_decl(self, decl):
         node = decl.type
         if isinstance(node, pycparser.c_ast.FuncDecl):
-            tp = self._get_type(node)
-            assert isinstance(tp, model.FunctionType)
-            self.declarations.append(model.FuncDecl(decl.name, tp))
+            qualtp = self.get_qualtype(node)
+            assert isinstance(qualtp.type, model.FunctionType)
+            self.declarations.append(model.FuncDecl(decl.name, qualtp))
         else:
             const = 'const' in decl.quals
             if isinstance(node, pycparser.c_ast.Struct):
                 if node.decls is not None:
-                    self._get_struct_union_enum_type('struct', node, const)
+                    self.get_struct_union_enum_type('struct', node, const)
             elif isinstance(node, pycparser.c_ast.Union):
                 if node.decls is not None:
-                    self._get_struct_union_enum_type('union', node, const)
+                    self.get_struct_union_enum_type('union', node, const)
             elif isinstance(node, pycparser.c_ast.Enum):
                 if node.values is not None:
-                    self._get_struct_union_enum_type('enum', node, const)
+                    self.get_struct_union_enum_type('enum', node, const)
             elif not decl.name:
                 raise api.CDefError("construct does not declare any variable",
                                     decl)
             #
             if decl.name:
-                tp = self._get_type(node, partial_length_ok=True)
-                if self._is_constant_globalvar(node):
-                    self.declarations.append(model.ConstDecl(decl.name, tp))
+                qualtp = self.get_qualtype(node)
+                if self.is_constant_globalvar(node):
+                    self.declarations.append(model.ConstDecl(decl.name, qualtp))
                 else:
-                    self.declarations.append(model.VarDecl(decl.name, tp))
+                    self.declarations.append(model.VarDecl(decl.name, qualtp))
 
-    def _parse_typedef(self, decl):
+    def parse_typedef(self, decl):
         if not decl.name:
-            self._parse_error("typedef does not declare any name", decl)
-        if (isinstance(decl.type.type, pycparser.c_ast.IdentifierType)
-                and decl.type.type.names == ['__dotdotdot__']):
-            realtype = model.unknown_type(decl.name)
-        elif (isinstance(decl.type, pycparser.c_ast.PtrDecl) and
-              isinstance(decl.type.type, pycparser.c_ast.TypeDecl) and
-              isinstance(decl.type.type.type,
-                         pycparser.c_ast.IdentifierType) and
-              decl.type.type.type.names == ['__dotdotdot__']):
-            realtype = model.unknown_ptr_type(decl.name)
-        else:
-            realtype = self._get_type(decl.type, approx_name = '$' + decl.name)
+            self.parse_error("typedef does not declare any name", decl)
+        qualtype = self.get_qualtype(decl.type, approx_name = '$'+decl.name)
         if decl.name in self.typedefs:
-            self._parse_error("duplicate typedef declaring %r" % (decl.name,))
-        self.typedefs[decl.name] = model.UserType(decl.name, realtype)
-        self.declarations.append(model.TypeDefDecl(decl.name, realtype))
+            self.parse_error("duplicate typedef declaring %r" % (decl.name,))
+        self.typedefs[decl.name] = model.UserType(decl.name)
+        self.declarations.append(model.TypeDefDecl(decl.name, qualtype))
 
-    def _get_type(self, typenode, approx_name=None, partial_length_ok=False):
+    def get_qualtype(self, typenode, approx_name=None):
         # first, dereference typedefs, if we have it already parsed, we're good
         if (isinstance(typenode, pycparser.c_ast.TypeDecl) and
             isinstance(typenode.type, pycparser.c_ast.IdentifierType) and
             len(typenode.type.names) == 1):
             typename = typenode.type.names[0]
             if typename in self.typedefs:
-                return self.typedefs[typenode.type.names[0]]
+                return model.QualType(self.typedefs[typenode.type.names[0]],
+                                      quals_num(typenode.quals))
             if typename == '__crx_unknown_t':
                 assert approx_name, "XXX"
-                return model.UnknownType(approx_name)
+                return model.QualType(model.UnknownType(approx_name),
+                                      quals_num(typenode.quals))
         #
         if isinstance(typenode, pycparser.c_ast.ArrayDecl):
             # array type
             if typenode.dim is None:
                 length = None
             else:
-                length = self._parse_constant(
-                    typenode.dim, partial_length_ok=partial_length_ok)
-            return model.ArrayType(self._get_type(typenode.type), length)
+                length = self.parse_constant(typenode.dim)
+            qualtype = self.get_qualtype(typenode.type)
+            return model.QualType(model.ArrayType(qualtype.type, length),
+                                  qualtype.quals)
         #
         if isinstance(typenode, pycparser.c_ast.PtrDecl):
             # pointer type
-            const = 'const' in typenode.quals
             if approx_name:
                 approx_name = '$' + approx_name
-            realtype = self._get_type(typenode.type, approx_name)
-            return model.PointerType(realtype, const)
+            qualtype = self.get_qualtype(typenode.type, approx_name)
+            return model.QualType(model.PointerType(qualtype),
+                                  quals_num(typenode.quals))
         #
         if isinstance(typenode, pycparser.c_ast.TypeDecl):
             const = 'const' in typenode.quals
             type = typenode.type
             if isinstance(type, pycparser.c_ast.IdentifierType):
                 # assume a primitive type.
-                return resolve_common_type(type.names, const)
+                realtype = resolve_common_type(type.names)
+                return model.QualType(realtype, quals_num(typenode.quals))
             #
             if isinstance(type, pycparser.c_ast.Struct):
                 # 'struct foobar'
-                return self._get_struct_union_enum_type('struct', type, const,
-                                                        approx_name)
+                return self.get_struct_union_enum_type('struct', type, const,
+                                                       approx_name)
             #
             if isinstance(type, pycparser.c_ast.Union):
                 # 'union foobar'
-                return self._get_struct_union_enum_type('union', type, const,
-                                                        approx_name)
+                return self.get_struct_union_enum_type('union', type, const,
+                                                       approx_name)
             #
             if isinstance(type, pycparser.c_ast.Enum):
                 # 'enum foobar'
-                return self._get_struct_union_enum_type('enum', type, const,
-                                                        approx_name)
+                return self.get_struct_union_enum_type('enum', type, const,
+                                                       approx_name)
         #
         if isinstance(typenode, pycparser.c_ast.FuncDecl):
-            # a function type
-            return self._parse_function_type(typenode, approx_name)
+            # a function type (it is never qualified)
+            return model.QualType(
+                self.parse_function_type(typenode, approx_name), 0)
         #
         # nested anonymous structs or unions end up here
         if isinstance(typenode, pycparser.c_ast.Struct):
-            return self._get_struct_union_enum_type('struct', typenode, const,
-                                                    name, nested=True)
+            return self.get_struct_union_enum_type('struct', typenode, const,
+                                                   name, nested=True)
         if isinstance(typenode, pycparser.c_ast.Union):
-            return self._get_struct_union_enum_type('union', typenode, const,
-                                                    name, nested=True)
+            return self.get_struct_union_enum_type('union', typenode, const,
+                                                   name, nested=True)
         #
         raise api.FFIError(":%d: bad or unsupported type declaration" %
                 typenode.coord.line)
 
-    def _parse_function_type(self, typenode, funcname=None):
+    def parse_function_type(self, typenode, funcname=None):
         params = list(getattr(typenode.args, 'params', []))
         ellipsis = (
             len(params) > 0 and
@@ -243,37 +238,32 @@
             isinstance(params[0].type.type, pycparser.c_ast.IdentifierType)
                 and list(params[0].type.type.names) == ['void']):
             del params[0]
-        args = [self._as_func_arg(self._get_type(argdeclnode.type))
-                for argdeclnode in params]
-        result = self._get_type(typenode.type)
-        return model.FunctionType(tuple(args), result, ellipsis)
+        #
+        args_quals = [self.get_qualtype(argdeclnode.type)
+                      for argdeclnode in params]
+        # ignore the quals of the result type (GCC produces a warning with
+        # -Wignored-qualifiers, but it is not even included in -Wall)
+        qual_result = self.get_qualtype(typenode.type)
+        return model.FunctionType(tuple(args_quals), qual_result.type, ellipsis)
 
-    def _as_func_arg(self, type):
-        if isinstance(type, model.ArrayType):
-            return model.PointerType(type.item, const=False)
-        elif isinstance(type, model.FunctionType):
-            return model.PointerType(type, const=False)
-        else:
-            return type
-
-    def _is_constant_globalvar(self, typenode):
+    def is_constant_globalvar(self, typenode):
         if isinstance(typenode, pycparser.c_ast.PtrDecl):
             return 'const' in typenode.quals
         if isinstance(typenode, pycparser.c_ast.TypeDecl):
             return 'const' in typenode.quals
         return False
 
-    def _get_struct_union_enum_type(self, kind, type, const, approx_name=None):
+    def get_struct_union_enum_type(self, kind, type, const, approx_name=None):
         name = type.name or approx_name
         if not name or name.startswith('$$$'):
-            self._parse_error("not implemented: anonymous 'struct' elsewhere "
-                              "than in 'typedef struct { ... } typename;' or "
-                              "'typedef struct { ... } *typename;'", type)
+            self.parse_error("not implemented: anonymous 'struct' elsewhere "
+                             "than in 'typedef struct { ... } typename;' or "
+                             "'typedef struct { ... } *typename;'", type)
         result = model.StructOrUnionOrEnum(kind, name, const)
         #
         # get the type declaration or create it if needed
         key = '%s %s' % (kind, name)
-        typedecl = self._struct_union_enum_decl.get(key, None)
+        typedecl = self.struct_union_enum_decl.get(key, None)
         #
         if typedecl is None:
             if kind == 'struct' or kind == 'union':
@@ -282,7 +272,7 @@
                 typedecl = XXX
             else:
                 raise AssertionError("kind = %r" % (kind,))
-            self._struct_union_enum_decl[key] = typedecl
+            self.struct_union_enum_decl[key] = typedecl
             must_add = True
         else:
             must_add = False
@@ -290,12 +280,12 @@
         # is there a 'type.decls'?  If yes, then this is the place in the
         # C sources that declare the fields.
         if type.decls is not None:
-            self._add_fields_declaration(typedecl, type.decls)
+            self.add_fields_declaration(typedecl, type.decls)
         if must_add:
             self.declarations.append(typedecl)
         return result
 
-    def _add_fields_declaration(self, typedecl, fields):
+    def add_fields_declaration(self, typedecl, fields):
         if typedecl.fldnames is not None:
             raise api.CDefError("duplicate declaration of struct %s" % name)
         fldnames = []
@@ -307,13 +297,13 @@
             if decl.bitsize is None:
                 bitsize = -1
             else:
-                bitsize = self._parse_constant(decl.bitsize)
-            self._partial_length = False
-            type = self._get_type(decl.type, partial_length_ok=True)
-            if self._partial_length:
-                self._make_partial(tp, nested)
+                bitsize = self.parse_constant(decl.bitsize)
+            self.partial_length = False
+            type = self.get_type(decl.type)
+            if self.partial_length:
+                self.make_partial(tp, nested)
             #if isinstance(type, model.StructType) and type.partial:
-            #    self._make_partial(tp, nested)
+            #    self.make_partial(tp, nested)
             fldnames.append(decl.name or '')
             fldtypes.append(type)
             fldbitsize.append(bitsize)
@@ -326,5 +316,18 @@
                 raise NotImplementedError("%s: using both bitfields and '...;'"
                                           % (tp,))
 
-    def _parse_constant(self, *args, **kwds):
-        return None     # XXX
+    def parse_constant(self, constant):
+        return int(constant.value)   # xxx
+
+
+def quals_num(quals):
+    assert isinstance(quals, (list, str))
+    if isinstance(quals, str):
+        quals = [quals]
+    result = 0
+    for qual in quals:
+        if qual == 'const':
+            result |= model.CRX_CONST
+        if qual == 'volatile':
+            result |= model.CRX_VOLATILE
+    return result
diff --git a/creflect/creflect.h b/creflect/creflect.h
--- a/creflect/creflect.h
+++ b/creflect/creflect.h
@@ -10,8 +10,14 @@
 typedef void (*_crx_trampoline1_fn)(void *fn, void *args[], void *res);
 
 typedef struct {
+    _crx_type_t *type;
+    int qualifiers;
+} _crx_qual_type;
+
+typedef struct {
     const char *name;
     _crx_type_t *type;
+    int qualifiers;
     size_t offset;
     int numbits, bitshift;
 } _crx_field_t;
@@ -29,6 +35,9 @@
     long double as_long_double;
 } _crx_num_const_t;
 
+#define _CRX_CONST     1
+#define _CRX_VOLATILE  2
+
 #define _CRX_SELF struct _crx_builder_s *
 typedef struct _crx_builder_s {
     _crx_type_t *(*get_void_type)(_CRX_SELF);
@@ -41,27 +50,25 @@
     _crx_type_t *(*get_float_type)(_CRX_SELF, size_t,
                                    const char *);
     _crx_type_t *(*get_function_type)(_CRX_SELF, _crx_type_t *,
-                                      _crx_type_t *[], int,
+                                      _crx_qual_type[], int,
                                       _crx_trampoline1_fn);
     _crx_type_t *(*get_ellipsis_function_type)(_CRX_SELF, _crx_type_t *,
-                                               _crx_type_t *[], int);
-    _crx_type_t *(*get_pointer_type)(_CRX_SELF, _crx_type_t *);
-    _crx_type_t *(*get_const_type)(_CRX_SELF, _crx_type_t *);
+                                               _crx_qual_type[], int);
+    _crx_type_t *(*get_pointer_type)(_CRX_SELF, _crx_type_t *, int);
     _crx_type_t *(*get_array_type)(_CRX_SELF, _crx_type_t *, size_t);
     _crx_type_t *(*get_incomplete_array_type)(_CRX_SELF, _crx_type_t *);
     _crx_type_t *(*get_struct_type)(_CRX_SELF, const char *);
     _crx_type_t *(*get_union_type)(_CRX_SELF, const char *);
     _crx_type_t *(*get_enum_type)(_CRX_SELF, const char *);
-    _crx_type_t *(*get_user_type)(_CRX_SELF, const char *);
+    _crx_qual_type(*get_user_type)(_CRX_SELF, const char *);
     _crx_type_t *(*get_unknown_type)(_CRX_SELF, const char *);
     void (*complete)(_CRX_SELF, _crx_type_t *,
                      size_t, size_t, _crx_field_t[], int);
     void (*complete_enum)(_CRX_SELF, _crx_type_t *, _crx_type_t *);
-    void (*define_type)(_CRX_SELF, const char *, _crx_type_t *);
-    void (*define_var)(_CRX_SELF, const char *, _crx_type_t *,
-                       void *);
+    void (*define_type)(_CRX_SELF, const char *, _crx_type_t *, int);
+    void (*define_var)(_CRX_SELF, const char *, _crx_type_t *, int, void *);
     void (*define_func)(_CRX_SELF, const char *, _crx_type_t *,
-                        _crx_type_t *[], int, _crx_trampoline0_fn, void *);
+                        _crx_qual_type[], int, _crx_trampoline0_fn, void *);
     void (*define_num_const)(_CRX_SELF, const char *, _crx_type_t *,
                              _crx_num_const_t *);
     void (*error)(_CRX_SELF, const char *);
diff --git a/creflect/creflect_debug_print.c b/creflect/creflect_debug_print.c
--- a/creflect/creflect_debug_print.c
+++ b/creflect/creflect_debug_print.c
@@ -32,6 +32,31 @@
     return t;
 }
 
+static const char *show_qualtype2(_crx_type_t *type, int qualifiers)
+{
+    size_t la = strlen(type->text);
+    size_t extra = 1;
+    if (qualifiers & _CRX_VOLATILE) extra += strlen("VOLATILE ");
+    if (qualifiers & _CRX_CONST)    extra += strlen("CONST "   );
+    char *p = malloc(la + extra);
+    char *q = p;
+    if (qualifiers & _CRX_VOLATILE) {
+        strcpy(q, "VOLATILE ");
+        q += strlen("VOLATILE ");
+    }
+    if (qualifiers & _CRX_CONST) {
+        strcpy(q, "CONST ");
+        q += strlen("CONST ");
+    }
+    strcat(q, type->text);
+    return p;
+}
+
+static const char *show_qualtype1(_crx_qual_type qualtype)
+{
+    return show_qualtype2(qualtype.type, qualtype.qualifiers);
+}
+
 static _crx_type_t *tst_get_void_type(_crx_builder_t *cb)
 {
     return newtype("void");
@@ -108,13 +133,13 @@
 }
 
 static _crx_type_t *tst_get_function_type(_crx_builder_t *cb, _crx_type_t *ret,
-                                          _crx_type_t *args[], int nargs,
+                                          _crx_qual_type args[], int nargs,
                                           _crx_trampoline1_fn trampl)
 {
     int i;
     const char *prev = "FUNC( ";
     for (i = 0; i < nargs; i++) {
-        prev = newtype2(prev, args[i]->text)->text;
+        prev = newtype2(prev, show_qualtype1(args[i]))->text;
         prev = newtype2(prev, " -> ")->text;
     }
     prev = newtype2(prev, ret->text)->text;
@@ -123,21 +148,17 @@
 
 static _crx_type_t *tst_get_ellipsis_function_type(_crx_builder_t *cb,
                                                    _crx_type_t *ret,
-                                                   _crx_type_t *args[],
+                                                   _crx_qual_type args[],
                                                    int nargs)
 {
     _crx_type_t *dotdotdot = newtype2("... -> ", ret->text);
     return tst_get_function_type(cb, dotdotdot, args, nargs, 0);
 }
 
-static _crx_type_t *tst_get_pointer_type(_crx_builder_t *cb, _crx_type_t *t)
+static _crx_type_t *tst_get_pointer_type(_crx_builder_t *cb,
+                                         _crx_type_t *type, int qualifiers)
 {
-    return newtype2("PTR ", t->text);
-}
-
-static _crx_type_t *tst_get_const_type(_crx_builder_t *cb, _crx_type_t *t)
-{
-    return newtype2("CONST ", t->text);
+    return newtype2("PTR ", show_qualtype2(type, qualifiers));
 }
 
 static _crx_type_t *tst_get_array_type(_crx_builder_t *cb, _crx_type_t *t,
@@ -169,9 +190,12 @@
     return newtype2("ENUM ", name);
 }
 
-static _crx_type_t *tst_get_user_type(_crx_builder_t *cb, const char *name)
+static _crx_qual_type tst_get_user_type(_crx_builder_t *cb, const char *name)
 {
-    return newtype(name);
+    _crx_qual_type result;
+    result.type = newtype(name);
+    result.qualifiers = 0;   // approximation: we don't really know here
+    return result;
 }
 
 static _crx_type_t *tst_get_unknown_type(_crx_builder_t *cb, const char *name)
@@ -186,7 +210,8 @@
     int i;
     printf("%s:\n", t->text);
     for (i = 0; i < nfields; i++) {
-        printf("| %s: %s\n", fields[i].name, fields[i].type->text);
+        printf("| %s: %s\n", fields[i].name,
+               show_qualtype2(fields[i].type, fields[i].qualifiers));
     }
 }
 
@@ -196,25 +221,26 @@
     abort();
 }
 
-static void tst_define_type(_crx_builder_t *cb, const char *name, _crx_type_t *t)
+static void tst_define_type(_crx_builder_t *cb, const char *name,
+                            _crx_type_t *type, int qualifiers)
 {
-    printf("TYPEDEF %s = %s\n", name, t->text);
+    printf("TYPEDEF %s = %s\n", name, show_qualtype2(type, qualifiers));
 }
 
-static void tst_define_var(_crx_builder_t *cb, const char *name, _crx_type_t *t,
-                           void *addr)
+static void tst_define_var(_crx_builder_t *cb, const char *name,
+                           _crx_type_t *type, int qualifiers, void *addr)
 {
-    printf("VAR %s: %s\n", name, t->text);
+    printf("VAR %s: %s\n", name, show_qualtype2(type, qualifiers));
 }
 
 static void tst_define_func(_crx_builder_t *cb, const char *name,
-                            _crx_type_t *ret, _crx_type_t *args[], int nargs,
+                            _crx_type_t *ret, _crx_qual_type args[], int nargs,
                             _crx_trampoline0_fn trampl, void *directcall)
 {
     int i;
     printf("FUNC %s: ", name);
     for (i = 0; i < nargs; i++)
-        printf("%s -> ", args[i]->text);
+        printf("%s -> ", show_qualtype1(args[i]));
     printf("%s\n", ret->text);
 }
 
@@ -263,7 +289,6 @@
     tst_get_function_type,
     tst_get_ellipsis_function_type,
     tst_get_pointer_type,
-    tst_get_const_type,
     tst_get_array_type,
     tst_get_incomplete_array_type,
     tst_get_struct_type,
diff --git a/creflect/model.py b/creflect/model.py
--- a/creflect/model.py
+++ b/creflect/model.py
@@ -1,26 +1,61 @@
 from .codegen import CodeBlock
 
+CRX_CONST    = 1
+CRX_VOLATILE = 2
+
+
+class QualType(object):
+    def __init__(self, type, quals=0):
+        self.type = type
+        self.quals = quals
+
+    def get_c_name(self, replace_with=''):
+        if self.quals & CRX_CONST:
+            replace_with = ('const ' + replace_with).rstrip()
+        if self.quals & CRX_VOLATILE:
+            replace_with = ('volatile ' + replace_with).rstrip()
+        #
+        if replace_with == '' and ' &' in self.type.c_name_with_marker:
+            replace_from = ' &'
+        else:
+            replace_from = '&'
+        return self.type.c_name_with_marker.replace(replace_from, replace_with)
+
+    def inspect_qualtype(self, block, inspect):
+        return self.type.inspect_type(block, inspect, self.quals)
+
+    def as_func_arg(self):
+        if isinstance(self.type, ArrayType):
+            return QualType(PointerType(QualType(self.type.item, self.quals)),
+                            0)
+        if isinstance(self.type, FunctionType):
+            return QualType(PointerType(self), 0)
+        return self
+
+    def __repr__(self):
+        return '' % (self.get_c_name(),)
+
+    def __eq__(self, other):
+        return (self.__class__ == other.__class__ and
+                self.type == other.type and self.quals == other.quals)
+
+    def __ne__(self, other):
+        return not self == other
+
+    def __hash__(self):
+        return hash((self.type, self.quals))
+
 
 class BaseType(object):
 
-    def get_c_name(self, replace_with=''):
-        return self.c_name_with_marker.replace('&', replace_with).rstrip()
-
     def __repr__(self):
-        return '<%s>' % (self.get_c_name(),)
+        return '<%s %s>' % (self.__class__.__name__,
+                            QualType(self).get_c_name(),)
 
     def _get_items(self):
-        return [(name, getattr(self, name))
-                for name in (self._attrs_ + 'const')]
+        return [(name, getattr(self, name)) for name in self._attrs_]
 
-    def inspect_type(self, block, inspect):
-        t1 = self.inspect_nonconst_type(block, inspect)
-        if self.const:
-            return block.write_crx_type_var('cb->get_const_type(cb, %s)' % t1)
-        else:
-            return t1
-
-    def inspect_nonconst_type(self, block, inspect):
+    def inspect_type(self, block, inspect, qualifiers):
         raise NotImplementedError
 
     def shadow_global_var(self, top_level_block, name):
@@ -40,18 +75,14 @@
 class VoidType(BaseType):
     _attrs_ = ()
 
-    def __init__(self, const):
-        self.const = const
+    def __init__(self):
         self.c_name_with_marker = 'void &'
-        if const:
-            self.c_name_with_marker = 'const ' + self.c_name_with_marker
 
-    def inspect_nonconst_type(self, block, inspect):
+    def inspect_type(self, block, inspect, qualifiers):
         inspect.flush()
         return block.write_crx_type_var('cb->get_void_type(cb)')
 
-void_type = VoidType(const=False)
-const_void_type = VoidType(const=True)
+void_type = VoidType()
 
 
 class PrimitiveType(BaseType):
@@ -75,13 +106,10 @@
         '_Bool':              'u',
         }
 
-    def __init__(self, name, const):
+    def __init__(self, name):
         assert name in self.ALL_PRIMITIVE_TYPES, repr(name)
         self.name = name
-        self.const = const
         self.c_name_with_marker = name + ' &'
-        if const:
-            self.c_name_with_marker = 'const ' + self.c_name_with_marker
 
     def is_char_type(self):
         return self.ALL_PRIMITIVE_TYPES[self.name] == 'c'
@@ -94,7 +122,7 @@
     def is_float_type(self):
         return self.ALL_PRIMITIVE_TYPES[self.name] == 'f'
 
-    def inspect_nonconst_type(self, block, inspect):
+    def inspect_type(self, block, inspect, qualifiers):
         if isinstance(inspect, TypeInspector):
             star_p1 = inspect.fetch_star_p1()
             comment1 = inspect.get_comment(0, False, "a valid type", minlevel=1)
@@ -106,7 +134,7 @@
             block.writedecl("char b[sizeof(%s)];%s" % (star_p1, comment1))
             if self.is_float_type():
                 block.writeline("memset(b, 0, sizeof(b));")
-            elif self.const:
+            elif qualifiers & CRX_CONST:
                 block.writeline("memset(b, -1, sizeof(b));")
             inspect.assign_to_p1("b")
             if self.is_float_type():
@@ -114,7 +142,7 @@
             else:
                 op = '<< 1'
             block.writeline("(void)(%s %s);%s" % (star_p1, op, comment2))
-            if not self.const:
+            if not (qualifiers & CRX_CONST):
                 block.writeline("%s = -1;%s" % (star_p1, comment3))
             if self.is_integer_type():
                 hint = self.name.split()
@@ -196,14 +224,13 @@
 
 class FunctionType(BaseType):
     _attrs_ = ('args', 'result', 'ellipsis')
-    const = True
 
     def __init__(self, args, result, ellipsis):
-        self.args = args
-        self.result = result
+        self.args = args        # list of QualTypes
+        self.result = result    # a plain BaseType
         self.ellipsis = ellipsis
         #
-        reprargs = [arg.get_c_name() for arg in self.args]
+        reprargs = [qualarg.get_c_name() for qualarg in self.args]
         if self.ellipsis:
             reprargs.append('...')
         reprargs = reprargs or ['void']
@@ -213,30 +240,30 @@
 
     def _get_arg_ret_types(self, block):
         inspect = MissingInspector()
-        t1 = self.result.inspect_type(block, inspect)
-        t_args = [arg.inspect_type(block, inspect) for arg in self.args]
-        a2 = block.add_array_crx_types(len(t_args))
-        for i, t in enumerate(t_args):
-            block.writeline('%s[%d] = %s;' % (a2, i, t))
+        t1 = self.result.inspect_type(block, inspect, 0)
+        a2 = block.write_crx_qualtype_var(len(self.args))
+        for i, arg in enumerate(self.args):
+            t3 = arg.inspect_qualtype(block, inspect)
+            block.writeline('%s[%d].type = %s;' % (a2, i, t3))
+            block.writeline('%s[%d].qualifiers = %d;' % (a2, i, arg.quals))
         return t1, a2
 
     def _get_c_call_sequence(self, varname):
         call = ['    ']
         if not isinstance(self.result, VoidType):
-            call.append('*(%s)result = ' % (self.result.get_c_name('*'),))
+            call.append('*(%s)result = ' % (
+                QualType(self.result).get_c_name('*'),))
         call.append(varname)
         call.append('(')
         for i in range(len(self.args)):
             if i > 0:
                 call.append(', ')
-            call.append('*(%s)args[%d]' % (self.args[i].get_c_name('*'), i))
+            call.append('*(%s)args[%d]' % (
+                self.args[i].as_func_arg().get_c_name('*'), i))
         call.append(');')
         return ''.join(call)
 
-    def inspect_type(self, block, inspect):
-        # this class overrides inspect_type() instead of
-        # inspect_nonconst_type(), to avoid the extra call to get_const_type()
-
+    def inspect_type(self, block, inspect, qualifiers):
         # this variable is set if we support slightly misdeclared function
         # args or return value; this is the case only for typedefs that
         # declare a pointer to a function without ellipsis.
@@ -245,7 +272,7 @@
         if common_case:
             inspect.flush()
         else:
-            inspect.flush('(%s)' % self.get_c_name('(*)'),
+            inspect.flush('(%s)' % QualType(self).get_c_name('(*)'),
                 "a function pointer type with exactly the given signature")
 
         if self.ellipsis:
@@ -254,7 +281,7 @@
         if common_case:
             decl = '%s f' % inspect.typename
         else:
-            decl = self.get_c_name('(*f)')
+            decl = QualType(self).get_c_name('(*f)')
 
         toplevel = block.crx_top_level
         extraname = block._get_next_name('f')
@@ -273,7 +300,7 @@
 
     def inspect_type_ellipsis(self, block, inspect):
         if isinstance(inspect, VarInspector):
-            decl = self.get_c_name("(*p1)")
+            decl = QualType(self).get_c_name("(*p1)")
             block.writeline("%s = &%s;  /* check that '%s' is a function with"
                             " exactly the given signature */" % (
                                 decl, inspect.varname, inspect.varname))
@@ -290,7 +317,7 @@
         wrline = top_level_block.writeline
         args = [arg.get_c_name('a%d' % i) for i, arg in enumerate(self.args)]
         decl = '%s(%s)' % (shadowname, ', '.join(args) or 'void')
-        wrline('static %s {' % self.result.get_c_name(decl))
+        wrline('static %s {' % QualType(self.result).get_c_name(decl))
         args = ['a%d' % i for i, arg in enumerate(self.args)]
         wrline('    %s%s(%s);' % ('' if isinstance(self.result, VoidType)
                                   else 'return ',
@@ -317,30 +344,25 @@
 
 
 class PointerType(BaseType):
-    _attrs_ = ('totype',)
-    _base_pattern       = {
-        (False, False): "*&",
-        (True, False): "(*&)",
-        (False, True): "*const &",
-        (True, True): "(*const &)",
-        }
+    _attrs_ = ('toqualtype',)
 
-    def __init__(self, totype, const):
-        self.totype = totype
-        self.const = const
-        base = self.totype.c_name_with_marker
-        parens = isinstance(self.totype, (ArrayType, FunctionType))
-        extra = self._base_pattern[parens, self.const]
-        self.c_name_with_marker = base.replace('&', extra)
+    def __init__(self, toqualtype):
+        assert isinstance(toqualtype, QualType)
+        self.toqualtype = toqualtype
+        if isinstance(toqualtype.type, (ArrayType, FunctionType)):
+            name = self.toqualtype.get_c_name('(*&)')
+        else:
+            name = self.toqualtype.get_c_name('*&')
+        self.c_name_with_marker = name
 
-    def inspect_nonconst_type(self, block, inspect):
+    def inspect_type(self, block, inspect, qualifiers):
         if isinstance(inspect, TypeInspector):
             star_p1 = inspect.fetch_star_p1()
             new_var = 'p%d' % (len(inspect.levels) + 2,)
             block.writedecl("char *%s;" % (new_var,))
             inspect.assign_to_p1("&%s" % (new_var,))
             #
-            if self.const:
+            if qualifiers & CRX_CONST:
                 errmsg = "type '%s' is not a pointer, but an array type" % (
                     inspect.get_comment_type(0, False),)
                 inspect.assign_target = new_var
@@ -371,13 +393,13 @@
                                          inspect.varname, self.get_c_name()))
                 block.writeline("(void)p1;")
             inspect = MissingInspector()
-        t1 = self.totype.inspect_type(block, inspect)
-        return block.write_crx_type_var('cb->get_pointer_type(cb, %s)' % t1)
+        t1 = self.toqualtype.inspect_qualtype(block, inspect)
+        return block.write_crx_type_var('cb->get_pointer_type(cb, %s, %d)'
+                                        % (t1, self.toqualtype.quals))
 
 
 class ArrayType(BaseType):
     _attrs_ = ('item', 'length')
-    const = True     # not really applicable, but must have one
 
     def __init__(self, item, length):
         self.item = item
@@ -385,20 +407,12 @@
         #
         if length is None:
             brackets = '&[]'
-        elif length == '...':
-            brackets = '&[/*...*/]'
         else:
             brackets = '&[%d]' % length
-        if ' &' in self.item.c_name_with_marker:
-            replace_from = ' &'
-        else:
-            replace_from = '&'
         self.c_name_with_marker = (
-            self.item.c_name_with_marker.replace(replace_from, brackets))
+            self.item.c_name_with_marker.replace('&', brackets))
 
-    def inspect_type(self, block, inspect):
-        # this class overrides inspect_type() instead of
-        # inspect_nonconst_type(), to avoid the extra call to get_const_type()
+    def inspect_type(self, block, inspect, qualifiers):
         if isinstance(inspect, TypeInspector):
             star_p1 = inspect.fetch_star_p1()
             errmsg = "type '%s' is not an array, but a pointer type" % (
@@ -424,7 +438,7 @@
             inspect = MissingInspector()
         else:
             star_p1 = None
-        t1 = self.item.inspect_type(block, inspect)
+        t1 = self.item.inspect_type(block, inspect, qualifiers)
         if star_p1 is not None:
             expr = 'cb->get_array_type(cb, %s, sizeof(%s) / sizeof(*%s))' % (
                 t1, star_p1, star_p1)
@@ -436,10 +450,9 @@
 class StructOrUnionOrEnum(BaseType):
     _attrs_ = ('kind', 'name')
 
-    def __init__(self, kind, name, const):
+    def __init__(self, kind, name):
         self.kind = kind
         self.name = name
-        self.const = const
         self.c_name_with_marker = '%s %s &' % (self.kind, self.name)
         if const:
             self.c_name_with_marker = 'const ' + self.c_name_with_marker
@@ -448,23 +461,19 @@
         return block.write_crx_type_var('cb->get_%s_type(cb, "%s")' % (
             self.kind, self.name))
 
-    def inspect_nonconst_type(self, block, inspect):
+    def inspect_type(self, block, inspect, qualifiers):
         inspect.flush()
         return self.get_type_var(block)
 
 
 class UserType(BaseType):
-    _attrs_ = ('name', 'realtype')
+    _attrs_ = ('name',)
 
-    def __init__(self, name, realtype, const=False):
+    def __init__(self, name):
         self.name = name
-        self.realtype = realtype
-        self.const = const or realtype.const
         self.c_name_with_marker = name + ' &'
-        if const:
-            self.c_name_with_marker = 'const ' + self.c_name_with_marker
 
-    def inspect_nonconst_type(self, block, inspect):
+    def inspect_type(self, block, inspect, qualifiers):
         inspect.flush()
         if isinstance(inspect, VarInspector):
             decl = self.get_c_name("*p1")
@@ -478,13 +487,12 @@
 
 class UnknownType(BaseType):
     _attrs_ = ('approx_name',)
-    const = False
 
     def __init__(self, approx_name):
         self.approx_name = approx_name
         self.c_name_with_marker = '... &'
 
-    def inspect_nonconst_type(self, block, inspect):
+    def inspect_type(self, block, inspect, qualifiers):
         inspect.flush()
         return block.write_crx_type_var('cb->get_unknown_type(cb, "%s")' % (
             self.approx_name,))
@@ -688,41 +696,43 @@
 
 
 class TypeDefDecl(object):
-    def __init__(self, name, type):
+    def __init__(self, name, qualtype):
         self.name = name
-        self.type = type
+        self.qualtype = qualtype
 
     def write_declaration(self, funcblock):
         block = CodeBlock(funcblock)
         inspect = TypeInspector(block, self.name)
-        t1 = self.type.inspect_type(block, inspect)
+        t1 = self.qualtype.inspect_qualtype(block, inspect)
         inspect.stop()
-        block.writeline('cb->define_type(cb, "%s", %s);' % (self.name, t1))
+        block.writeline('cb->define_type(cb, "%s", %s, %d);' % (
+            self.name, t1, self.qualtype.quals))
         funcblock.write_subblock(block)
 
 
 class VarDecl(object):
-    def __init__(self, name, type):
+    def __init__(self, name, qualtype):
         self.name = name
-        self.type = type
+        self.qualtype = qualtype
 
     def write_declaration(self, funcblock):
         block = CodeBlock(funcblock)
         inspect = VarInspector(block, self.name)
-        t1 = self.type.inspect_type(block, inspect)
-        shadow = self.type.shadow_global_var(block.crx_top_level, self.name)
-        block.writeline('cb->define_var(cb, "%s", %s, &%s);' % (
-            self.name, t1, shadow))
+        t1 = self.qualtype.inspect_qualtype(block, inspect)
+        shadow = self.qualtype.type.shadow_global_var(block.crx_top_level,
+                                                      self.name)
+        block.writeline('cb->define_var(cb, "%s", %s, %d, &%s);' % (
+            self.name, t1, self.qualtype.quals, shadow))
         funcblock.write_subblock(block)
 
 
 class FuncDecl(VarDecl):
     def write_declaration(self, funcblock):
-        if self.type.ellipsis:
+        if self.qualtype.type.ellipsis:
             VarDecl.write_declaration(self, funcblock)
         else:
             block = CodeBlock(funcblock)
-            self.type.write_func_decl(block, self.name)
+            self.qualtype.type.write_func_decl(block, self.name)
             funcblock.write_subblock(block)
 
 
diff --git a/creflect/test/codegen/002.c b/creflect/test/codegen/002.c
--- a/creflect/test/codegen/002.c
+++ b/creflect/test/codegen/002.c
@@ -11,8 +11,8 @@
         p1 = (void *)&p2;
         *p1 = (void *)0;    /* check that 'num_t' is a pointer type */
         t1 = cb->get_void_type(cb);
-        t2 = cb->get_pointer_type(cb, t1);
-        cb->define_type(cb, "num_t", t2);
+        t2 = cb->get_pointer_type(cb, t1, 0);
+        cb->define_type(cb, "num_t", t2, 0);
 #expect TYPEDEF num_t = PTR void
     }
 }
diff --git a/creflect/test/codegen/002b.c b/creflect/test/codegen/002b.c
--- a/creflect/test/codegen/002b.c
+++ b/creflect/test/codegen/002b.c
@@ -15,10 +15,10 @@
         **p1 = (void *)&p4;  /* check that '*num_t' is a pointer type */
         ***p1 = (void *)0;    /* check that '**num_t' is a pointer type */
         t1 = cb->get_void_type(cb);
-        t2 = cb->get_pointer_type(cb, t1);
-        t3 = cb->get_pointer_type(cb, t2);
-        t4 = cb->get_pointer_type(cb, t3);
-        cb->define_type(cb, "num_t", t4);
+        t2 = cb->get_pointer_type(cb, t1, 0);
+        t3 = cb->get_pointer_type(cb, t2, 0);
+        t4 = cb->get_pointer_type(cb, t3, 0);
+        cb->define_type(cb, "num_t", t4, 0);
 #expect TYPEDEF num_t = PTR PTR PTR void
     }
 }
diff --git a/creflect/test/codegen/002c.c b/creflect/test/codegen/002c.c
--- a/creflect/test/codegen/002c.c
+++ b/creflect/test/codegen/002c.c
@@ -4,16 +4,15 @@
 
 void test002c(_crx_builder_t *cb)
 {
-    _crx_type_t *t1, *t2, *t3;
+    _crx_type_t *t1, *t2;
     {
         num_t *p1;
         char *p2;
         p1 = (void *)&p2;
         *p1 = (void *)0;    /* check that 'num_t' is a pointer type */
         t1 = cb->get_void_type(cb);
-        t2 = cb->get_const_type(cb, t1);
-        t3 = cb->get_pointer_type(cb, t2);
-        cb->define_type(cb, "num_t", t3);
+        t2 = cb->get_pointer_type(cb, t1, 1);
+        cb->define_type(cb, "num_t", t2, 0);
 #expect TYPEDEF num_t = PTR CONST void
     }
 }
diff --git a/creflect/test/codegen/002d.c b/creflect/test/codegen/002d.c
new file mode 100644
--- /dev/null
+++ b/creflect/test/codegen/002d.c
@@ -0,0 +1,22 @@
+typedef void *const num_t;
+
+# ____________________________________________________________
+
+void test002d(_crx_builder_t *cb)
+{
+    _crx_type_t *t1, *t2;
+    {
+        num_t *p1;
+        char *p2;
+        p1 = (void *)&p2;
+        p2 = (void *)0;
+        if ((void *)p1 == (void *)*p1) {
+            cb->error(cb, "type 'num_t' is not a pointer, but an array type");
+            return;
+        }
+        t1 = cb->get_void_type(cb);
+        t2 = cb->get_pointer_type(cb, t1, 0);
+        cb->define_type(cb, "num_t", t2, 1);
+#expect TYPEDEF num_t = CONST PTR void
+    }
+}
diff --git a/creflect/test/codegen/003.c b/creflect/test/codegen/003.c
--- a/creflect/test/codegen/003.c
+++ b/creflect/test/codegen/003.c
@@ -12,7 +12,7 @@
         (void)(*p1 << 1);  /* check that 'num_t' is an integer type */
         *p1 = -1;  /* check that 'num_t' is not declared 'const' */
         t1 = _CRX_INT_TYPE(cb, *p1, "int");
-        cb->define_type(cb, "num_t", t1);
+        cb->define_type(cb, "num_t", t1, 0);
 #expect TYPEDEF num_t = int
     }
 }
diff --git a/creflect/test/codegen/003b.c b/creflect/test/codegen/003b.c
--- a/creflect/test/codegen/003b.c
+++ b/creflect/test/codegen/003b.c
@@ -12,7 +12,7 @@
         (void)(*p1 << 1);  /* check that 'num_t' is an integer type */
         *p1 = -1;  /* check that 'num_t' is not declared 'const' */
         t1 = _CRX_INT_TYPE(cb, *p1, "long");
-        cb->define_type(cb, "num_t", t1);
+        cb->define_type(cb, "num_t", t1, 0);
 #expect TYPEDEF num_t = long
     }
 }
diff --git a/creflect/test/codegen/003c.c b/creflect/test/codegen/003c.c
--- a/creflect/test/codegen/003c.c
+++ b/creflect/test/codegen/003c.c
@@ -12,7 +12,7 @@
         (void)(*p1 << 1);  /* check that 'num_t' is an integer type */
         *p1 = -1;  /* check that 'num_t' is not declared 'const' */
         t1 = _CRX_INT_TYPE(cb, *p1, "long long");
-        cb->define_type(cb, "num_t", t1);
+        cb->define_type(cb, "num_t", t1, 0);
 #expect TYPEDEF num_t = long long
     }
 }
diff --git a/creflect/test/codegen/003d.c b/creflect/test/codegen/003d.c
--- a/creflect/test/codegen/003d.c
+++ b/creflect/test/codegen/003d.c
@@ -12,7 +12,7 @@
         (void)(*p1 << 1);  /* check that 'num_t' is an integer type */
         *p1 = -1;  /* check that 'num_t' is not declared 'const' */
         t1 = _CRX_INT_TYPE(cb, *p1, "char");
-        cb->define_type(cb, "num_t", t1);
+        cb->define_type(cb, "num_t", t1, 0);
 #expect TYPEDEF num_t = signed char
     }
 }
diff --git a/creflect/test/codegen/003e.c b/creflect/test/codegen/003e.c
--- a/creflect/test/codegen/003e.c
+++ b/creflect/test/codegen/003e.c
@@ -16,7 +16,7 @@
             return;
         }
         t1 = cb->get_char_type(cb);
-        cb->define_type(cb, "num_t", t1);
+        cb->define_type(cb, "num_t", t1, 0);
 #expect TYPEDEF num_t = char
     }
 }
diff --git a/creflect/test/codegen/003f.c b/creflect/test/codegen/003f.c
--- a/creflect/test/codegen/003f.c
+++ b/creflect/test/codegen/003f.c
@@ -12,7 +12,7 @@
         (void)(*p1 << 1);  /* check that 'num_t' is an integer type */
         *p1 = -1;  /* check that 'num_t' is not declared 'const' */
         t1 = _CRX_INT_TYPE(cb, *p1, "long long");
-        cb->define_type(cb, "num_t", t1);
+        cb->define_type(cb, "num_t", t1, 0);
 #expect TYPEDEF num_t = unsigned long long
     }
 }
diff --git a/creflect/test/codegen/003g.c b/creflect/test/codegen/003g.c
--- a/creflect/test/codegen/003g.c
+++ b/creflect/test/codegen/003g.c
@@ -17,7 +17,7 @@
             return;
         }
         t1 = cb->get_float_type(cb, sizeof(*p1), "double");
-        cb->define_type(cb, "num_t", t1);
+        cb->define_type(cb, "num_t", t1, 0);
 #expect TYPEDEF num_t = double
     }
 }
diff --git a/creflect/test/codegen/003h.c b/creflect/test/codegen/003h.c
--- a/creflect/test/codegen/003h.c
+++ b/creflect/test/codegen/003h.c
@@ -16,7 +16,7 @@
         (void)(*p1 << 1);  /* check that 'num_t' is an integer type */
         *p1 = -1;  /* check that 'num_t' is not declared 'const' */
         t1 = _CRX_INT_TYPE(cb, *p1, "long long");
-        cb->define_type(cb, "num_t", t1);
+        cb->define_type(cb, "num_t", t1, 0);
 #expect TYPEDEF num_t = int
     }
 }
diff --git a/creflect/test/codegen/003i.c b/creflect/test/codegen/003i.c
--- a/creflect/test/codegen/003i.c
+++ b/creflect/test/codegen/003i.c
@@ -12,7 +12,7 @@
         (void)(*p1 << 1);  /* check that 'num_t' is an integer type */
         *p1 = -1;  /* check that 'num_t' is not declared 'const' */
         t1 = _CRX_INT_TYPE(cb, *p1, "char");
-        cb->define_type(cb, "num_t", t1);
+        cb->define_type(cb, "num_t", t1, 0);
 #expect TYPEDEF num_t = signed char
     }
 }
diff --git a/creflect/test/codegen/004.c b/creflect/test/codegen/004.c
--- a/creflect/test/codegen/004.c
+++ b/creflect/test/codegen/004.c
@@ -14,8 +14,8 @@
         (void)(**p1 << 1);  /* check that '*num_t' is an integer type */
         **p1 = -1;  /* check that '*num_t' is not declared 'const' */
         t1 = _CRX_INT_TYPE(cb, **p1, "int");
-        t2 = cb->get_pointer_type(cb, t1);
-        cb->define_type(cb, "num_t", t2);
+        t2 = cb->get_pointer_type(cb, t1, 0);
+        cb->define_type(cb, "num_t", t2, 0);
 #expect TYPEDEF num_t = PTR int
     }
 }
diff --git a/creflect/test/codegen/004b.c b/creflect/test/codegen/004b.c
--- a/creflect/test/codegen/004b.c
+++ b/creflect/test/codegen/004b.c
@@ -16,9 +16,9 @@
         (void)(***p1 << 1);  /* check that '**num_t' is an integer type */
         ***p1 = -1;  /* check that '**num_t' is not declared 'const' */
         t1 = _CRX_INT_TYPE(cb, ***p1, "int");
-        t2 = cb->get_pointer_type(cb, t1);
-        t3 = cb->get_pointer_type(cb, t2);
-        cb->define_type(cb, "num_t", t3);
+        t2 = cb->get_pointer_type(cb, t1, 0);
+        t3 = cb->get_pointer_type(cb, t2, 0);
+        cb->define_type(cb, "num_t", t3, 0);
 #expect TYPEDEF num_t = PTR PTR int
     }
 }
diff --git a/creflect/test/codegen/005.c b/creflect/test/codegen/005.c
--- a/creflect/test/codegen/005.c
+++ b/creflect/test/codegen/005.c
@@ -17,7 +17,7 @@
         **p1 = -1;  /* check that 'foo_t[]' is not declared 'const' */
         t1 = _CRX_INT_TYPE(cb, **p1, "int");
         t2 = cb->get_array_type(cb, t1, sizeof(*p1) / sizeof(**p1));
-        cb->define_type(cb, "foo_t", t2);
+        cb->define_type(cb, "foo_t", t2, 0);
 #expect TYPEDEF foo_t = ARRAY[27] int
     }
 }
diff --git a/creflect/test/codegen/005b.c b/creflect/test/codegen/005b.c
--- a/creflect/test/codegen/005b.c
+++ b/creflect/test/codegen/005b.c
@@ -18,9 +18,9 @@
         (void)(***p1 << 1);  /* check that '*foo_t[]' is an integer type */
         ***p1 = -1;  /* check that '*foo_t[]' is not declared 'const' */
         t1 = _CRX_INT_TYPE(cb, ***p1, "int");
-        t2 = cb->get_pointer_type(cb, t1);
+        t2 = cb->get_pointer_type(cb, t1, 0);
         t3 = cb->get_array_type(cb, t2, sizeof(*p1) / sizeof(**p1));
-        cb->define_type(cb, "foo_t", t3);
+        cb->define_type(cb, "foo_t", t3, 0);
 #expect TYPEDEF foo_t = ARRAY[27] PTR int
     }
 }
diff --git a/creflect/test/codegen/005c.c b/creflect/test/codegen/005c.c
--- a/creflect/test/codegen/005c.c
+++ b/creflect/test/codegen/005c.c
@@ -19,8 +19,8 @@
         ***p1 = -1;  /* check that '(*foo_t)[]' is not declared 'const' */
         t1 = _CRX_INT_TYPE(cb, ***p1, "int");
         t2 = cb->get_array_type(cb, t1, sizeof(**p1) / sizeof(***p1));
-        t3 = cb->get_pointer_type(cb, t2);
-        cb->define_type(cb, "foo_t", t3);
+        t3 = cb->get_pointer_type(cb, t2, 0);
+        cb->define_type(cb, "foo_t", t3, 0);
 #expect TYPEDEF foo_t = PTR ARRAY[27] int
     }
 }
diff --git a/creflect/test/codegen/005d.c b/creflect/test/codegen/005d.c
--- a/creflect/test/codegen/005d.c
+++ b/creflect/test/codegen/005d.c
@@ -38,16 +38,16 @@
         (void)(**********p1 << 1);  /* check that '**(***foo_t[][])[][]' is an integer type */
         **********p1 = -1;  /* check that '**(***foo_t[][])[][]' is not declared 'const' */
         t1 = _CRX_INT_TYPE(cb, **********p1, "int");
-        t2 = cb->get_pointer_type(cb, t1);
-        t3 = cb->get_pointer_type(cb, t2);
+        t2 = cb->get_pointer_type(cb, t1, 0);
+        t3 = cb->get_pointer_type(cb, t2, 0);
         t4 = cb->get_array_type(cb, t3, sizeof(*******p1) / sizeof(********p1));
         t5 = cb->get_array_type(cb, t4, sizeof(******p1) / sizeof(*******p1));
-        t6 = cb->get_pointer_type(cb, t5);
-        t7 = cb->get_pointer_type(cb, t6);
-        t8 = cb->get_pointer_type(cb, t7);
+        t6 = cb->get_pointer_type(cb, t5, 0);
+        t7 = cb->get_pointer_type(cb, t6, 0);
+        t8 = cb->get_pointer_type(cb, t7, 0);
         t9 = cb->get_array_type(cb, t8, sizeof(**p1) / sizeof(***p1));
         t10 = cb->get_array_type(cb, t9, sizeof(*p1) / sizeof(**p1));
-        cb->define_type(cb, "foo_t", t10);
+        cb->define_type(cb, "foo_t", t10, 0);
 #expect TYPEDEF foo_t = ARRAY[4] ARRAY[3] PTR PTR PTR ARRAY[27] ARRAY[91] PTR PTR int
     }
 }
diff --git a/creflect/test/codegen/006.c b/creflect/test/codegen/006.c
--- a/creflect/test/codegen/006.c
+++ b/creflect/test/codegen/006.c
@@ -4,7 +4,7 @@
 
 void test006(_crx_builder_t *cb)
 {
-    _crx_type_t *t1, *t2;
+    _crx_type_t *t1;
     {
         num_t *p1;
         char b[sizeof(*p1)];
@@ -12,8 +12,7 @@
         p1 = (void *)b;
         (void)(*p1 << 1);  /* check that 'num_t' is an integer type */
         t1 = _CRX_INT_TYPE(cb, *p1, "int");
-        t2 = cb->get_const_type(cb, t1);
-        cb->define_type(cb, "num_t", t2);
+        cb->define_type(cb, "num_t", t1, 1);
 #expect TYPEDEF num_t = CONST unsigned int
     }
 }
diff --git a/creflect/test/codegen/006b.c b/creflect/test/codegen/006b.c
--- a/creflect/test/codegen/006b.c
+++ b/creflect/test/codegen/006b.c
@@ -4,7 +4,7 @@
 
 void test006b(_crx_builder_t *cb)
 {
-    _crx_type_t *t1, *t2, *t3;
+    _crx_type_t *t1, *t2;
     {
         num_t *p1;
         char *p2;
@@ -18,9 +18,8 @@
         (void)(**p1 << 1);  /* check that '*num_t' is an integer type */
         **p1 = -1;  /* check that '*num_t' is not declared 'const' */
         t1 = _CRX_INT_TYPE(cb, **p1, "int");
-        t2 = cb->get_pointer_type(cb, t1);
-        t3 = cb->get_const_type(cb, t2);
-        cb->define_type(cb, "num_t", t3);
+        t2 = cb->get_pointer_type(cb, t1, 0);
+        cb->define_type(cb, "num_t", t2, 1);
 #expect TYPEDEF num_t = CONST PTR int
     }
 }
diff --git a/creflect/test/codegen/006c.c b/creflect/test/codegen/006c.c
--- a/creflect/test/codegen/006c.c
+++ b/creflect/test/codegen/006c.c
@@ -4,7 +4,7 @@
 
 void test006c(_crx_builder_t *cb)
 {
-    _crx_type_t *t1, *t2, *t3, *t4;
+    _crx_type_t *t1, *t2, *t3;
     {
         foo_t *p1;
         char *p2;
@@ -23,9 +23,8 @@
         ***p1 = -1;  /* check that '(*foo_t)[]' is not declared 'const' */
         t1 = _CRX_INT_TYPE(cb, ***p1, "int");
         t2 = cb->get_array_type(cb, t1, sizeof(**p1) / sizeof(***p1));
-        t3 = cb->get_pointer_type(cb, t2);
-        t4 = cb->get_const_type(cb, t3);
-        cb->define_type(cb, "foo_t", t4);
+        t3 = cb->get_pointer_type(cb, t2, 0);
+        cb->define_type(cb, "foo_t", t3, 1);
 #expect TYPEDEF foo_t = CONST PTR ARRAY[27] int
     }
 }
diff --git a/creflect/test/codegen/007.c b/creflect/test/codegen/007.c
--- a/creflect/test/codegen/007.c
+++ b/creflect/test/codegen/007.c
@@ -7,7 +7,7 @@
     _crx_type_t *t1;
     {
         t1 = cb->get_unknown_type(cb, "$num_t");
-        cb->define_type(cb, "num_t", t1);
+        cb->define_type(cb, "num_t", t1, 0);
 #expect TYPEDEF num_t = UNKNOWN $num_t
     }
 }
diff --git a/creflect/test/codegen/func-001b.c b/creflect/test/codegen/func-001b.c
--- a/creflect/test/codegen/func-001b.c
+++ b/creflect/test/codegen/func-001b.c
@@ -36,25 +36,33 @@
 
 void testfunc_001b(_crx_builder_t *cb)
 {
-    _crx_type_t *t1, *t2, *a3[1], *t4, *a5[2], *a6[2];
+    _crx_type_t *t1, *t2, *t3;
+    _crx_qual_type a1[1];
+    _crx_qual_type a2[2];
+    _crx_qual_type a3[2];
     {
         t1 = cb->get_unsigned_type(cb, sizeof(unsigned int), "unsigned int");
         t2 = cb->get_signed_type(cb, sizeof(long), "long");
-        a3[0] = t2;
-        cb->define_func(cb, "f", t1, a3, 1, &testfunc_001b__c_f, &testfunc_001b__d_f);
+        a1[0].type = t2;
+        a1[0].qualifiers = 0;
+        cb->define_func(cb, "f", t1, a1, 1, &testfunc_001b__c_f, &testfunc_001b__d_f);
 #expect FUNC f: long -> unsigned int
     }
     {
-        t4 = cb->get_signed_type(cb, sizeof(int), "int");
-        a5[0] = t4;
-        a5[1] = t4;
-        cb->define_func(cb, "g", t2, a5, 2, &testfunc_001b__c_g, &testfunc_001b__d_g);
+        t3 = cb->get_signed_type(cb, sizeof(int), "int");
+        a2[0].type = t3;
+        a2[0].qualifiers = 0;
+        a2[1].type = t3;
+        a2[1].qualifiers = 0;
+        cb->define_func(cb, "g", t2, a2, 2, &testfunc_001b__c_g, &testfunc_001b__d_g);
 #expect FUNC g: int -> int -> long
     }
     {
-        a6[0] = t4;
-        a6[1] = t4;
-        cb->define_func(cb, "h", t2, a6, 2, &testfunc_001b__c_h, &testfunc_001b__d_h);
+        a3[0].type = t3;
+        a3[0].qualifiers = 0;
+        a3[1].type = t3;
+        a3[1].qualifiers = 0;
+        cb->define_func(cb, "h", t2, a3, 2, &testfunc_001b__c_h, &testfunc_001b__d_h);
 #expect FUNC h: int -> int -> long
     }
 }
diff --git a/creflect/test/codegen/func-001c.c b/creflect/test/codegen/func-001c.c
--- a/creflect/test/codegen/func-001c.c
+++ b/creflect/test/codegen/func-001c.c
@@ -16,12 +16,14 @@
 
 void testfunc_001c(_crx_builder_t *cb)
 {
-    _crx_type_t *t1, *t2, *a3[1];
+    _crx_type_t *t1, *t2;
+    _crx_qual_type a1[1];
     {
         t1 = cb->get_float_type(cb, sizeof(double), "double");
         t2 = cb->get_float_type(cb, sizeof(float), "float");
-        a3[0] = t2;
-        cb->define_func(cb, "f", t1, a3, 1, &testfunc_001c__c_f, &testfunc_001c__d_f);
+        a1[0].type = t2;
+        a1[0].qualifiers = 0;
+        cb->define_func(cb, "f", t1, a1, 1, &testfunc_001c__c_f, &testfunc_001c__d_f);
 #expect FUNC f: float -> double
     }
 }
diff --git a/creflect/test/codegen/func-002.c b/creflect/test/codegen/func-002.c
--- a/creflect/test/codegen/func-002.c
+++ b/creflect/test/codegen/func-002.c
@@ -9,14 +9,16 @@
 
 void testfunc_002(_crx_builder_t *cb)
 {
-    _crx_type_t *t1, *a2[1], *t3;
+    _crx_type_t *t1, *t2;
+    _crx_qual_type a1[1];
     {
         int (*p1)(int, ...) = &f;  /* check that 'f' is a function with exactly the given signature */
         (void)p1;
         t1 = cb->get_signed_type(cb, sizeof(int), "int");
-        a2[0] = t1;
-        t3 = cb->get_ellipsis_function_type(cb, t1, a2, 1);
-        cb->define_var(cb, "f", t3, &f);
+        a1[0].type = t1;
+        a1[0].qualifiers = 0;
+        t2 = cb->get_ellipsis_function_type(cb, t1, a1, 1);
+        cb->define_var(cb, "f", t2, 0, &f);
 #expect VAR f: FUNC( int -> ... -> int )
     }
 }
diff --git a/creflect/test/codegen/func-003.c b/creflect/test/codegen/func-003.c
--- a/creflect/test/codegen/func-003.c
+++ b/creflect/test/codegen/func-003.c
@@ -9,7 +9,8 @@
 
 void testfunc_003(_crx_builder_t *cb)
 {
-    _crx_type_t *t1, *t2, *a3[3], *t4, *t5;
+    _crx_type_t *t1, *t2, *t3, *t4;
+    _crx_qual_type a1[3];
     {
         func_t *p1;
         char *p2;
@@ -17,12 +18,15 @@
         *p1 = (void *)0;    /* check that 'func_t' is a pointer type */
         t1 = cb->get_signed_type(cb, sizeof(int), "int");
         t2 = cb->get_signed_type(cb, sizeof(long), "long");
-        a3[0] = t2;
-        a3[1] = t2;
-        a3[2] = t1;
-        t4 = cb->get_function_type(cb, t1, a3, 3, &testfunc_003__f1);
-        t5 = cb->get_pointer_type(cb, t4);
-        cb->define_type(cb, "func_t", t5);
+        a1[0].type = t2;
+        a1[0].qualifiers = 0;
+        a1[1].type = t2;
+        a1[1].qualifiers = 0;
+        a1[2].type = t1;
+        a1[2].qualifiers = 0;
+        t3 = cb->get_function_type(cb, t1, a1, 3, &testfunc_003__f1);
+        t4 = cb->get_pointer_type(cb, t3, 0);
+        cb->define_type(cb, "func_t", t4, 0);
 #expect TYPEDEF func_t = PTR FUNC( long -> long -> int -> int )
     }
 }
diff --git a/creflect/test/codegen/func-003b.c b/creflect/test/codegen/func-003b.c
--- a/creflect/test/codegen/func-003b.c
+++ b/creflect/test/codegen/func-003b.c
@@ -4,7 +4,8 @@
 
 void testfunc_003b(_crx_builder_t *cb)
 {
-    _crx_type_t *t1, *t2, *a3[1], *t4, *t5;
+    _crx_type_t *t1, *t2, *t3, *t4;
+    _crx_qual_type a1[1];
     {
         func_t *p1;
         char *p2;
@@ -12,10 +13,11 @@
         *p1 = (int (*)(long, ...))0;    /* check that 'func_t' is a function pointer type with exactly the given signature */
         t1 = cb->get_signed_type(cb, sizeof(int), "int");
         t2 = cb->get_signed_type(cb, sizeof(long), "long");
-        a3[0] = t2;
-        t4 = cb->get_ellipsis_function_type(cb, t1, a3, 1);
-        t5 = cb->get_pointer_type(cb, t4);
-        cb->define_type(cb, "func_t", t5);
+        a1[0].type = t2;
+        a1[0].qualifiers = 0;
+        t3 = cb->get_ellipsis_function_type(cb, t1, a1, 1);
+        t4 = cb->get_pointer_type(cb, t3, 0);
+        cb->define_type(cb, "func_t", t4, 0);
 #expect TYPEDEF func_t = PTR FUNC( long -> ... -> int )
     }
 }
diff --git a/creflect/test/codegen/func-004.c b/creflect/test/codegen/func-004.c
--- a/creflect/test/codegen/func-004.c
+++ b/creflect/test/codegen/func-004.c
@@ -9,7 +9,8 @@
 
 void testfunc_004(_crx_builder_t *cb)
 {
-    _crx_type_t *t1, *t2, *a3[2], *t4, *t5, *t6;
+    _crx_type_t *t1, *t2, *t3, *t4, *t5;
+    _crx_qual_type a1[2];
     {
         func_t *p1;
         char *p2;
@@ -19,12 +20,14 @@
         **p1 = (int (*)(long, int))0;    /* check that '*func_t' is a function pointer type with exactly the given signature */
         t1 = cb->get_signed_type(cb, sizeof(int), "int");
         t2 = cb->get_signed_type(cb, sizeof(long), "long");
-        a3[0] = t2;
-        a3[1] = t1;
-        t4 = cb->get_function_type(cb, t1, a3, 2, &testfunc_004__f1);
-        t5 = cb->get_pointer_type(cb, t4);
-        t6 = cb->get_pointer_type(cb, t5);
-        cb->define_type(cb, "func_t", t6);
+        a1[0].type = t2;
+        a1[0].qualifiers = 0;
+        a1[1].type = t1;
+        a1[1].qualifiers = 0;
+        t3 = cb->get_function_type(cb, t1, a1, 2, &testfunc_004__f1);
+        t4 = cb->get_pointer_type(cb, t3, 0);
+        t5 = cb->get_pointer_type(cb, t4, 0);
+        cb->define_type(cb, "func_t", t5, 0);
 #expect TYPEDEF func_t = PTR PTR FUNC( long -> int -> int )
     }
 }
diff --git a/creflect/test/codegen/func-005.c b/creflect/test/codegen/func-005.c
--- a/creflect/test/codegen/func-005.c
+++ b/creflect/test/codegen/func-005.c
@@ -1,16 +1,16 @@
-void f(int[], long(char, short));
+void f(const int[], long(char, short));
 
 # ____________________________________________________________
 
-void f(int a[], long g(char, short)) { }
+void f(const int a[], long g(char, short)) { }
 
 # ____________________________________________________________
 
 static void testfunc_005__c_f(void *args[], void *result) {
-    f(*(int **)args[0], *(long (**)(char, short))args[1]);
+    f(*(int const **)args[0], *(long (**)(char, short))args[1]);
 }
 
-static void testfunc_005__d_f(int *a0, long (*a1)(char, short)) {
+static void testfunc_005__d_f(int const a0[], long a1(char, short)) {
     f(a0, a1);
 }
 
@@ -21,21 +21,26 @@
 
 void testfunc_005(_crx_builder_t *cb)
 {
-    _crx_type_t *t1, *t2, *t3, *t4, *t5, *t6, *a7[2], *t8, *t9, *a10[2];
+    _crx_type_t *t1, *t2, *t3, *t4, *t5, *t6, *t7;
+    _crx_qual_type a1[2];
+    _crx_qual_type a2[2];
     {
         t1 = cb->get_void_type(cb);
         t2 = cb->get_signed_type(cb, sizeof(int), "int");
-        t3 = cb->get_pointer_type(cb, t2);
+        t3 = cb->get_incomplete_array_type(cb, t2);
+        a1[0].type = t3;
+        a1[0].qualifiers = 1;
         t4 = cb->get_signed_type(cb, sizeof(long), "long");
         t5 = cb->get_char_type(cb);
+        a2[0].type = t5;
+        a2[0].qualifiers = 0;
         t6 = cb->get_signed_type(cb, sizeof(short), "short");
-        a7[0] = t5;
-        a7[1] = t6;
-        t8 = cb->get_function_type(cb, t4, a7, 2, &testfunc_005__f4);
-        t9 = cb->get_pointer_type(cb, t8);
-        a10[0] = t3;
-        a10[1] = t9;
-        cb->define_func(cb, "f", t1, a10, 2, &testfunc_005__c_f, &testfunc_005__d_f);
-#expect FUNC f: PTR int -> PTR FUNC( char -> short -> long ) -> void
+        a2[1].type = t6;
+        a2[1].qualifiers = 0;
+        t7 = cb->get_function_type(cb, t4, a2, 2, &testfunc_005__f4);
+        a1[1].type = t7;
+        a1[1].qualifiers = 0;
+        cb->define_func(cb, "f", t1, a1, 2, &testfunc_005__c_f, &testfunc_005__d_f);
+#expect FUNC f: CONST ARRAY[] int -> FUNC( char -> short -> long ) -> void
     }
 }
diff --git a/creflect/test/codegen/glob-001.c b/creflect/test/codegen/glob-001.c
--- a/creflect/test/codegen/glob-001.c
+++ b/creflect/test/codegen/glob-001.c
@@ -9,8 +9,8 @@
         void *p1 = someglob;  /* check that 'someglob' is a pointer */
         if (0) { someglob = p1; }  /* check that 'someglob' is a pointer variable, and not an array or a constant */
         t1 = cb->get_void_type(cb);
-        t2 = cb->get_pointer_type(cb, t1);
-        cb->define_var(cb, "someglob", t2, &someglob);
+        t2 = cb->get_pointer_type(cb, t1, 0);
+        cb->define_var(cb, "someglob", t2, 0, &someglob);
 #expect VAR someglob: PTR void
     }
 }
diff --git a/creflect/test/codegen/glob-002.c b/creflect/test/codegen/glob-002.c
--- a/creflect/test/codegen/glob-002.c
+++ b/creflect/test/codegen/glob-002.c
@@ -10,7 +10,7 @@
         (void)p1;
         t1 = cb->get_char_type(cb);
         t2 = cb->get_array_type(cb, t1, sizeof(someglob) / sizeof(*someglob));
-        cb->define_var(cb, "someglob", t2, &someglob);
+        cb->define_var(cb, "someglob", t2, 0, &someglob);
 #expect VAR someglob: ARRAY[100] char
     }
 }
diff --git a/creflect/test/codegen/glob-002b.c b/creflect/test/codegen/glob-002b.c
--- a/creflect/test/codegen/glob-002b.c
+++ b/creflect/test/codegen/glob-002b.c
@@ -12,7 +12,7 @@
         (void)p1;
         t1 = cb->get_signed_type(cb, sizeof(int), "int");
         t2 = cb->get_array_type(cb, t1, sizeof(someglob) / sizeof(*someglob));
-        cb->define_var(cb, "someglob", t2, &someglob);
+        cb->define_var(cb, "someglob", t2, 0, &someglob);
 #expect VAR someglob: ARRAY[100] int
     }
 }
diff --git a/creflect/test/codegen/glob-002c.c b/creflect/test/codegen/glob-002c.c
--- a/creflect/test/codegen/glob-002c.c
+++ b/creflect/test/codegen/glob-002c.c
@@ -11,7 +11,7 @@
         int *p1 = &someglob;  /* check that 'someglob' is of type 'int' */
         (void)p1;
         t1 = cb->get_signed_type(cb, sizeof(int), "int");
-        cb->define_var(cb, "someglob", t1, &someglob);
+        cb->define_var(cb, "someglob", t1, 0, &someglob);
 #expect VAR someglob: int
     }
 }
diff --git a/creflect/test/codegen/glob-002d.c b/creflect/test/codegen/glob-002d.c
--- a/creflect/test/codegen/glob-002d.c
+++ b/creflect/test/codegen/glob-002d.c
@@ -9,8 +9,8 @@
         int **p1 = &someglob;  /* check that 'someglob' is of type 'int *' */
         (void)p1;
         t1 = cb->get_signed_type(cb, sizeof(int), "int");
-        t2 = cb->get_pointer_type(cb, t1);
-        cb->define_var(cb, "someglob", t2, &someglob);
+        t2 = cb->get_pointer_type(cb, t1, 0);
+        cb->define_var(cb, "someglob", t2, 0, &someglob);
 #expect VAR someglob: PTR int
     }
 }
diff --git a/creflect/test/codegen/glob-004.c b/creflect/test/codegen/glob-004.c
--- a/creflect/test/codegen/glob-004.c
+++ b/creflect/test/codegen/glob-004.c
@@ -18,8 +18,8 @@
         a3[0] = t2;
         a3[1] = t2;
         t4 = cb->get_function_type(cb, t1, a3, 2, &testglob_004__f1);
-        t5 = cb->get_pointer_type(cb, t4);
-        cb->define_var(cb, "someglob", t5, &someglob);
+        t5 = cb->get_pointer_type(cb, t4, 0);
+        cb->define_var(cb, "someglob", t5, 0, &someglob);
 #expect VAR someglob: PTR FUNC( long -> long -> int )
     }
 }
diff --git a/creflect/test/codegen/glob-005.c b/creflect/test/codegen/glob-005.c
--- a/creflect/test/codegen/glob-005.c
+++ b/creflect/test/codegen/glob-005.c
@@ -13,15 +13,16 @@
         p1 = (void *)&p2;
         *p1 = (void *)0;    /* check that 'mytype_t' is a pointer type */
         t1 = cb->get_void_type(cb);
-        t2 = cb->get_pointer_type(cb, t1);
-        cb->define_type(cb, "mytype_t", t2);
+        t2 = cb->get_pointer_type(cb, t1, 0);
+        cb->define_type(cb, "mytype_t", t2, 0);
 #expect TYPEDEF mytype_t = PTR void
     }
     {
         mytype_t *p1 = &foobar;  /* check that 'foobar' is of type 'mytype_t' */
+        int q1 = 0;
         (void)p1;
-        t3 = cb->get_user_type(cb, "mytype_t");
-        cb->define_var(cb, "foobar", t3, &foobar);
+        t3 = cb->get_user_type(cb, "mytype_t", &q1);
+        cb->define_var(cb, "foobar", t3, q1, &foobar);
 #expect VAR foobar: mytype_t
     }
 }
diff --git a/creflect/test/codegen/struct-001.c b/creflect/test/codegen/struct-001.c
--- a/creflect/test/codegen/struct-001.c
+++ b/creflect/test/codegen/struct-001.c
@@ -20,6 +20,7 @@
         t2 = _CRX_INT_TYPE(cb, p1->aa, "int");
         d1[0].name = "aa";
         d1[0].type = t2;
+        d1[0].qualifiers = 0;
         d1[0].offset = o;
         d1[0].numbits = -1;
         d1[0].bitshift = -1;
@@ -34,6 +35,7 @@
         t3 = _CRX_INT_TYPE(cb, p1->bb, "int");
         d1[1].name = "bb";
         d1[1].type = t3;
+        d1[1].qualifiers = 0;
         d1[1].offset = o;
         d1[1].numbits = -1;
         d1[1].bitshift = -1;
diff --git a/creflect/test/codegen/struct-001b.c b/creflect/test/codegen/struct-001b.c
--- a/creflect/test/codegen/struct-001b.c
+++ b/creflect/test/codegen/struct-001b.c
@@ -6,7 +6,7 @@
 
 void teststruct_001b(_crx_builder_t *cb)
 {
-    _crx_type_t *t1, *t2, *t3;
+    _crx_type_t *t1, *t2;
     _crx_field_t d1[1];
     t1 = cb->get_struct_type(cb, "foo_s");
     {
@@ -17,9 +17,9 @@
         p1 = (void *)(((char *)b) - o);
         (void)(p1->aa << 1);  /* check that 'struct foo_s::aa' is an integer type */
         t2 = _CRX_INT_TYPE(cb, p1->aa, "int");
-        t3 = cb->get_const_type(cb, t2);
         d1[0].name = "aa";
-        d1[0].type = t3;
+        d1[0].type = t2;
+        d1[0].qualifiers = _CRX_CONST;
         d1[0].offset = o;
         d1[0].numbits = -1;
         d1[0].bitshift = -1;
diff --git a/creflect/test/codegen/struct-002.c b/creflect/test/codegen/struct-002.c
--- a/creflect/test/codegen/struct-002.c
+++ b/creflect/test/codegen/struct-002.c
@@ -16,9 +16,10 @@
         p1 = (void *)(((char *)&p2) - o);
         p1->aa = (void *)0;    /* check that 'struct foo_s::aa' is a pointer type */
         t2 = cb->get_void_type(cb);
-        t3 = cb->get_pointer_type(cb, t2);
+        t3 = cb->get_pointer_type(cb, t2, 0);
         d1[0].name = "aa";
         d1[0].type = t3;
+        d1[0].qualifiers = 0;
         d1[0].offset = o;
         d1[0].numbits = -1;
         d1[0].bitshift = -1;
diff --git a/creflect/test/codegen/struct-003.c b/creflect/test/codegen/struct-003.c
--- a/creflect/test/codegen/struct-003.c
+++ b/creflect/test/codegen/struct-003.c
@@ -19,9 +19,10 @@
         (void)(*p1->aa << 1);  /* check that '*struct foo_s::aa' is an integer type */
         *p1->aa = -1;  /* check that '*struct foo_s::aa' is not declared 'const' */
         t2 = _CRX_INT_TYPE(cb, *p1->aa, "int");
-        t3 = cb->get_pointer_type(cb, t2);
+        t3 = cb->get_pointer_type(cb, t2, 0);
         d1[0].name = "aa";
         d1[0].type = t3;
+        d1[0].qualifiers = 0;
         d1[0].offset = o;
         d1[0].numbits = -1;
         d1[0].bitshift = -1;
diff --git a/creflect/test/codegen/struct-004.c b/creflect/test/codegen/struct-004.c
--- a/creflect/test/codegen/struct-004.c
+++ b/creflect/test/codegen/struct-004.c
@@ -24,6 +24,7 @@
         t3 = cb->get_array_type(cb, t2, sizeof(p1->aa) / sizeof(*p1->aa));
         d1[0].name = "aa";
         d1[0].type = t3;
+        d1[0].qualifiers = 0;
         d1[0].offset = o;
         d1[0].numbits = -1;
         d1[0].bitshift = -1;
diff --git a/creflect/test/codegen/struct-005.c b/creflect/test/codegen/struct-005.c
--- a/creflect/test/codegen/struct-005.c
+++ b/creflect/test/codegen/struct-005.c
@@ -17,6 +17,7 @@
         t2 = _CRX_INT_TYPE(cb, p1->aa, "int");
         d1[0].name = "aa";
         d1[0].type = t2;
+        d1[0].qualifiers = 0;
         d1[0].offset = o;
         d1[0].numbits = -1;
         d1[0].bitshift = -1;
@@ -27,7 +28,7 @@
 #expect STRUCT $foo_t:
 #expect | aa: int
     {
-        cb->define_type(cb, "foo_t", t1);
+        cb->define_type(cb, "foo_t", t1, 0);
 #expect TYPEDEF foo_t = STRUCT $foo_t
     }
 }
diff --git a/creflect/test/codegen/struct-005b.c b/creflect/test/codegen/struct-005b.c
--- a/creflect/test/codegen/struct-005b.c
+++ b/creflect/test/codegen/struct-005b.c
@@ -17,6 +17,7 @@
         t2 = _CRX_INT_TYPE(cb, p1->aa, "int");
         d1[0].name = "aa";
         d1[0].type = t2;
+        d1[0].qualifiers = 0;
         d1[0].offset = o;
         d1[0].numbits = -1;
         d1[0].bitshift = -1;
@@ -31,8 +32,8 @@
         char *p2;
         p1 = (void *)&p2;
         *p1 = (void *)0;    /* check that 'foo_p' is a pointer type */
-        t3 = cb->get_pointer_type(cb, t1);
-        cb->define_type(cb, "foo_p", t3);
+        t3 = cb->get_pointer_type(cb, t1, 0);
+        cb->define_type(cb, "foo_p", t3, 0);
 #expect TYPEDEF foo_p = PTR STRUCT $$foo_p
     }
 }
diff --git a/creflect/test/codegen/struct-006.c b/creflect/test/codegen/struct-006.c
--- a/creflect/test/codegen/struct-006.c
+++ b/creflect/test/codegen/struct-006.c
@@ -11,7 +11,7 @@
                  NULL, 0);
 #expect STRUCT $foo_t:
     {
-        cb->define_type(cb, "foo_t", t1);
+        cb->define_type(cb, "foo_t", t1, 0);
 #expect TYPEDEF foo_t = STRUCT $foo_t
     }
 }
diff --git a/creflect/test/test_cparser.py b/creflect/test/test_cparser.py
--- a/creflect/test/test_cparser.py
+++ b/creflect/test/test_cparser.py
@@ -1,6 +1,7 @@
 import py
 from cStringIO import StringIO
 from creflect.cparser import CSource, remove_comments, CDefError
+from creflect.model import TypeDefDecl, VarDecl
 
 
 def test_remove_comments():
@@ -41,3 +42,46 @@
       TypeDecl: f, []
         IdentifierType: ['unsigned', 'long']
 """
+
+def get_c_name(csrc):
+    csource = CSource("typedef %s;" % (csrc,))
+    typedefdecl = csource.get_declarations()[-1]
+    assert isinstance(typedefdecl, TypeDefDecl)
+    return typedefdecl.qualtype.get_c_name('x')
+
+def get_decl_name(csrc):
+    csource = CSource("%s;" % (csrc,))
+    vardecl = csource.get_declarations()[-1]
+    assert isinstance(vardecl, VarDecl)   # or subclass FuncDecl or ConstDecl
+    return vardecl.qualtype.get_c_name('x')
+
+def test_c_name():
+    assert get_c_name('long int x') == 'long x'
+    assert get_c_name('int***x') == 'int ***x'
+    assert get_c_name('const int x') == 'int const x'
+    assert get_c_name('int(x[5])') == 'int x[5]'
+    assert get_c_name('int(*x[5])') == 'int *x[5]'
+    assert get_c_name('int(*x)[5]') == 'int (*x)[5]'
+    assert get_c_name('const int *x[5]') == 'int const *x[5]'
+    assert get_c_name('int *const x[5]') == 'int *const x[5]'
+    assert get_c_name('int (*const x)[5]') == 'int (*const x)[5]'
+    assert get_c_name('const int(*x)[5]') == 'int const (*x)[5]'
+    #
+    assert get_c_name('const int x[5]') == 'int const x[5]'
+    assert get_c_name('void* x') == 'void *x'
+    assert get_c_name('int x(long,...)') == 'int x(long, ...)'
+    assert get_c_name('const int x(long,...)') == 'int x(long, ...)'  # ignored
+    assert get_c_name('int *(x(long,...))') == 'int *x(long, ...)'
+    assert get_c_name('int (*x)(long,...)') == 'int (*x)(long, ...)'
+    assert get_c_name('const int *x') == 'int const *x'
+    assert get_c_name('int * const x') == 'int *const x'
+    #
+    # hacks to declare two types
+    assert get_c_name('int tt; typedef tt x') == 'tt x'
+    assert get_c_name('int tt; typedef const tt x') == 'tt const x'
+    assert get_c_name('const __crx_unknown_t *x') == '... const *x'
+
+def test_decl_name():
+    assert get_decl_name('int x') == 'int x'
+    assert get_decl_name('const int x') == 'int const x'
+    assert get_decl_name('int x(long)') == 'int x(long)'
diff --git a/setup.py b/setup.py
new file mode 100644
--- /dev/null
+++ b/setup.py
@@ -0,0 +1,10 @@
+from distutils.core import setup
+from distutils.extension import Extension
+
+setup(
+    ext_modules=[
+        Extension(name = 'zeffir',
+                  sources=['zeffir/zeffir.c',
+                           'creflect/creflect_cdecl.c'],
+        )
+    ])
diff --git a/zeffir/zef_builder.c b/zeffir/zef_builder.c
--- a/zeffir/zef_builder.c
+++ b/zeffir/zef_builder.c
@@ -165,10 +165,12 @@
 }
 
 static void zef_define_num_const(_crx_builder_t *cb, const char *name,
-                                 _crx_type_t *t, _crx_num_const_t *value)
+                                 _crx_type_t *ct, _crx_num_const_t *value)
 {
     PyObject *dict = ((zeffir_builder_t *)cb)->dict;
 
+    assert(ct->ct_flags & CT_PRIMITIVE_ANY);
+
     PyObject *x = PyInt_FromLong(value->as_int);
     if (x == NULL)
         return;

From noreply at buildbot.pypy.org  Sun Nov 30 23:08:42 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Sun, 30 Nov 2014 23:08:42 +0100 (CET)
Subject: [pypy-commit] creflect default: still in-progress
Message-ID: <20141130220842.116231D2865@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r130:47d913cc3830
Date: 2014-11-30 23:09 +0100
http://bitbucket.org/cffi/creflect/changeset/47d913cc3830/

Log:	still in-progress

diff --git a/creflect/codegen.py b/creflect/codegen.py
--- a/creflect/codegen.py
+++ b/creflect/codegen.py
@@ -32,12 +32,16 @@
     def _get_next_name(self, prefix):
         return '%s%d' % (prefix, len(self.crx_type_vars) + 1)
 
-    def write_crx_type_var(self, expr):
+    def write_crx_type_var(self, expr, is_qualtype=False):
         if expr in self.crx_type_cache:
             return self.crx_type_cache[expr]
         #
-        t1 = self._get_next_name('t')
-        self.crx_type_vars.append('*%s' % t1)
+        if not is_qualtype:
+            t1 = 't%d' % (len(self.crx_type_vars) + 1)
+            self.crx_type_vars.append('*%s' % t1)
+        else:
+            t1 = 'q%d' % (len(self.crx_qualtype_vars) + 1)
+            self.crx_qualtype_vars.append('%s' % t1)
         self.crx_type_cache[expr] = t1
         self.writeline('%s = %s;' % (t1, expr))
         return t1
diff --git a/creflect/creflect.h b/creflect/creflect.h
--- a/creflect/creflect.h
+++ b/creflect/creflect.h
@@ -66,7 +66,8 @@
                      size_t, size_t, _crx_field_t[], int);
     void (*complete_enum)(_CRX_SELF, _crx_type_t *, _crx_type_t *);
     void (*define_type)(_CRX_SELF, const char *, _crx_type_t *, int);
-    void (*define_var)(_CRX_SELF, const char *, _crx_type_t *, int, void *);
+    void (*define_var)(_CRX_SELF, const char *, _crx_type_t *, int,
+                       const void *);
     void (*define_func)(_CRX_SELF, const char *, _crx_type_t *,
                         _crx_qual_type[], int, _crx_trampoline0_fn, void *);
     void (*define_num_const)(_CRX_SELF, const char *, _crx_type_t *,
diff --git a/creflect/creflect_debug_print.c b/creflect/creflect_debug_print.c
--- a/creflect/creflect_debug_print.c
+++ b/creflect/creflect_debug_print.c
@@ -228,7 +228,7 @@
 }
 
 static void tst_define_var(_crx_builder_t *cb, const char *name,
-                           _crx_type_t *type, int qualifiers, void *addr)
+                           _crx_type_t *type, int qualifiers, const void *addr)
 {
     printf("VAR %s: %s\n", name, show_qualtype2(type, qualifiers));
 }
diff --git a/creflect/model.py b/creflect/model.py
--- a/creflect/model.py
+++ b/creflect/model.py
@@ -15,14 +15,26 @@
         if self.quals & CRX_VOLATILE:
             replace_with = ('volatile ' + replace_with).rstrip()
         #
-        if replace_with == '' and ' &' in self.type.c_name_with_marker:
+        if (replace_with == '' or replace_with.startswith('[')) and (
+                ' &' in self.type.c_name_with_marker):
             replace_from = ' &'
         else:
             replace_from = '&'
         return self.type.c_name_with_marker.replace(replace_from, replace_with)
 
     def inspect_qualtype(self, block, inspect):
-        return self.type.inspect_type(block, inspect, self.quals)
+        t1, qualexpr = self.type.inspect_type(block, inspect, self.quals)
+        if self.quals & CRX_VOLATILE:
+            qualexpr.append('_CRX_VOLATILE')
+        if self.quals & CRX_CONST:
+            qualexpr.append('_CRX_CONST')
+        if len(qualexpr) == 0:
+            q1 = '0'
+        elif len(qualexpr) == 1:
+            q1 = qualexpr[0]
+        else:
+            q1 = '(%s)' % (' | '.join(qualexpr),)
+        return t1, q1
 
     def as_func_arg(self):
         if isinstance(self.type, ArrayType):
@@ -80,7 +92,7 @@
 
     def inspect_type(self, block, inspect, qualifiers):
         inspect.flush()
-        return block.write_crx_type_var('cb->get_void_type(cb)')
+        return block.write_crx_type_var('cb->get_void_type(cb)'), []
 
 void_type = VoidType()
 
@@ -170,10 +182,12 @@
                 raise AssertionError
         else:
             if isinstance(inspect, VarInspector):
-                decl = self.get_c_name("*p1")
+                qualtype = QualType(self, qualifiers)
+                decl = qualtype.get_c_name("*p1")
                 block.writeline("%s = &%s;  /* check that '%s' is of type '%s'"
                                 " */" % (decl, inspect.varname,
-                                         inspect.varname, self.get_c_name()))
+                                         inspect.varname,
+                                         qualtype.get_c_name()))
                 block.writeline("(void)p1;")
             if self.is_signed_type():
                 expr = 'cb->get_signed_type(cb, sizeof(%s), "%s")' % (
@@ -188,7 +202,7 @@
                     self.name, self.name)
             else:
                 raise AssertionError
-        return block.write_crx_type_var(expr)
+        return block.write_crx_type_var(expr), []
 
     def write_num_const_decl(self, block, varname):
         block.writedecl("_crx_num_const_t v;")
@@ -240,12 +254,12 @@
 
     def _get_arg_ret_types(self, block):
         inspect = MissingInspector()
-        t1 = self.result.inspect_type(block, inspect, 0)
+        t1, q1_ignored = self.result.inspect_type(block, inspect, 0)
         a2 = block.write_crx_qualtype_var(len(self.args))
         for i, arg in enumerate(self.args):
-            t3 = arg.inspect_qualtype(block, inspect)
+            t3, q3 = arg.inspect_qualtype(block, inspect)
             block.writeline('%s[%d].type = %s;' % (a2, i, t3))
-            block.writeline('%s[%d].qualifiers = %d;' % (a2, i, arg.quals))
+            block.writeline('%s[%d].qualifiers = %s;' % (a2, i, q3))
         return t1, a2
 
     def _get_c_call_sequence(self, varname):
@@ -296,7 +310,7 @@
         t1, a2 = self._get_arg_ret_types(block)
         expr = 'cb->get_function_type(cb, %s, %s, %d, &%s)' % (
             t1, a2, len(self.args), crx_func_name)
-        return block.write_crx_type_var(expr)
+        return block.write_crx_type_var(expr), []
 
     def inspect_type_ellipsis(self, block, inspect):
         if isinstance(inspect, VarInspector):
@@ -308,7 +322,7 @@
         t1, a2 = self._get_arg_ret_types(block)
         expr = 'cb->get_ellipsis_function_type(cb, %s, %s, %d)' % (
             t1, a2, len(self.args))
-        return block.write_crx_type_var(expr)
+        return block.write_crx_type_var(expr), []
 
     def shadow_global_var(self, top_level_block, fnname):
         if self.ellipsis:
@@ -378,7 +392,7 @@
                 inspect.after_star_p1_assignment.append(after)
             inspect.levels.append('*')
         elif isinstance(inspect, VarInspector):
-            if isinstance(self.totype, VoidType):
+            if isinstance(self.toqualtype.type, VoidType):
                 block.writeline("void *p1 = %s;  /* check that '%s' is a"
                                 " pointer */" % (inspect.varname,
                                                  inspect.varname))
@@ -387,15 +401,17 @@
                                 " constant */" % (inspect.varname,
                                                   inspect.varname))
             else:
-                decl = self.get_c_name("*p1")
+                qualtype = QualType(self, qualifiers)
+                decl = qualtype.get_c_name("*p1")
                 block.writeline("%s = &%s;  /* check that '%s' is of type '%s'"
                                 " */" % (decl, inspect.varname,
-                                         inspect.varname, self.get_c_name()))
+                                         inspect.varname,
+                                         qualtype.get_c_name()))
                 block.writeline("(void)p1;")
             inspect = MissingInspector()
-        t1 = self.toqualtype.inspect_qualtype(block, inspect)
-        return block.write_crx_type_var('cb->get_pointer_type(cb, %s, %d)'
-                                        % (t1, self.toqualtype.quals))
+        t1, q1 = self.toqualtype.inspect_qualtype(block, inspect)
+        return block.write_crx_type_var('cb->get_pointer_type(cb, %s, %s)'
+                                        % (t1, q1)), []
 
 
 class ArrayType(BaseType):
@@ -429,22 +445,24 @@
             inspect.levels.append('[]')
             inspect.after_star_p1_assignment.append(after)
         elif isinstance(inspect, VarInspector):
-            declitem = self.item.get_c_name("(*p1)")
+            qualtype = QualType(self.item, qualifiers)
+            declitem = qualtype.get_c_name("(*p1)")
             block.writeline("%s[] = &%s;  /* check that '%s' is of type '%s'"
                             " */" % (declitem, inspect.varname,
-                                     inspect.varname, self.get_c_name()))
+                                     inspect.varname,
+                                     qualtype.get_c_name('[]')))
             block.writeline("(void)p1;")
             star_p1 = inspect.varname
             inspect = MissingInspector()
         else:
             star_p1 = None
-        t1 = self.item.inspect_type(block, inspect, qualifiers)
+        t1, q1 = self.item.inspect_type(block, inspect, qualifiers)
         if star_p1 is not None:
             expr = 'cb->get_array_type(cb, %s, sizeof(%s) / sizeof(*%s))' % (
                 t1, star_p1, star_p1)
         else:
             expr = 'cb->get_incomplete_array_type(cb, %s)' % (t1,)
-        return block.write_crx_type_var(expr)
+        return block.write_crx_type_var(expr), []
 
 
 class StructOrUnionOrEnum(BaseType):
@@ -463,7 +481,7 @@
 
     def inspect_type(self, block, inspect, qualifiers):
         inspect.flush()
-        return self.get_type_var(block)
+        return self.get_type_var(block), []
 
 
 class UserType(BaseType):
@@ -476,13 +494,15 @@
     def inspect_type(self, block, inspect, qualifiers):
         inspect.flush()
         if isinstance(inspect, VarInspector):
-            decl = self.get_c_name("*p1")
+            qualtype = QualType(self, qualifiers)
+            decl = qualtype.get_c_name("*p1")
             block.writeline("%s = &%s;  /* check that '%s' is of type '%s'"
                             " */" % (decl, inspect.varname,
-                                     inspect.varname, self.get_c_name()))
+                                     inspect.varname, qualtype.get_c_name()))
             block.writeline("(void)p1;")
-        return block.write_crx_type_var('cb->get_user_type(cb, "%s")' % (
-            self.name,))
+        qt1 = block.write_crx_type_var('cb->get_user_type(cb, "%s")' % (
+            self.name,), is_qualtype=True)
+        return '%s.type' % qt1, ['%s.qualifiers' % qt1]
 
 
 class UnknownType(BaseType):
@@ -495,7 +515,7 @@
     def inspect_type(self, block, inspect, qualifiers):
         inspect.flush()
         return block.write_crx_type_var('cb->get_unknown_type(cb, "%s")' % (
-            self.approx_name,))
+            self.approx_name,)), []
 
 
 # ____________________________________________________________
@@ -680,11 +700,12 @@
                 o_decl = ("size_t o = ((char *)&((%s)0)->%s)"
                           " - (char *)0;%s" % (ptrtp, fldname, comment))
             #
-            t2 = fldtype.inspect_type(block, inspect)
+            t2, q2 = fldtype.inspect_type(block, inspect)
             inspect.stop()
             block.writedecl(o_decl)
             block.writeline('%s[%d].name = "%s";' % (d1, i, fldname))
             block.writeline('%s[%d].type = %s;'   % (d1, i, t2))
+            block.writeline('%s[%d].qualifiers = %s;'   % (d1, i, q2))
             block.writeline('%s[%d].offset = o;'  % (d1, i))
             block.writeline('%s[%d].numbits = -1;' % (d1, i))
             block.writeline('%s[%d].bitshift = -1;' % (d1, i))
@@ -703,10 +724,10 @@
     def write_declaration(self, funcblock):
         block = CodeBlock(funcblock)
         inspect = TypeInspector(block, self.name)
-        t1 = self.qualtype.inspect_qualtype(block, inspect)
+        t1, q1 = self.qualtype.inspect_qualtype(block, inspect)
         inspect.stop()
-        block.writeline('cb->define_type(cb, "%s", %s, %d);' % (
-            self.name, t1, self.qualtype.quals))
+        block.writeline('cb->define_type(cb, "%s", %s, %s);' % (
+            self.name, t1, q1))
         funcblock.write_subblock(block)
 
 
@@ -718,11 +739,11 @@
     def write_declaration(self, funcblock):
         block = CodeBlock(funcblock)
         inspect = VarInspector(block, self.name)
-        t1 = self.qualtype.inspect_qualtype(block, inspect)
+        t1, q1 = self.qualtype.inspect_qualtype(block, inspect)
         shadow = self.qualtype.type.shadow_global_var(block.crx_top_level,
                                                       self.name)
-        block.writeline('cb->define_var(cb, "%s", %s, %d, &%s);' % (
-            self.name, t1, self.qualtype.quals, shadow))
+        block.writeline('cb->define_var(cb, "%s", %s, %s, &%s);' % (
+            self.name, t1, q1, shadow))
         funcblock.write_subblock(block)
 
 
@@ -738,9 +759,9 @@
 
 class ConstDecl(VarDecl):
     def write_declaration(self, funcblock):
-        if isinstance(self.type, PrimitiveType):
+        if isinstance(self.qualtype.type, PrimitiveType):
             block = CodeBlock(funcblock)
-            self.type.write_num_const_decl(block, self.name)
+            self.qualtype.type.write_num_const_decl(block, self.name)
             funcblock.write_subblock(block)
         else:
             VarDecl.write_declaration(self, funcblock)
diff --git a/creflect/test/codegen/002c.c b/creflect/test/codegen/002c.c
--- a/creflect/test/codegen/002c.c
+++ b/creflect/test/codegen/002c.c
@@ -11,7 +11,7 @@
         p1 = (void *)&p2;
         *p1 = (void *)0;    /* check that 'num_t' is a pointer type */
         t1 = cb->get_void_type(cb);
-        t2 = cb->get_pointer_type(cb, t1, 1);
+        t2 = cb->get_pointer_type(cb, t1, _CRX_CONST);
         cb->define_type(cb, "num_t", t2, 0);
 #expect TYPEDEF num_t = PTR CONST void
     }
diff --git a/creflect/test/codegen/002d.c b/creflect/test/codegen/002d.c
--- a/creflect/test/codegen/002d.c
+++ b/creflect/test/codegen/002d.c
@@ -16,7 +16,7 @@
         }
         t1 = cb->get_void_type(cb);
         t2 = cb->get_pointer_type(cb, t1, 0);
-        cb->define_type(cb, "num_t", t2, 1);
+        cb->define_type(cb, "num_t", t2, _CRX_CONST);
 #expect TYPEDEF num_t = CONST PTR void
     }
 }
diff --git a/creflect/test/codegen/002e.c b/creflect/test/codegen/002e.c
new file mode 100644
--- /dev/null
+++ b/creflect/test/codegen/002e.c
@@ -0,0 +1,18 @@
+typedef volatile const void *num_t;
+
+# ____________________________________________________________
+
+void test002e(_crx_builder_t *cb)
+{
+    _crx_type_t *t1, *t2;
+    {
+        num_t *p1;
+        char *p2;
+        p1 = (void *)&p2;
+        *p1 = (void *)0;    /* check that 'num_t' is a pointer type */
+        t1 = cb->get_void_type(cb);
+        t2 = cb->get_pointer_type(cb, t1, (_CRX_VOLATILE | _CRX_CONST));
+        cb->define_type(cb, "num_t", t2, 0);
+#expect TYPEDEF num_t = PTR VOLATILE CONST void
+    }
+}
diff --git a/creflect/test/codegen/006.c b/creflect/test/codegen/006.c
--- a/creflect/test/codegen/006.c
+++ b/creflect/test/codegen/006.c
@@ -12,7 +12,7 @@
         p1 = (void *)b;
         (void)(*p1 << 1);  /* check that 'num_t' is an integer type */
         t1 = _CRX_INT_TYPE(cb, *p1, "int");
-        cb->define_type(cb, "num_t", t1, 1);
+        cb->define_type(cb, "num_t", t1, _CRX_CONST);
 #expect TYPEDEF num_t = CONST unsigned int
     }
 }
diff --git a/creflect/test/codegen/006b.c b/creflect/test/codegen/006b.c
--- a/creflect/test/codegen/006b.c
+++ b/creflect/test/codegen/006b.c
@@ -19,7 +19,7 @@
         **p1 = -1;  /* check that '*num_t' is not declared 'const' */
         t1 = _CRX_INT_TYPE(cb, **p1, "int");
         t2 = cb->get_pointer_type(cb, t1, 0);
-        cb->define_type(cb, "num_t", t2, 1);
+        cb->define_type(cb, "num_t", t2, _CRX_CONST);
 #expect TYPEDEF num_t = CONST PTR int
     }
 }
diff --git a/creflect/test/codegen/006c.c b/creflect/test/codegen/006c.c
--- a/creflect/test/codegen/006c.c
+++ b/creflect/test/codegen/006c.c
@@ -24,7 +24,7 @@
         t1 = _CRX_INT_TYPE(cb, ***p1, "int");
         t2 = cb->get_array_type(cb, t1, sizeof(**p1) / sizeof(***p1));
         t3 = cb->get_pointer_type(cb, t2, 0);
-        cb->define_type(cb, "foo_t", t3, 1);
+        cb->define_type(cb, "foo_t", t3, _CRX_CONST);
 #expect TYPEDEF foo_t = CONST PTR ARRAY[27] int
     }
 }
diff --git a/creflect/test/codegen/func-005.c b/creflect/test/codegen/func-005.c
--- a/creflect/test/codegen/func-005.c
+++ b/creflect/test/codegen/func-005.c
@@ -29,7 +29,7 @@
         t2 = cb->get_signed_type(cb, sizeof(int), "int");
         t3 = cb->get_incomplete_array_type(cb, t2);
         a1[0].type = t3;
-        a1[0].qualifiers = 1;
+        a1[0].qualifiers = _CRX_CONST;
         t4 = cb->get_signed_type(cb, sizeof(long), "long");
         t5 = cb->get_char_type(cb);
         a2[0].type = t5;
diff --git a/creflect/test/codegen/glob-004.c b/creflect/test/codegen/glob-004.c
--- a/creflect/test/codegen/glob-004.c
+++ b/creflect/test/codegen/glob-004.c
@@ -9,17 +9,20 @@
 
 void testglob_004(_crx_builder_t *cb)
 {
-    _crx_type_t *t1, *t2, *a3[2], *t4, *t5;
+    _crx_type_t *t1, *t2, *t3, *t4;
+    _crx_qual_type a1[2];
     {
         int (**p1)(long, long) = &someglob;  /* check that 'someglob' is of type 'int (*)(long, long)' */
         (void)p1;
         t1 = cb->get_signed_type(cb, sizeof(int), "int");
         t2 = cb->get_signed_type(cb, sizeof(long), "long");
-        a3[0] = t2;
-        a3[1] = t2;
-        t4 = cb->get_function_type(cb, t1, a3, 2, &testglob_004__f1);
-        t5 = cb->get_pointer_type(cb, t4, 0);
-        cb->define_var(cb, "someglob", t5, 0, &someglob);
+        a1[0].type = t2;
+        a1[0].qualifiers = 0;
+        a1[1].type = t2;
+        a1[1].qualifiers = 0;
+        t3 = cb->get_function_type(cb, t1, a1, 2, &testglob_004__f1);
+        t4 = cb->get_pointer_type(cb, t3, 0);
+        cb->define_var(cb, "someglob", t4, 0, &someglob);
 #expect VAR someglob: PTR FUNC( long -> long -> int )
     }
 }
diff --git a/creflect/test/codegen/glob-005.c b/creflect/test/codegen/glob-005.c
--- a/creflect/test/codegen/glob-005.c
+++ b/creflect/test/codegen/glob-005.c
@@ -6,7 +6,8 @@
 
 void testglob_005(_crx_builder_t *cb)
 {
-    _crx_type_t *t1, *t2, *t3;
+    _crx_type_t *t1, *t2;
+    _crx_qual_type q1;
     {
         mytype_t *p1;
         char *p2;
@@ -19,10 +20,9 @@
     }
     {
         mytype_t *p1 = &foobar;  /* check that 'foobar' is of type 'mytype_t' */
-        int q1 = 0;
         (void)p1;
-        t3 = cb->get_user_type(cb, "mytype_t", &q1);
-        cb->define_var(cb, "foobar", t3, q1, &foobar);
+        q1 = cb->get_user_type(cb, "mytype_t");
+        cb->define_var(cb, "foobar", q1.type, q1.qualifiers, &foobar);
 #expect VAR foobar: mytype_t
     }
 }
diff --git a/creflect/test/codegen/glob-005b.c b/creflect/test/codegen/glob-005b.c
new file mode 100644
--- /dev/null
+++ b/creflect/test/codegen/glob-005b.c
@@ -0,0 +1,28 @@
+typedef void *mytype_t;
+
+const mytype_t foobar;
+
+# ____________________________________________________________
+
+void testglob_005b(_crx_builder_t *cb)
+{
+    _crx_type_t *t1, *t2;
+    _crx_qual_type q1;
+    {
+        mytype_t *p1;
+        char *p2;
+        p1 = (void *)&p2;
+        *p1 = (void *)0;    /* check that 'mytype_t' is a pointer type */
+        t1 = cb->get_void_type(cb);
+        t2 = cb->get_pointer_type(cb, t1, 0);
+        cb->define_type(cb, "mytype_t", t2, 0);
+#expect TYPEDEF mytype_t = PTR void
+    }
+    {
+        mytype_t const *p1 = &foobar;  /* check that 'foobar' is of type 'mytype_t const' */
+        (void)p1;
+        q1 = cb->get_user_type(cb, "mytype_t");
+        cb->define_var(cb, "foobar", q1.type, (q1.qualifiers | _CRX_CONST), &foobar);
+#expect VAR foobar: CONST mytype_t
+    }
+}

From noreply at buildbot.pypy.org  Sun Nov 30 23:12:42 2014
From: noreply at buildbot.pypy.org (pjenvey)
Date: Sun, 30 Nov 2014 23:12:42 +0100 (CET)
Subject: [pypy-commit] pypy py3k: merge default
Message-ID: <20141130221242.DAE941D24AA@cobra.cs.uni-duesseldorf.de>

Author: Philip Jenvey 
Branch: py3k
Changeset: r74767:e01964ac6c50
Date: 2014-11-30 11:51 -0800
http://bitbucket.org/pypy/pypy/changeset/e01964ac6c50/

Log:	merge default

diff too long, truncating to 2000 out of 3374 lines

diff --git a/lib-python/2.7/subprocess.py b/lib-python/2.7/subprocess.py
--- a/lib-python/2.7/subprocess.py
+++ b/lib-python/2.7/subprocess.py
@@ -655,6 +655,21 @@
         """Create new Popen instance."""
         _cleanup()
 
+        # --- PyPy hack, see _pypy_install_libs_after_virtualenv() ---
+        # match arguments passed by different versions of virtualenv
+        if args[1:] in (
+            ['-c', 'import sys; print(sys.prefix)'],        # 1.6 10ba3f3c
+            ['-c', "\nimport sys\nprefix = sys.prefix\n"    # 1.7 0e9342ce
+             "if sys.version_info[0] == 3:\n"
+             "    prefix = prefix.encode('utf8')\n"
+             "if hasattr(sys.stdout, 'detach'):\n"
+             "    sys.stdout = sys.stdout.detach()\n"
+             "elif hasattr(sys.stdout, 'buffer'):\n"
+             "    sys.stdout = sys.stdout.buffer\nsys.stdout.write(prefix)\n"],
+            ['-c', 'import sys;out=sys.stdout;getattr(out, "buffer"'
+             ', out).write(sys.prefix.encode("utf-8"))']):  # 1.7.2 a9454bce
+            _pypy_install_libs_after_virtualenv(args[0])
+
         if not isinstance(bufsize, (int, long)):
             raise TypeError("bufsize must be an integer")
 
@@ -1560,6 +1575,27 @@
             self.send_signal(signal.SIGKILL)
 
 
+def _pypy_install_libs_after_virtualenv(target_executable):
+    # https://bitbucket.org/pypy/pypy/issue/1922/future-proofing-virtualenv
+    #
+    # PyPy 2.4.1 turned --shared on by default.  This means the pypy binary
+    # depends on the 'libpypy-c.so' shared library to be able to run.
+    # The virtualenv code existing at the time did not account for this
+    # and would break.  Try to detect that we're running under such a
+    # virtualenv in the "Testing executable with" phase and copy the
+    # library ourselves.
+    caller = sys._getframe(2)
+    if ('virtualenv_version' in caller.f_globals and
+                  'copyfile' in caller.f_globals):
+        dest_dir = sys.pypy_resolvedirof(target_executable)
+        src_dir = sys.pypy_resolvedirof(sys.executable)
+        for libname in ['libpypy-c.so']:
+            dest_library = os.path.join(dest_dir, libname)
+            src_library = os.path.join(src_dir, libname)
+            if os.path.exists(src_library):
+                caller.f_globals['copyfile'](src_library, dest_library)
+
+
 def _demo_posix():
     #
     # Example 1: Simple redirection: Get process list
diff --git a/pypy/doc/cpython_differences.rst b/pypy/doc/cpython_differences.rst
--- a/pypy/doc/cpython_differences.rst
+++ b/pypy/doc/cpython_differences.rst
@@ -205,23 +205,28 @@
 The above is true both in CPython and in PyPy.  Differences
 can occur about whether a built-in function or method will
 call an overridden method of *another* object than ``self``.
-In PyPy, they are generally always called, whereas not in
-CPython.  For example, in PyPy, ``dict1.update(dict2)``
-considers that ``dict2`` is just a general mapping object, and
-will thus call overridden ``keys()``  and ``__getitem__()``
-methods on it.  So the following code prints ``42`` on PyPy
-but ``foo`` on CPython::
+In PyPy, they are often called in cases where CPython would not.
+Two examples::
 
-    >>>> class D(dict):
-    ....     def __getitem__(self, key):
-    ....         return 42
-    ....
-    >>>>
-    >>>> d1 = {}
-    >>>> d2 = D(a='foo')
-    >>>> d1.update(d2)
-    >>>> print d1['a']
-    42
+    class D(dict):
+        def __getitem__(self, key):
+            return "%r from D" % (key,)
+
+    class A(object):
+        pass
+
+    a = A()
+    a.__dict__ = D()
+    a.foo = "a's own foo"
+    print a.foo
+    # CPython => a's own foo
+    # PyPy => 'foo' from D
+
+    glob = D(foo="base item")
+    loc = {}
+    exec "print foo" in glob, loc
+    # CPython => base item
+    # PyPy => 'foo' from D
 
 
 Mutating classes of objects which are already used as dictionary keys
@@ -292,6 +297,9 @@
 above types will return a value that is computed from the argument, and can
 thus be larger than ``sys.maxint`` (i.e. it can be an arbitrary long).
 
+Notably missing from the list above are ``str`` and ``unicode``.  If your
+code relies on comparing strings with ``is``, then it might break in PyPy.
+
 
 Miscellaneous
 -------------
diff --git a/pypy/interpreter/astcompiler/assemble.py b/pypy/interpreter/astcompiler/assemble.py
--- a/pypy/interpreter/astcompiler/assemble.py
+++ b/pypy/interpreter/astcompiler/assemble.py
@@ -1,15 +1,13 @@
-"""
-Python control flow graph generation and bytecode assembly.
-"""
+"""Python control flow graph generation and bytecode assembly."""
 
-from pypy.interpreter.astcompiler import ast, consts, symtable
-from pypy.interpreter import pycode
+from rpython.rlib import rfloat
+from rpython.rlib.objectmodel import specialize, we_are_translated
+
+from pypy.interpreter.astcompiler import ast, consts, misc, symtable
+from pypy.interpreter.error import OperationError
+from pypy.interpreter.pycode import PyCode
 from pypy.tool import stdlib_opcode as ops
 
-from pypy.interpreter.error import OperationError
-from rpython.rlib.objectmodel import specialize, we_are_translated
-from rpython.rlib import rfloat
-
 
 class Instruction(object):
     """Represents a single opcode."""
@@ -21,14 +19,12 @@
         self.has_jump = False
 
     def size(self):
-        """Return the size of bytes of this instruction when it is encoded."""
+        """Return the size of bytes of this instruction when it is
+        encoded.
+        """
         if self.opcode >= ops.HAVE_ARGUMENT:
-            if self.arg > 0xFFFF:
-                return 6
-            else:
-                return 3
-        else:
-            return 1
+            return (6 if self.arg > 0xFFFF else 3)
+        return 1
 
     def jump_to(self, target, absolute=False):
         """Indicate the target this jump instruction.
@@ -54,9 +50,9 @@
 class Block(object):
     """A basic control flow block.
 
-    It has one entry point and several possible exit points.  Its instructions
-    may be jumps to other blocks, or if control flow reaches the end of the
-    block, it continues to next_block.
+    It has one entry point and several possible exit points.  Its
+    instructions may be jumps to other blocks, or if control flow
+    reaches the end of the block, it continues to next_block.
     """
 
     def __init__(self):
@@ -71,10 +67,10 @@
             stack.append(nextblock)
 
     def post_order(self):
-        """Return this block and its children in post order.
-        This means that the graph of blocks is first cleaned up to
-        ignore back-edges, thus turning it into a DAG.  Then the DAG
-        is linearized.  For example:
+        """Return this block and its children in post order.  This means
+        that the graph of blocks is first cleaned up to ignore
+        back-edges, thus turning it into a DAG.  Then the DAG is
+        linearized.  For example:
 
                    A --> B -\           =>     [A, D, B, C]
                      \-> D ---> C
@@ -105,7 +101,9 @@
         return resultblocks
 
     def code_size(self):
-        """Return the encoded size of all the instructions in this block."""
+        """Return the encoded size of all the instructions in this
+        block.
+        """
         i = 0
         for instr in self.instructions:
             i += instr.size()
@@ -141,6 +139,7 @@
             i += 1
     return result
 
+
 @specialize.argtype(0)
 def _iter_to_dict(iterable, offset=0):
     result = {}
@@ -302,11 +301,11 @@
     def _resolve_block_targets(self, blocks):
         """Compute the arguments of jump instructions."""
         last_extended_arg_count = 0
-        # The reason for this loop is extended jumps.  EXTENDED_ARG extends the
-        # bytecode size, so it might invalidate the offsets we've already given.
-        # Thus we have to loop until the number of extended args is stable.  Any
-        # extended jump at all is extremely rare, so performance is not too
-        # concerning.
+        # The reason for this loop is extended jumps.  EXTENDED_ARG
+        # extends the bytecode size, so it might invalidate the offsets
+        # we've already given.  Thus we have to loop until the number of
+        # extended args is stable.  Any extended jump at all is
+        # extremely rare, so performance is not too concerning.
         while True:
             extended_arg_count = 0
             offset = 0
@@ -332,7 +331,8 @@
                                     instr.opcode = ops.JUMP_ABSOLUTE
                                     absolute = True
                                 elif target_op == ops.RETURN_VALUE:
-                                    # Replace JUMP_* to a RETURN into just a RETURN
+                                    # Replace JUMP_* to a RETURN into
+                                    # just a RETURN
                                     instr.opcode = ops.RETURN_VALUE
                                     instr.arg = 0
                                     instr.has_jump = False
@@ -347,7 +347,8 @@
                         instr.arg = jump_arg
                         if jump_arg > 0xFFFF:
                             extended_arg_count += 1
-            if extended_arg_count == last_extended_arg_count and not force_redo:
+            if (extended_arg_count == last_extended_arg_count and
+                not force_redo):
                 break
             else:
                 last_extended_arg_count = extended_arg_count
@@ -362,12 +363,14 @@
         while True:
             try:
                 w_key = space.next(w_iter)
-            except OperationError, e:
+            except OperationError as e:
                 if not e.match(space, space.w_StopIteration):
                     raise
                 break
             w_index = space.getitem(w_consts, w_key)
-            consts_w[space.int_w(w_index)] = space.getitem(w_key, first)
+            w_constant = space.getitem(w_key, first)
+            w_constant = misc.intern_if_common_string(space, w_constant)
+            consts_w[space.int_w(w_index)] = w_constant
         return consts_w
 
     def _get_code_flags(self):
@@ -435,15 +438,16 @@
                         continue
                     addr = offset - current_off
                     # Python assumes that lineno always increases with
-                    # increasing bytecode address (lnotab is unsigned char).
-                    # Depending on when SET_LINENO instructions are emitted this
-                    # is not always true.  Consider the code:
+                    # increasing bytecode address (lnotab is unsigned
+                    # char).  Depending on when SET_LINENO instructions
+                    # are emitted this is not always true.  Consider the
+                    # code:
                     #     a = (1,
                     #          b)
-                    # In the bytecode stream, the assignment to "a" occurs after
-                    # the loading of "b".  This works with the C Python compiler
-                    # because it only generates a SET_LINENO instruction for the
-                    # assignment.
+                    # In the bytecode stream, the assignment to "a"
+                    # occurs after the loading of "b".  This works with
+                    # the C Python compiler because it only generates a
+                    # SET_LINENO instruction for the assignment.
                     if line or addr:
                         while addr > 255:
                             push(chr(255))
@@ -488,23 +492,23 @@
         # (Only) inherit compilerflags in PyCF_MASK
         flags |= (self.compile_info.flags & consts.PyCF_MASK)
         bytecode = ''.join([block.get_code() for block in blocks])
-        return pycode.PyCode(self.space,
-                             self.argcount,
-                             self.kwonlyargcount,
-                             len(self.var_names),
-                             stack_depth,
-                             flags,
-                             bytecode,
-                             list(consts_w),
-                             names,
-                             var_names,
-                             self.compile_info.filename,
-                             self.name,
-                             self.first_lineno,
-                             lnotab,
-                             free_names,
-                             cell_names,
-                             self.compile_info.hidden_applevel)
+        return PyCode(self.space,
+                      self.argcount,
+                      self.kwonlyargcount,
+                      len(self.var_names),
+                      stack_depth,
+                      flags,
+                      bytecode,
+                      list(consts_w),
+                      names,
+                      var_names,
+                      self.compile_info.filename,
+                      self.name,
+                      self.first_lineno,
+                      lnotab,
+                      free_names,
+                      cell_names,
+                      self.compile_info.hidden_applevel)
 
 
 def _list_from_dict(d, offset=0):
@@ -515,116 +519,116 @@
 
 
 _static_opcode_stack_effects = {
-    ops.NOP : 0,
-    ops.STOP_CODE : 0,
+    ops.NOP: 0,
+    ops.STOP_CODE: 0,
 
-    ops.POP_TOP : -1,
-    ops.ROT_TWO : 0,
-    ops.ROT_THREE : 0,
-    ops.DUP_TOP : 1,
-    ops.DUP_TOP_TWO : 2,
+    ops.POP_TOP: -1,
+    ops.ROT_TWO: 0,
+    ops.ROT_THREE: 0,
+    ops.DUP_TOP: 1,
+    ops.DUP_TOP_TWO: 2,
 
-    ops.UNARY_POSITIVE : 0,
-    ops.UNARY_NEGATIVE : 0,
-    ops.UNARY_NOT : 0,
-    ops.UNARY_INVERT : 0,
+    ops.UNARY_POSITIVE: 0,
+    ops.UNARY_NEGATIVE: 0,
+    ops.UNARY_NOT: 0,
+    ops.UNARY_INVERT: 0,
 
-    ops.LIST_APPEND : -1,
-    ops.SET_ADD : -1,
-    ops.MAP_ADD : -2,
-    ops.STORE_MAP : -2,
+    ops.LIST_APPEND: -1,
+    ops.SET_ADD: -1,
+    ops.MAP_ADD: -2,
+    ops.STORE_MAP: -2,
 
-    ops.BINARY_POWER : -1,
-    ops.BINARY_MULTIPLY : -1,
-    ops.BINARY_MODULO : -1,
-    ops.BINARY_ADD : -1,
-    ops.BINARY_SUBTRACT : -1,
-    ops.BINARY_SUBSCR : -1,
-    ops.BINARY_FLOOR_DIVIDE : -1,
-    ops.BINARY_TRUE_DIVIDE : -1,
-    ops.BINARY_LSHIFT : -1,
-    ops.BINARY_RSHIFT : -1,
-    ops.BINARY_AND : -1,
-    ops.BINARY_OR : -1,
-    ops.BINARY_XOR : -1,
+    ops.BINARY_POWER: -1,
+    ops.BINARY_MULTIPLY: -1,
+    ops.BINARY_MODULO: -1,
+    ops.BINARY_ADD: -1,
+    ops.BINARY_SUBTRACT: -1,
+    ops.BINARY_SUBSCR: -1,
+    ops.BINARY_FLOOR_DIVIDE: -1,
+    ops.BINARY_TRUE_DIVIDE: -1,
+    ops.BINARY_LSHIFT: -1,
+    ops.BINARY_RSHIFT: -1,
+    ops.BINARY_AND: -1,
+    ops.BINARY_OR: -1,
+    ops.BINARY_XOR: -1,
 
-    ops.INPLACE_FLOOR_DIVIDE : -1,
-    ops.INPLACE_TRUE_DIVIDE : -1,
-    ops.INPLACE_ADD : -1,
-    ops.INPLACE_SUBTRACT : -1,
-    ops.INPLACE_MULTIPLY : -1,
-    ops.INPLACE_MODULO : -1,
-    ops.INPLACE_POWER : -1,
-    ops.INPLACE_LSHIFT : -1,
-    ops.INPLACE_RSHIFT : -1,
-    ops.INPLACE_AND : -1,
-    ops.INPLACE_OR : -1,
-    ops.INPLACE_XOR : -1,
+    ops.INPLACE_FLOOR_DIVIDE: -1,
+    ops.INPLACE_TRUE_DIVIDE: -1,
+    ops.INPLACE_ADD: -1,
+    ops.INPLACE_SUBTRACT: -1,
+    ops.INPLACE_MULTIPLY: -1,
+    ops.INPLACE_MODULO: -1,
+    ops.INPLACE_POWER: -1,
+    ops.INPLACE_LSHIFT: -1,
+    ops.INPLACE_RSHIFT: -1,
+    ops.INPLACE_AND: -1,
+    ops.INPLACE_OR: -1,
+    ops.INPLACE_XOR: -1,
 
-    ops.STORE_SUBSCR : -2,
-    ops.DELETE_SUBSCR : -2,
+    ops.STORE_SUBSCR: -2,
+    ops.DELETE_SUBSCR: -2,
 
-    ops.GET_ITER : 0,
-    ops.FOR_ITER : 1,
-    ops.BREAK_LOOP : 0,
-    ops.CONTINUE_LOOP : 0,
-    ops.SETUP_LOOP : 0,
+    ops.GET_ITER: 0,
+    ops.FOR_ITER: 1,
+    ops.BREAK_LOOP: 0,
+    ops.CONTINUE_LOOP: 0,
+    ops.SETUP_LOOP: 0,
 
-    ops.PRINT_EXPR : -1,
+    ops.PRINT_EXPR: -1,
 
-    ops.WITH_CLEANUP : -1,
-    ops.LOAD_BUILD_CLASS : 1,
-    ops.STORE_LOCALS : -1,
-    ops.POP_BLOCK : 0,
-    ops.POP_EXCEPT : 0,
-    ops.END_FINALLY : -1,
-    ops.SETUP_WITH : 1,
-    ops.SETUP_FINALLY : 0,
-    ops.SETUP_EXCEPT : 4,
+    ops.WITH_CLEANUP: -1,
+    ops.LOAD_BUILD_CLASS: 1,
+    ops.STORE_LOCALS: -1,
+    ops.POP_BLOCK: 0,
+    ops.POP_EXCEPT: 0,
+    ops.END_FINALLY: -1,
+    ops.SETUP_WITH: 1,
+    ops.SETUP_FINALLY: 0,
+    ops.SETUP_EXCEPT: 4,
 
-    ops.RETURN_VALUE : -1,
-    ops.YIELD_VALUE : 0,
-    ops.BUILD_MAP : 1,
-    ops.BUILD_SET : 1,
-    ops.COMPARE_OP : -1,
+    ops.RETURN_VALUE: -1,
+    ops.YIELD_VALUE: 0,
+    ops.BUILD_MAP: 1,
+    ops.BUILD_SET: 1,
+    ops.COMPARE_OP: -1,
 
-    ops.LOOKUP_METHOD : 1,
+    ops.LOOKUP_METHOD: 1,
 
-    ops.LOAD_NAME : 1,
-    ops.STORE_NAME : -1,
-    ops.DELETE_NAME : 0,
+    ops.LOAD_NAME: 1,
+    ops.STORE_NAME: -1,
+    ops.DELETE_NAME: 0,
 
-    ops.LOAD_FAST : 1,
-    ops.STORE_FAST : -1,
-    ops.DELETE_FAST : 0,
+    ops.LOAD_FAST: 1,
+    ops.STORE_FAST: -1,
+    ops.DELETE_FAST: 0,
 
-    ops.LOAD_ATTR : 0,
-    ops.STORE_ATTR : -2,
-    ops.DELETE_ATTR : -1,
+    ops.LOAD_ATTR: 0,
+    ops.STORE_ATTR: -2,
+    ops.DELETE_ATTR: -1,
 
-    ops.LOAD_GLOBAL : 1,
-    ops.STORE_GLOBAL : -1,
-    ops.DELETE_GLOBAL : 0,
-    ops.DELETE_DEREF : 0,
-
-    ops.LOAD_CLOSURE : 1,
-    ops.LOAD_DEREF : 1,
-    ops.STORE_DEREF : -1,
+    ops.LOAD_GLOBAL: 1,
+    ops.STORE_GLOBAL: -1,
+    ops.DELETE_GLOBAL: 0,
     ops.DELETE_DEREF: 0,
 
-    ops.LOAD_CONST : 1,
+    ops.LOAD_CLOSURE: 1,
+    ops.LOAD_DEREF: 1,
+    ops.STORE_DEREF: -1,
+    ops.DELETE_DEREF: 0,
 
-    ops.IMPORT_STAR : -1,
-    ops.IMPORT_NAME : -1,
-    ops.IMPORT_FROM : 1,
+    ops.LOAD_CONST: 1,
 
-    ops.JUMP_FORWARD : 0,
-    ops.JUMP_ABSOLUTE : 0,
-    ops.JUMP_IF_TRUE_OR_POP : 0,
-    ops.JUMP_IF_FALSE_OR_POP : 0,
-    ops.POP_JUMP_IF_TRUE : -1,
-    ops.POP_JUMP_IF_FALSE : -1,
-    ops.JUMP_IF_NOT_DEBUG : 0,
+    ops.IMPORT_STAR: -1,
+    ops.IMPORT_NAME: -1,
+    ops.IMPORT_FROM: 1,
+
+    ops.JUMP_FORWARD: 0,
+    ops.JUMP_ABSOLUTE: 0,
+    ops.JUMP_IF_TRUE_OR_POP: 0,
+    ops.JUMP_IF_FALSE_OR_POP: 0,
+    ops.POP_JUMP_IF_TRUE: -1,
+    ops.POP_JUMP_IF_FALSE: -1,
+    ops.JUMP_IF_NOT_DEBUG: 0,
 
     ops.BUILD_LIST_FROM_ARG: 1,
 }
diff --git a/pypy/interpreter/astcompiler/misc.py b/pypy/interpreter/astcompiler/misc.py
--- a/pypy/interpreter/astcompiler/misc.py
+++ b/pypy/interpreter/astcompiler/misc.py
@@ -108,6 +108,16 @@
     return "_%s%s" % (klass[i:], name)
 
 
+def intern_if_common_string(space, w_const):
+    # only intern identifier-like strings
+    if not space.is_w(space.type(w_const), space.w_str):
+        return w_const
+    for c in space.str_w(w_const):
+        if not (c.isalnum() or c == '_'):
+            return w_const
+    return space.new_interned_w_str(w_const)
+
+
 def new_identifier(space, name):
     # Check whether there are non-ASCII characters in the identifier; if
     # so, normalize to NFKC
diff --git a/pypy/interpreter/astcompiler/optimize.py b/pypy/interpreter/astcompiler/optimize.py
--- a/pypy/interpreter/astcompiler/optimize.py
+++ b/pypy/interpreter/astcompiler/optimize.py
@@ -279,6 +279,11 @@
                 if w_const is None:
                     return tup
                 consts_w[i] = w_const
+            # intern the string constants packed into the tuple here,
+            # because assemble.py will see the result as just a tuple constant
+            for i in range(len(consts_w)):
+                consts_w[i] = misc.intern_if_common_string(
+                    self.space, consts_w[i])
         else:
             consts_w = []
         w_consts = self.space.newtuple(consts_w)
diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -15,7 +15,7 @@
     UserDelAction)
 from pypy.interpreter.error import OperationError, new_exception_class, oefmt
 from pypy.interpreter.argument import Arguments
-from pypy.interpreter.miscutils import ThreadLocals
+from pypy.interpreter.miscutils import ThreadLocals, make_weak_value_dictionary
 
 
 __all__ = ['ObjSpace', 'OperationError', 'W_Root']
@@ -362,7 +362,7 @@
         self.builtin_modules = {}
         self.reloading_modules = {}
 
-        self.interned_strings = {}
+        self.interned_strings = make_weak_value_dictionary(self, str, W_Root)
         self.actionflag = ActionFlag()    # changed by the signal module
         self.check_signal_action = None   # changed by the signal module
         self.user_del_action = UserDelAction(self)
@@ -766,25 +766,30 @@
             return self.w_False
 
     def new_interned_w_str(self, w_s):
+        assert isinstance(w_s, W_Root)   # and is not None
         s = self.str_w(w_s)
         if not we_are_translated():
             assert type(s) is str
-        try:
-            return self.interned_strings[s]
-        except KeyError:
-            pass
-        self.interned_strings[s] = w_s
-        return w_s
+        w_s1 = self.interned_strings.get(s)
+        if w_s1 is None:
+            w_s1 = w_s
+            self.interned_strings.set(s, w_s1)
+        return w_s1
 
     def new_interned_str(self, s):
         if not we_are_translated():
             assert type(s) is str
-        try:
-            return self.interned_strings[s]
-        except KeyError:
-            pass
-        w_s = self.interned_strings[s] = self.wrap(s)
-        return w_s
+        w_s1 = self.interned_strings.get(s)
+        if w_s1 is None:
+            w_s1 = self.wrap(s)
+            self.interned_strings.set(s, w_s1)
+        return w_s1
+
+    def is_interned_str(self, s):
+        # interface for marshal_impl
+        if not we_are_translated():
+            assert type(s) is str
+        return self.interned_strings.get(s) is not None
 
     def descr_self_interp_w(self, RequiredClass, w_obj):
         if not isinstance(w_obj, RequiredClass):
diff --git a/pypy/interpreter/miscutils.py b/pypy/interpreter/miscutils.py
--- a/pypy/interpreter/miscutils.py
+++ b/pypy/interpreter/miscutils.py
@@ -31,3 +31,19 @@
 
     def getallvalues(self):
         return {0: self._value}
+
+
+def make_weak_value_dictionary(space, keytype, valuetype):
+    "NOT_RPYTHON"
+    if space.config.translation.rweakref:
+        from rpython.rlib.rweakref import RWeakValueDictionary
+        return RWeakValueDictionary(keytype, valuetype)
+    else:
+        class FakeWeakValueDict(object):
+            def __init__(self):
+                self._dict = {}
+            def get(self, key):
+                return self._dict.get(key, None)
+            def set(self, key, value):
+                self._dict[key] = value
+        return FakeWeakValueDict()
diff --git a/pypy/interpreter/pyframe.py b/pypy/interpreter/pyframe.py
--- a/pypy/interpreter/pyframe.py
+++ b/pypy/interpreter/pyframe.py
@@ -131,7 +131,6 @@
         # class bodies only have CO_NEWLOCALS.
         # CO_NEWLOCALS: make a locals dict unless optimized is also set
         # CO_OPTIMIZED: no locals dict needed at all
-        # NB: this method is overridden in nestedscope.py
         flags = code.co_flags
         if not (flags & pycode.CO_OPTIMIZED):
             if flags & pycode.CO_NEWLOCALS:
diff --git a/pypy/interpreter/test/test_compiler.py b/pypy/interpreter/test/test_compiler.py
--- a/pypy/interpreter/test/test_compiler.py
+++ b/pypy/interpreter/test/test_compiler.py
@@ -1094,7 +1094,12 @@
             sys.stdout = out
         output = s.getvalue()
         assert "CALL_METHOD" in output
-            
+
+    def test_interned_strings(self):
+        source = """x = ('foo_bar42', 5); y = 'foo_bar42'; z = x[0]"""
+        exec source
+        assert y is z
+
 
 class AppTestExceptions:
     def test_indentation_error(self):
diff --git a/pypy/interpreter/test/test_objspace.py b/pypy/interpreter/test/test_objspace.py
--- a/pypy/interpreter/test/test_objspace.py
+++ b/pypy/interpreter/test/test_objspace.py
@@ -378,3 +378,41 @@
         assert space.str_w(space.getattr(space.sys, w_executable)) == 'foobar'
         space.startup()
         assert space.str_w(space.getattr(space.sys, w_executable)) == 'foobar'
+
+    def test_interned_strings_are_weak(self):
+        import weakref, gc, random
+        space = self.space
+        assert space.config.translation.rweakref
+        w1 = space.new_interned_str("abcdef")
+        w2 = space.new_interned_str("abcdef")
+        assert w2 is w1
+        #
+        # check that 'w1' goes away if we don't hold a reference to it
+        rw1 = weakref.ref(w1)
+        del w1, w2
+        i = 10
+        while rw1() is not None:
+            i -= 1
+            assert i >= 0
+            gc.collect()
+        #
+        s = "foobar%r" % random.random()
+        w0 = space.wrap(s)
+        w1 = space.new_interned_w_str(w0)
+        assert w1 is w0
+        w2 = space.new_interned_w_str(w0)
+        assert w2 is w0
+        w3 = space.wrap(s)
+        assert w3 is not w0
+        w4 = space.new_interned_w_str(w3)
+        assert w4 is w0
+        #
+        # check that 'w0' goes away if we don't hold a reference to it
+        # (even if we hold a reference to 'w3')
+        rw0 = weakref.ref(w0)
+        del w0, w1, w2, w4
+        i = 10
+        while rw0() is not None:
+            i -= 1
+            assert i >= 0
+            gc.collect()
diff --git a/pypy/module/marshal/interp_marshal.py b/pypy/module/marshal/interp_marshal.py
--- a/pypy/module/marshal/interp_marshal.py
+++ b/pypy/module/marshal/interp_marshal.py
@@ -147,7 +147,6 @@
     atom_int(tc, int)           puts code and int
     atom_int64(tc, int64)       puts code and int64
     atom_str(tc, str)           puts code, len and string
-    atom_strlist(tc, strlist)   puts code, len and list of strings
 
     building blocks for compound types:
 
@@ -201,15 +200,6 @@
         self.atom_int(typecode, len(x))
         self.put(x)
 
-    def atom_strlist(self, typecode, tc2, x):
-        self.atom_int(typecode, len(x))
-        atom_str = self.atom_str
-        for item in x:
-            # type(str) seems to be forbidden
-            #if type(item) is not str:
-            #    self.raise_exc('object with wrong type in strlist')
-            atom_str(tc2, item)
-
     def start(self, typecode):
         # type(char) not supported
         self.put(typecode)
@@ -382,16 +372,6 @@
         self.start(typecode)
         return self.get_lng()
 
-    def atom_strlist(self, typecode, tc2):
-        self.start(typecode)
-        lng = self.get_lng()
-        res = [None] * lng
-        idx = 0
-        while idx < lng:
-            res[idx] = self.atom_str(tc2)
-            idx += 1
-        return res
-
     def start(self, typecode):
         tc = self.get1()
         if tc != typecode:
@@ -439,7 +419,6 @@
 
     def get_w_obj(self, allow_null=False):
         space = self.space
-        w_ret = space.w_None # something not None
         tc = self.get1()
         w_ret = self._dispatch[ord(tc)](space, self, tc)
         if w_ret is None and not allow_null:
diff --git a/pypy/module/pypyjit/test_pypy_c/model.py b/pypy/module/pypyjit/test_pypy_c/model.py
--- a/pypy/module/pypyjit/test_pypy_c/model.py
+++ b/pypy/module/pypyjit/test_pypy_c/model.py
@@ -275,9 +275,19 @@
 
     @classmethod
     def parse_op(cls, line):
-        # strip comment
+        # strip comment after '#', but not if it appears inside parentheses
         if '#' in line:
-            line = line[:line.index('#')]
+            nested = 0
+            for i, c in enumerate(line):
+                if c == '(':
+                    nested += 1
+                elif c == ')':
+                    assert nested > 0, "more ')' than '(' in %r" % (line,)
+                    nested -= 1
+                elif c == '#' and nested == 0:
+                    line = line[:i]
+                    break
+        #
         if line.strip() == 'guard_not_invalidated?':
             return 'guard_not_invalidated', None, [], '...', False
         # find the resvar, if any
@@ -314,7 +324,7 @@
         # to repeat it every time
         ticker_check = """
             guard_not_invalidated?
-            ticker0 = getfield_raw(ticker_address, descr=)
+            ticker0 = getfield_raw(#, descr=)
             ticker_cond0 = int_lt(ticker0, 0)
             guard_false(ticker_cond0, descr=...)
         """
@@ -323,9 +333,9 @@
         # this is the ticker check generated if we have threads
         thread_ticker_check = """
             guard_not_invalidated?
-            ticker0 = getfield_raw(ticker_address, descr=)
-            ticker1 = int_sub(ticker0, _)
-            setfield_raw(ticker_address, ticker1, descr=)
+            ticker0 = getfield_raw(#, descr=)
+            ticker1 = int_sub(ticker0, #)
+            setfield_raw(#, ticker1, descr=)
             ticker_cond0 = int_lt(ticker1, 0)
             guard_false(ticker_cond0, descr=...)
         """
@@ -333,7 +343,7 @@
         #
         # this is the ticker check generated in PyFrame.handle_operation_error
         exc_ticker_check = """
-            ticker2 = getfield_raw(ticker_address, descr=)
+            ticker2 = getfield_raw(#, descr=)
             ticker_cond1 = int_lt(ticker2, 0)
             guard_false(ticker_cond1, descr=...)
         """
@@ -351,18 +361,31 @@
 
     @staticmethod
     def as_numeric_const(v1):
+        # returns one of:  ('int', value)  ('float', value)  None
         try:
-            return int(v1)
-        except (ValueError, TypeError):
-            return None
+            return ('int', int(v1))
+        except ValueError:
+            pass
+        if '.' in v1:
+            try:
+                return ('float', float(v1))
+            except ValueError:
+                pass
+        return None
 
     def match_var(self, v1, exp_v2):
         assert v1 != '_'
-        if exp_v2 == '_':
+        if exp_v2 == '_':           # accept anything
             return True
+        if exp_v2 is None:
+            return v1 is None
+        assert exp_v2 != '...'      # bogus use of '...' in the expected code
         n1 = self.as_numeric_const(v1)
+        if exp_v2 == '#':           # accept any (integer or float) number
+            return n1 is not None
         n2 = self.as_numeric_const(exp_v2)
-        if n1 is not None and n2 is not None:
+        if n1 is not None or n2 is not None:
+            # at least one is a number; check that both are, and are equal
             return n1 == n2
         if self.is_const(v1) or self.is_const(exp_v2):
             return v1[:-1].startswith(exp_v2[:-1])
@@ -382,10 +405,13 @@
     def match_op(self, op, (exp_opname, exp_res, exp_args, exp_descr, _)):
         self._assert(op.name == exp_opname, "operation mismatch")
         self.match_var(op.res, exp_res)
-        if exp_args != ['...']:
+        if exp_args[-1:] == ['...']:      # exp_args ends with '...'
+            exp_args = exp_args[:-1]
+            self._assert(len(op.args) >= len(exp_args), "not enough arguments")
+        else:
             self._assert(len(op.args) == len(exp_args), "wrong number of arguments")
-            for arg, exp_arg in zip(op.args, exp_args):
-                self._assert(self.match_var(arg, exp_arg), "variable mismatch: %r instead of %r" % (arg, exp_arg))
+        for arg, exp_arg in zip(op.args, exp_args):
+            self._assert(self.match_var(arg, exp_arg), "variable mismatch: %r instead of %r" % (arg, exp_arg))
         self.match_descr(op.descr, exp_descr)
 
 
diff --git a/pypy/module/pypyjit/test_pypy_c/test_00_model.py b/pypy/module/pypyjit/test_pypy_c/test_00_model.py
--- a/pypy/module/pypyjit/test_pypy_c/test_00_model.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_00_model.py
@@ -158,6 +158,24 @@
         assert match_var('v0', 'V0')
         assert match_var('ConstPtr(ptr0)', '_')
         py.test.raises(AssertionError, "match_var('_', 'v0')")
+        #
+        # numerics
+        assert match_var('1234', '1234')
+        assert not match_var('1234', '1235')
+        assert not match_var('v0', '1234')
+        assert not match_var('1234', 'v0')
+        assert match_var('1234', '#')        # the '#' char matches any number
+        assert not match_var('v0', '#')
+        assert match_var('1234', '_')        # the '_' char matches anything
+        #
+        # float numerics
+        assert match_var('0.000000', '0.0')
+        assert not match_var('0.000000', '0')
+        assert not match_var('0', '0.0')
+        assert not match_var('v0', '0.0')
+        assert not match_var('0.0', 'v0')
+        assert match_var('0.0', '#')
+        assert match_var('0.0', '_')
 
     def test_parse_op(self):
         res = OpMatcher.parse_op("  a =   int_add(  b,  3 ) # foo")
@@ -210,6 +228,19 @@
         """
         assert not self.match(loop, expected)
 
+    def test_dotdotdot_in_operation(self):
+        loop = """
+            [i0, i1]
+            jit_debug(i0, 1, ConstClass(myclass), i1)
+        """
+        assert self.match(loop, "jit_debug(...)")
+        assert self.match(loop, "jit_debug(i0, ...)")
+        assert self.match(loop, "jit_debug(i0, 1, ...)")
+        assert self.match(loop, "jit_debug(i0, 1, _, ...)")
+        assert self.match(loop, "jit_debug(i0, 1, _, i1, ...)")
+        py.test.raises(AssertionError, self.match,
+                       loop, "jit_debug(i0, 1, ..., i1)")
+
     def test_match_descr(self):
         loop = """
             [p0]
@@ -232,7 +263,7 @@
             jump(i4)
         """
         expected = """
-            i1 = int_add(0, 1)
+            i1 = int_add(i0, 1)
             ...
             i4 = int_mul(i1, 1000)
             jump(i4, descr=...)
@@ -249,7 +280,7 @@
             jump(i4, descr=...)
         """
         expected = """
-            i1 = int_add(0, 1)
+            i1 = int_add(i0, 1)
             ...
             _ = int_mul(_, 1000)
             jump(i4, descr=...)
@@ -268,7 +299,7 @@
             jump(i4)
         """
         expected = """
-            i1 = int_add(0, 1)
+            i1 = int_add(i0, 1)
             ...
         """
         assert self.match(loop, expected)
diff --git a/pypy/module/pypyjit/test_pypy_c/test_buffers.py b/pypy/module/pypyjit/test_pypy_c/test_buffers.py
--- a/pypy/module/pypyjit/test_pypy_c/test_buffers.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_buffers.py
@@ -21,7 +21,7 @@
             i65 = getfield_gc(p18, descr=...)
             i67 = int_gt(0, i65)
             guard_false(i67, descr=...)
-            i69 = int_gt(., i65)
+            i69 = int_gt(#, i65)
             guard_true(i69, descr=...)
             --TICK--
         """)
@@ -56,7 +56,7 @@
             guard_false(i99, descr=...)
             i100 = int_lshift(i98, 24)
             i101 = int_or(i97, i100)
-            i102 = getfield_raw(\d+, descr=)
+            i102 = getfield_raw(#, descr=)
             i103 = int_lt(i102, 0)
             guard_false(i103, descr=...)
         """)
diff --git a/pypy/module/pypyjit/test_pypy_c/test_call.py b/pypy/module/pypyjit/test_pypy_c/test_call.py
--- a/pypy/module/pypyjit/test_pypy_c/test_call.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_call.py
@@ -395,7 +395,7 @@
             setarrayitem_gc(p24, 0, p26, descr=)
             setfield_gc(p22, p24, descr=)
             }}}
-            p32 = call_may_force(..., p18, p22, descr=)
+            p32 = call_may_force(_, p18, p22, descr=)
             ...
         """)
 
diff --git a/pypy/module/pypyjit/test_pypy_c/test_containers.py b/pypy/module/pypyjit/test_pypy_c/test_containers.py
--- a/pypy/module/pypyjit/test_pypy_c/test_containers.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_containers.py
@@ -82,7 +82,7 @@
             guard_no_exception(descr=...)
             i23 = call(ConstClass(ll_dict_lookup_trampoline), p13, p10, i12, descr=)
             guard_no_exception(descr=...)
-            i26 = int_and(i23, .*)
+            i26 = int_and(i23, #)
             i27 = int_is_true(i26)
             guard_false(i27, descr=...)
             p28 = getfield_gc(p13, descr=)
diff --git a/pypy/module/pypyjit/test_pypy_c/test_math.py b/pypy/module/pypyjit/test_pypy_c/test_math.py
--- a/pypy/module/pypyjit/test_pypy_c/test_math.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_math.py
@@ -21,7 +21,7 @@
             guard_true(i2, descr=...)
             guard_not_invalidated(descr=...)
             f1 = cast_int_to_float(i0)
-            i3 = float_le(f1, 0)
+            i3 = float_le(f1, 0.0)
             guard_false(i3, descr=...)
             f2 = call(ConstClass(log), f1, descr=)
             f3 = call(ConstClass(log10), f1, descr=)
@@ -56,7 +56,7 @@
             f3 = call(ConstClass(cos), f1, descr=)
             f4 = float_sub(f2, f3)
             f5 = float_add(f0, f4)
-            i7 = int_add(i0, f1)
+            i7 = int_add(i0, 1)
             --TICK--
             jump(..., descr=)
         """)
diff --git a/pypy/module/pypyjit/test_pypy_c/test_micronumpy.py b/pypy/module/pypyjit/test_pypy_c/test_micronumpy.py
--- a/pypy/module/pypyjit/test_pypy_c/test_micronumpy.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_micronumpy.py
@@ -104,7 +104,7 @@
             setarrayitem_gc(p150, 1, 0, descr=)
             setarrayitem_gc(p150, 0, 0, descr=)
             guard_not_invalidated(descr=...)
-            i154 = getfield_raw(ticker_address, descr=)
+            i154 = getfield_raw(#, descr=)
             i155 = int_lt(i154, 0)
             guard_false(i155, descr=...)
             p156 = new_with_vtable(...)
@@ -142,7 +142,7 @@
             raw_store(i103, i132, 42.000000, descr=)
             p152 = getfield_gc_pure(p126, descr=)
             i153 = int_add(i120, 1)
-            i154 = getfield_raw(ticker_address, descr=)
+            i154 = getfield_raw(#, descr=)
             setarrayitem_gc(p152, 1, 0, descr=)
             setarrayitem_gc(p152, 0, 0, descr=)
             i157 = int_lt(i154, 0)
diff --git a/pypy/module/pypyjit/test_pypy_c/test_string.py b/pypy/module/pypyjit/test_pypy_c/test_string.py
--- a/pypy/module/pypyjit/test_pypy_c/test_string.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_string.py
@@ -67,15 +67,15 @@
             i11 = int_lt(i6, i7)
             guard_true(i11, descr=...)
             guard_not_invalidated(descr=...)
-            i13 = int_eq(i6, %d)
+            i13 = int_eq(i6, %d)         # value provided below
             guard_false(i13, descr=...)
-            i15 = int_mod(i6, i8)
-            i17 = int_rshift(i15, %d)
-            i18 = int_and(i8, i17)
+            i15 = int_mod(i6, 10)
+            i17 = int_rshift(i15, %d)    # value provided below
+            i18 = int_and(10, i17)
             i19 = int_add(i15, i18)
             i21 = int_lt(i19, 0)
             guard_false(i21, descr=...)
-            i22 = int_ge(i19, i8)
+            i22 = int_ge(i19, 10)
             guard_false(i22, descr=...)
             i23 = strgetitem(p10, i19)
             p25 = newstr(1)
@@ -83,7 +83,7 @@
             p93 = call(ConstClass(fromstr), p25, 16, descr=)
             guard_no_exception(descr=...)
             i95 = getfield_gc_pure(p93, descr=)
-            i96 = int_gt(i95, .*)
+            i96 = int_gt(i95, #)
             guard_false(i96, descr=...)
             i94 = call(ConstClass(rbigint._toint_helper), p93, descr=)
             guard_no_exception(descr=...)
diff --git a/pypy/module/pypyjit/test_pypy_c/test_thread.py b/pypy/module/pypyjit/test_pypy_c/test_thread.py
--- a/pypy/module/pypyjit/test_pypy_c/test_thread.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_thread.py
@@ -64,7 +64,7 @@
         guard_true(i56, descr=...)
         p57 = force_token()
         setfield_gc(p0, p57, descr=)
-        i58 = call_release_gil(..., i37, 1, descr=)
+        i58 = call_release_gil(_, i37, 1, descr=)
         guard_not_forced(descr=...)
         guard_no_exception(descr=...)
         i59 = int_is_true(i58)
@@ -72,14 +72,14 @@
         i60 = int_sub(i44, 1)
         p62 = force_token()
         setfield_gc(p0, p62, descr=)
-        i63 = call_release_gil(..., i37, 0, descr=)
+        i63 = call_release_gil(_, i37, 0, descr=)
         guard_not_forced(descr=...)
         guard_no_exception(descr=...)
         i64 = int_is_true(i63)
         guard_false(i64, descr=...)
         p65 = force_token()
         setfield_gc(p0, p65, descr=)
-        call_release_gil(..., i37, descr=)
+        call_release_gil(_, i37, descr=)
         guard_not_forced(descr=...)
         guard_no_exception(descr=...)
         guard_not_invalidated(descr=...)
diff --git a/pypy/objspace/std/marshal_impl.py b/pypy/objspace/std/marshal_impl.py
--- a/pypy/objspace/std/marshal_impl.py
+++ b/pypy/objspace/std/marshal_impl.py
@@ -322,8 +322,8 @@
     m.put_int(x.co_stacksize)
     m.put_int(x.co_flags)
     m.atom_str(TYPE_STRING, x.co_code)
-    m.put_tuple_w(TYPE_TUPLE, x.co_consts_w[:])
-    m.atom_strlist(TYPE_TUPLE, TYPE_STRING, [space.str_w(w_name) for w_name in x.co_names_w])
+    m.put_tuple_w(TYPE_TUPLE, x.co_consts_w)
+    m.put_tuple_w(TYPE_TUPLE, x.co_names_w)
     m.atom_strlist(TYPE_TUPLE, TYPE_STRING, x.co_varnames)
     m.atom_strlist(TYPE_TUPLE, TYPE_STRING, x.co_freevars)
     m.atom_strlist(TYPE_TUPLE, TYPE_STRING, x.co_cellvars)
@@ -332,9 +332,8 @@
     m.put_int(x.co_firstlineno)
     m.atom_str(TYPE_STRING, x.co_lnotab)
 
-# helper for unmarshalling string lists of code objects.
-# unfortunately they now can be interned or referenced,
-# so we no longer can handle it in interp_marshal.atom_strlist
+# helper for unmarshalling "tuple of string" objects
+# into rpython-level lists of strings.  Only for code objects.
 
 def unmarshal_str(u):
     w_obj = u.get_w_obj()
diff --git a/rpython/flowspace/bytecode.py b/rpython/flowspace/bytecode.py
--- a/rpython/flowspace/bytecode.py
+++ b/rpython/flowspace/bytecode.py
@@ -81,37 +81,37 @@
         and **varkwarg, if they exist."""
         return self.signature.scope_length()
 
-    def read(self, pos):
+    def read(self, offset):
         """
-        Decode the instruction starting at position ``next_instr``.
+        Decode the instruction starting at position ``offset``.
 
-        Returns (next_instr, opname, oparg).
+        Returns (next_offset, opname, oparg).
         """
         co_code = self.co_code
-        opnum = ord(co_code[pos])
-        next_instr = pos + 1
+        opnum = ord(co_code[offset])
+        next_offset = offset + 1
 
         if opnum >= HAVE_ARGUMENT:
-            lo = ord(co_code[next_instr])
-            hi = ord(co_code[next_instr+1])
-            next_instr += 2
+            lo = ord(co_code[next_offset])
+            hi = ord(co_code[next_offset + 1])
+            next_offset += 2
             oparg = (hi * 256) | lo
         else:
             oparg = 0
 
         while opnum == EXTENDED_ARG:
-            opnum = ord(co_code[next_instr])
+            opnum = ord(co_code[next_offset])
             if opnum < HAVE_ARGUMENT:
                 raise BytecodeCorruption
-            lo = ord(co_code[next_instr+1])
-            hi = ord(co_code[next_instr+2])
-            next_instr += 3
+            lo = ord(co_code[next_offset + 1])
+            hi = ord(co_code[next_offset + 2])
+            next_offset += 3
             oparg = (oparg * 65536) | (hi * 256) | lo
 
         if opnum in opcode.hasjrel:
-            oparg += next_instr
+            oparg += next_offset
         opname = self.opnames[opnum]
-        return next_instr, opname, oparg
+        return next_offset, opname, oparg
 
     @property
     def is_generator(self):
diff --git a/rpython/flowspace/flowcontext.py b/rpython/flowspace/flowcontext.py
--- a/rpython/flowspace/flowcontext.py
+++ b/rpython/flowspace/flowcontext.py
@@ -29,7 +29,7 @@
         msg = ["\n"]
         msg += map(str, self.args)
         msg += [""]
-        msg += source_lines(self.ctx.graph, None, offset=self.ctx.last_instr)
+        msg += source_lines(self.ctx.graph, None, offset=self.ctx.last_offset)
         return "\n".join(msg)
 
 
@@ -288,7 +288,7 @@
 
         self.init_closure(func.func_closure)
         self.f_lineno = code.co_firstlineno
-        self.last_instr = 0
+        self.last_offset = 0
 
         self.init_locals_stack(code)
 
@@ -359,7 +359,7 @@
         self.locals_stack_w[:len(items_w)] = items_w
         self.dropvaluesuntil(len(items_w))
 
-    def getstate(self, next_pos):
+    def getstate(self, next_offset):
         # getfastscope() can return real None, for undefined locals
         data = self.save_locals_stack()
         if self.last_exception is None:
@@ -369,7 +369,7 @@
             data.append(self.last_exception.w_type)
             data.append(self.last_exception.w_value)
         recursively_flatten(data)
-        return FrameState(data, self.blockstack[:], next_pos)
+        return FrameState(data, self.blockstack[:], next_offset)
 
     def setstate(self, state):
         """ Reset the context to the given frame state. """
@@ -393,7 +393,7 @@
         if getattr(recorder, 'final_state', None) is not None:
             self.mergeblock(recorder.crnt_block, recorder.final_state)
             raise StopFlowing
-        spaceop.offset = self.last_instr
+        spaceop.offset = self.last_offset
         recorder.append(spaceop)
 
     def do_op(self, op):
@@ -424,12 +424,12 @@
 
     def record_block(self, block):
         self.setstate(block.framestate)
-        next_pos = block.framestate.next_instr
+        next_offset = block.framestate.next_offset
         self.recorder = block.make_recorder()
         try:
             while True:
-                next_pos = self.handle_bytecode(next_pos)
-                self.recorder.final_state = self.getstate(next_pos)
+                next_offset = self.handle_bytecode(next_offset)
+                self.recorder.final_state = self.getstate(next_offset)
 
         except RaiseImplicit as e:
             w_exc = e.w_exc
@@ -467,10 +467,10 @@
         self.recorder = None
 
     def mergeblock(self, currentblock, currentstate):
-        next_instr = currentstate.next_instr
+        next_offset = currentstate.next_offset
         # can 'currentstate' be merged with one of the blocks that
         # already exist for this bytecode position?
-        candidates = self.joinpoints.setdefault(next_instr, [])
+        candidates = self.joinpoints.setdefault(next_offset, [])
         for block in candidates:
             newstate = block.framestate.union(currentstate)
             if newstate is not None:
@@ -526,12 +526,12 @@
                     stack_items_w[i] = w_new
                     break
 
-    def handle_bytecode(self, next_instr):
-        self.last_instr = next_instr
-        next_instr, methodname, oparg = self.pycode.read(next_instr)
+    def handle_bytecode(self, next_offset):
+        self.last_offset = next_offset
+        next_offset, methodname, oparg = self.pycode.read(next_offset)
         try:
-            res = getattr(self, methodname)(oparg)
-            return res if res is not None else next_instr
+            offset = getattr(self, methodname)(oparg)
+            return offset if offset is not None else next_offset
         except FlowSignal as signal:
             return self.unroll(signal)
 
@@ -856,14 +856,9 @@
     def WITH_CLEANUP(self, oparg):
         # Note: RPython context managers receive None in lieu of tracebacks
         # and cannot suppress the exception.
-        # This opcode changed a lot between CPython versions
-        if sys.version_info >= (2, 6):
-            unroller = self.popvalue()
-            w_exitfunc = self.popvalue()
-            self.pushvalue(unroller)
-        else:
-            w_exitfunc = self.popvalue()
-            unroller = self.peekvalue(0)
+        unroller = self.popvalue()
+        w_exitfunc = self.popvalue()
+        self.pushvalue(unroller)
 
         if isinstance(unroller, Raise):
             w_exc = unroller.w_exc
diff --git a/rpython/flowspace/framestate.py b/rpython/flowspace/framestate.py
--- a/rpython/flowspace/framestate.py
+++ b/rpython/flowspace/framestate.py
@@ -3,10 +3,10 @@
 
 
 class FrameState(object):
-    def __init__(self, mergeable, blocklist, next_instr):
+    def __init__(self, mergeable, blocklist, next_offset):
         self.mergeable = mergeable
         self.blocklist = blocklist
-        self.next_instr = next_instr
+        self.next_offset = next_offset
 
     def copy(self):
         "Make a copy of this state in which all Variables are fresh."
@@ -15,7 +15,7 @@
             if isinstance(w, Variable):
                 w = Variable(w)
             newstate.append(w)
-        return FrameState(newstate, self.blocklist, self.next_instr)
+        return FrameState(newstate, self.blocklist, self.next_offset)
 
     def getvariables(self):
         return [w for w in self.mergeable if isinstance(w, Variable)]
@@ -26,7 +26,7 @@
         # safety check, don't try to compare states with different
         # nonmergeable states
         assert self.blocklist == other.blocklist
-        assert self.next_instr == other.next_instr
+        assert self.next_offset == other.next_offset
         for w1, w2 in zip(self.mergeable, other.mergeable):
             if not (w1 == w2 or (isinstance(w1, Variable) and
                                  isinstance(w2, Variable))):
@@ -44,7 +44,7 @@
                 newstate.append(union(w1, w2))
         except UnionError:
             return None
-        return FrameState(newstate, self.blocklist, self.next_instr)
+        return FrameState(newstate, self.blocklist, self.next_offset)
 
     def getoutputargs(self, targetstate):
         "Return the output arguments needed to link self to targetstate."
diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py
--- a/rpython/jit/backend/x86/assembler.py
+++ b/rpython/jit/backend/x86/assembler.py
@@ -1832,15 +1832,17 @@
             offset = self.cpu.get_ofs_of_frame_field('jf_guard_exc')
             mc.MOV_br(offset, ebx.value)
 
-        # now we return from the complete frame, which starts from
-        # _call_header_with_stack_check().  The LEA in _call_footer below
-        # throws away most of the frame, including all the PUSHes that we
-        # did just above.
+        # fill in the jf_descr and jf_gcmap fields of the frame according
+        # to which failure we are resuming from.  These are constants
+        # pushed on the stack just before we jump to the current helper,
+        # in generate_quick_failure().
         ofs = self.cpu.get_ofs_of_frame_field('jf_descr')
         ofs2 = self.cpu.get_ofs_of_frame_field('jf_gcmap')
         mc.POP_b(ofs2)
         mc.POP_b(ofs)
 
+        # now we return from the complete frame, which starts from
+        # _call_header_with_stack_check().  The _call_footer below does it.
         self._call_footer()
         rawstart = mc.materialize(self.cpu.asmmemmgr, [])
         self.failure_recovery_code[exc + 2 * withfloats] = rawstart
diff --git a/rpython/memory/gc/incminimark.py b/rpython/memory/gc/incminimark.py
--- a/rpython/memory/gc/incminimark.py
+++ b/rpython/memory/gc/incminimark.py
@@ -216,8 +216,8 @@
         # fall-back number.
         "nursery_size": 896*1024,
 
-        # The system page size.  Like obmalloc.c, we assume that it is 4K
-        # for 32-bit systems; unlike obmalloc.c, we assume that it is 8K
+        # The system page size.  Like malloc, we assume that it is 4K
+        # for 32-bit systems; unlike malloc, we assume that it is 8K
         # for 64-bit systems, for consistent results.
         "page_size": 1024*WORD,
 
diff --git a/rpython/memory/gc/minimark.py b/rpython/memory/gc/minimark.py
--- a/rpython/memory/gc/minimark.py
+++ b/rpython/memory/gc/minimark.py
@@ -174,8 +174,8 @@
         # fall-back number.
         "nursery_size": 896*1024,
 
-        # The system page size.  Like obmalloc.c, we assume that it is 4K
-        # for 32-bit systems; unlike obmalloc.c, we assume that it is 8K
+        # The system page size.  Like malloc, we assume that it is 4K
+        # for 32-bit systems; unlike malloc, we assume that it is 8K
         # for 64-bit systems, for consistent results.
         "page_size": 1024*WORD,
 
diff --git a/rpython/memory/gc/minimarktest.py b/rpython/memory/gc/minimarktest.py
--- a/rpython/memory/gc/minimarktest.py
+++ b/rpython/memory/gc/minimarktest.py
@@ -5,7 +5,7 @@
 from rpython.rlib.rarithmetic import LONG_BIT
 
 # For testing, a simple implementation of ArenaCollection.
-# This version could be used together with obmalloc.c, but
+# This version could be used together with malloc, but
 # it requires an extra word per object in the 'all_objects'
 # list.
 
diff --git a/rpython/memory/gctransform/framework.py b/rpython/memory/gctransform/framework.py
--- a/rpython/memory/gctransform/framework.py
+++ b/rpython/memory/gctransform/framework.py
@@ -461,17 +461,18 @@
                                             annmodel.SomeInteger(nonneg=True)],
                                            annmodel.s_None)
 
-        self.pin_ptr = getfn(GCClass.pin,
-                             [s_gc, SomeAddress()],
-                             annmodel.SomeBool())
+        if GCClass.can_usually_pin_objects:
+            self.pin_ptr = getfn(GCClass.pin,
+                                 [s_gc, SomeAddress()],
+                                 annmodel.SomeBool())
 
-        self.unpin_ptr = getfn(GCClass.unpin,
-                               [s_gc, SomeAddress()],
-                               annmodel.s_None)
+            self.unpin_ptr = getfn(GCClass.unpin,
+                                   [s_gc, SomeAddress()],
+                                   annmodel.s_None)
 
-        self._is_pinned_ptr = getfn(GCClass._is_pinned,
-                                    [s_gc, SomeAddress()],
-                                    annmodel.SomeBool())
+            self._is_pinned_ptr = getfn(GCClass._is_pinned,
+                                        [s_gc, SomeAddress()],
+                                        annmodel.SomeBool())
 
         self.write_barrier_ptr = None
         self.write_barrier_from_array_ptr = None
@@ -1042,6 +1043,10 @@
                                   v_size])
 
     def gct_gc_pin(self, hop):
+        if not hasattr(self, 'pin_ptr'):
+            c_false = rmodel.inputconst(lltype.Bool, False)
+            hop.genop("same_as", [c_false], resultvar=hop.spaceop.result)
+            return
         op = hop.spaceop
         v_addr = hop.genop('cast_ptr_to_adr', [op.args[0]],
             resulttype=llmemory.Address)
@@ -1049,6 +1054,8 @@
                   resultvar=op.result)
 
     def gct_gc_unpin(self, hop):
+        if not hasattr(self, 'unpin_ptr'):
+            return
         op = hop.spaceop
         v_addr = hop.genop('cast_ptr_to_adr', [op.args[0]],
             resulttype=llmemory.Address)
@@ -1056,6 +1063,10 @@
                   resultvar=op.result)
 
     def gct_gc__is_pinned(self, hop):
+        if not hasattr(self, '_is_pinned_ptr'):
+            c_false = rmodel.inputconst(lltype.Bool, False)
+            hop.genop("same_as", [c_false], resultvar=hop.spaceop.result)
+            return
         op = hop.spaceop
         v_addr = hop.genop('cast_ptr_to_adr', [op.args[0]],
             resulttype=llmemory.Address)
diff --git a/rpython/rlib/rweakref.py b/rpython/rlib/rweakref.py
--- a/rpython/rlib/rweakref.py
+++ b/rpython/rlib/rweakref.py
@@ -105,7 +105,7 @@
                                                rtyper.getrepr(self.s_key))
 
     def rtyper_makekey(self):
-        return self.__class__,
+        return self.__class__, self.s_key.rtyper_makekey(), self.valueclassdef
 
     def method_get(self, s_key):
         return annmodel.SomeInstance(self.valueclassdef, can_be_None=True)
@@ -165,7 +165,7 @@
         return _rweakkeydict.WeakKeyDictRepr(rtyper)
 
     def rtyper_makekey(self):
-        return self.__class__,
+        return self.__class__, self.keyclassdef, self.valueclassdef
 
     def method_get(self, s_key):
         assert isinstance(s_key, annmodel.SomeInstance)
diff --git a/rpython/rlib/test/test_rsocket.py b/rpython/rlib/test/test_rsocket.py
--- a/rpython/rlib/test/test_rsocket.py
+++ b/rpython/rlib/test/test_rsocket.py
@@ -311,7 +311,7 @@
     assert isinstance(lst, list)
     found = False
     for family, socktype, protocol, canonname, addr in lst:
-        if addr.get_host() == '140.211.10.69':
+        if addr.get_host() == '104.130.43.121':
             found = True
     result[i] += found
 
diff --git a/rpython/rlib/test/test_rweakvaldict.py b/rpython/rlib/test/test_rweakvaldict.py
--- a/rpython/rlib/test/test_rweakvaldict.py
+++ b/rpython/rlib/test/test_rweakvaldict.py
@@ -144,3 +144,13 @@
             d = RWeakValueDictionary(str, Y)
         d.set("x", X())
     py.test.raises(Exception, interpret, g, [1])
+
+
+def test_bogus_makekey():
+    class X: pass
+    class Y: pass
+    def g():
+        X(); Y()
+        RWeakValueDictionary(str, X).get("foobar")
+        RWeakValueDictionary(int, Y).get(42)
+    interpret(g, [])
diff --git a/rpython/rtyper/lltypesystem/test/test_rffi.py b/rpython/rtyper/lltypesystem/test/test_rffi.py
--- a/rpython/rtyper/lltypesystem/test/test_rffi.py
+++ b/rpython/rtyper/lltypesystem/test/test_rffi.py
@@ -100,7 +100,6 @@
     def test_string_reverse(self):
         c_source = py.code.Source("""
         #include 
-        #include 
         #include 
 
         char *f(char* arg)
diff --git a/rpython/rtyper/normalizecalls.py b/rpython/rtyper/normalizecalls.py
--- a/rpython/rtyper/normalizecalls.py
+++ b/rpython/rtyper/normalizecalls.py
@@ -298,12 +298,16 @@
 
 # ____________________________________________________________
 
+class TooLateForNewSubclass(Exception):
+    pass
+
 class TotalOrderSymbolic(ComputedIntSymbolic):
 
     def __init__(self, orderwitness, peers):
         self.orderwitness = orderwitness
         self.peers = peers
         self.value = None
+        self._with_subclasses = None    # unknown
         peers.append(self)
 
     def __cmp__(self, other):
@@ -320,12 +324,34 @@
     def __rsub__(self, other):
         return other - self.compute_fn()
 
+    def check_any_subclass_in_peer_list(self, i):
+        # check if the next peer, in order, is or not the end
+        # marker for this start marker
+        assert self.peers[i] is self
+        return self.peers[i + 1].orderwitness != self.orderwitness + [MAX]
+
+    def number_with_subclasses(self):
+        # Return True or False depending on whether this is the
+        # subclassrange_min corresponding to a class which has subclasses
+        # or not.  If this is called and returns False, then adding later
+        # new subclasses will crash in compute_fn().
+        if self._with_subclasses is None:     # unknown so far
+            self.peers.sort()
+            i = self.peers.index(self)
+            self._with_subclasses = self.check_any_subclass_in_peer_list(i)
+        return self._with_subclasses
+
     def compute_fn(self):
         if self.value is None:
             self.peers.sort()
             for i, peer in enumerate(self.peers):
                 assert peer.value is None or peer.value == i
                 peer.value = i
+                #
+                if peer._with_subclasses is False:
+                    if peer.check_any_subclass_in_peer_list(i):
+                        raise TooLateForNewSubclass
+                #
             assert self.value is not None
         return self.value
 
diff --git a/rpython/rtyper/rclass.py b/rpython/rtyper/rclass.py
--- a/rpython/rtyper/rclass.py
+++ b/rpython/rtyper/rclass.py
@@ -1007,14 +1007,11 @@
         v_obj, v_cls = hop.inputargs(instance_repr, class_repr)
         if isinstance(v_cls, Constant):
             cls = v_cls.value
-            # XXX re-implement the following optimization
-            #if cls.subclassrange_max == cls.subclassrange_min:
-            #    # a class with no subclass
-            #    return hop.gendirectcall(rclass.ll_isinstance_exact, v_obj, v_cls)
-            #else:
-            minid = hop.inputconst(Signed, cls.subclassrange_min)
-            maxid = hop.inputconst(Signed, cls.subclassrange_max)
-            return hop.gendirectcall(ll_isinstance_const, v_obj, minid, maxid)
+            llf, llf_nonnull = make_ll_isinstance(self.rtyper, cls)
+            if hop.args_s[0].can_be_None:
+                return hop.gendirectcall(llf, v_obj)
+            else:
+                return hop.gendirectcall(llf_nonnull, v_obj)
         else:
             return hop.gendirectcall(ll_isinstance, v_obj, v_cls)
 
@@ -1128,16 +1125,26 @@
     obj_cls = obj.typeptr
     return ll_issubclass(obj_cls, cls)
 
-def ll_isinstance_const(obj, minid, maxid):
-    if not obj:
-        return False
-    return ll_issubclass_const(obj.typeptr, minid, maxid)
-
-def ll_isinstance_exact(obj, cls):
-    if not obj:
-        return False
-    obj_cls = obj.typeptr
-    return obj_cls == cls
+def make_ll_isinstance(rtyper, cls):
+    try:
+        return rtyper.isinstance_helpers[cls._obj]
+    except KeyError:
+        minid = cls.subclassrange_min
+        maxid = cls.subclassrange_max
+        if minid.number_with_subclasses():
+            def ll_isinstance_const_nonnull(obj):
+                objid = obj.typeptr.subclassrange_min
+                return llop.int_between(Bool, minid, objid, maxid)
+        else:
+            def ll_isinstance_const_nonnull(obj):
+                return obj.typeptr == cls
+        def ll_isinstance_const(obj):
+            if not obj:
+                return False
+            return ll_isinstance_const_nonnull(obj)
+        result = (ll_isinstance_const, ll_isinstance_const_nonnull)
+        rtyper.isinstance_helpers[cls._obj] = result
+        return result
 
 def ll_runtime_type_info(obj):
     return obj.typeptr.rtti
diff --git a/rpython/rtyper/rtyper.py b/rpython/rtyper/rtyper.py
--- a/rpython/rtyper/rtyper.py
+++ b/rpython/rtyper/rtyper.py
@@ -59,6 +59,7 @@
         self.typererror_count = 0
         # make the primitive_to_repr constant mapping
         self.primitive_to_repr = {}
+        self.isinstance_helpers = {}
         self.exceptiondata = ExceptionData(self)
         self.custom_trace_funcs = []
 
diff --git a/rpython/rtyper/test/test_normalizecalls.py b/rpython/rtyper/test/test_normalizecalls.py
--- a/rpython/rtyper/test/test_normalizecalls.py
+++ b/rpython/rtyper/test/test_normalizecalls.py
@@ -6,6 +6,7 @@
 from rpython.rtyper.test.test_llinterp import interpret
 from rpython.rtyper.lltypesystem import lltype
 from rpython.rtyper.normalizecalls import TotalOrderSymbolic, MAX
+from rpython.rtyper.normalizecalls import TooLateForNewSubclass
 
 
 def test_TotalOrderSymbolic():
@@ -21,6 +22,49 @@
     assert t1 <= 5
     assert t1.value == 0
 
+def test_TotalOrderSymbolic_with_subclasses():
+    lst = []
+    t3 = TotalOrderSymbolic([3, 4, 2, MAX], lst)
+    t1 = TotalOrderSymbolic([3, 4], lst)
+    t2 = TotalOrderSymbolic([3, 4, 2], lst)
+    t4 = TotalOrderSymbolic([3, 4, MAX], lst)
+    assert t1.number_with_subclasses()
+    assert not t2.number_with_subclasses()
+    assert [t.compute_fn() for t in [t1, t2, t3, t4]] == range(4)
+    #
+    lst = []
+    t1 = TotalOrderSymbolic([3, 4], lst)
+    t3 = TotalOrderSymbolic([3, 4, 2, MAX], lst)
+    t4 = TotalOrderSymbolic([3, 4, MAX], lst)
+    t2 = TotalOrderSymbolic([3, 4, 2], lst)
+    assert not t2.number_with_subclasses()
+    assert t1.number_with_subclasses()
+    assert [t.compute_fn() for t in [t1, t2, t3, t4]] == range(4)
+    #
+    lst = []
+    t1 = TotalOrderSymbolic([3, 4], lst)
+    t4 = TotalOrderSymbolic([3, 4, MAX], lst)
+    assert not t1.number_with_subclasses()
+    t2 = TotalOrderSymbolic([3, 4, 2], lst)
+    t3 = TotalOrderSymbolic([3, 4, 2, MAX], lst)
+    py.test.raises(TooLateForNewSubclass, t2.compute_fn)
+    #
+    lst = []
+    t1 = TotalOrderSymbolic([3, 4], lst)
+    t4 = TotalOrderSymbolic([3, 4, MAX], lst)
+    assert not t1.number_with_subclasses()
+    t2 = TotalOrderSymbolic([1], lst)
+    t3 = TotalOrderSymbolic([1, MAX], lst)
+    assert [t.compute_fn() for t in [t2, t3, t1, t4]] == range(4)
+    #
+    lst = []
+    t1 = TotalOrderSymbolic([3, 4], lst)
+    t4 = TotalOrderSymbolic([3, 4, MAX], lst)
+    assert not t1.number_with_subclasses()
+    t2 = TotalOrderSymbolic([6], lst)
+    t3 = TotalOrderSymbolic([6, MAX], lst)
+    assert [t.compute_fn() for t in [t1, t4, t2, t3]] == range(4)
+
 # ____________________________________________________________
 
 class TestNormalize(object):
diff --git a/rpython/tool/jitlogparser/parser.py b/rpython/tool/jitlogparser/parser.py
--- a/rpython/tool/jitlogparser/parser.py
+++ b/rpython/tool/jitlogparser/parser.py
@@ -216,7 +216,7 @@
     line_starts_here = property(getline_starts_here)
 
     def __repr__(self):
-        return "[%s]" % ", ".join([repr(op) for op in self.operations])
+        return "[%s\n]" % "\n    ".join([repr(op) for op in self.operations])
 
     def pretty_print(self, out):
         pass
diff --git a/rpython/translator/c/genc.py b/rpython/translator/c/genc.py
--- a/rpython/translator/c/genc.py
+++ b/rpython/translator/c/genc.py
@@ -405,8 +405,6 @@
             ('debug', '', '$(MAKE) CFLAGS="$(DEBUGFLAGS) -DRPY_ASSERT" debug_target'),
             ('debug_exc', '', '$(MAKE) CFLAGS="$(DEBUGFLAGS) -DRPY_ASSERT -DDO_LOG_EXC" debug_target'),
             ('debug_mem', '', '$(MAKE) CFLAGS="$(DEBUGFLAGS) -DRPY_ASSERT -DPYPY_USE_TRIVIAL_MALLOC" debug_target'),
-            ('no_obmalloc', '', '$(MAKE) CFLAGS="-g -O2 -DRPY_ASSERT -DPYPY_NO_OBMALLOC" $(TARGET)'),
-            ('linuxmemchk', '', '$(MAKE) CFLAGS="$(DEBUGFLAGS) -DRPY_ASSERT -DPYPY_USE_LINUXMEMCHK" debug_target'),
             ('llsafer', '', '$(MAKE) CFLAGS="-O2 -DRPY_LL_ASSERT" $(TARGET)'),
             ('lldebug', '', '$(MAKE) CFLAGS="$(DEBUGFLAGS) -DRPY_ASSERT -DRPY_LL_ASSERT" debug_target'),
             ('lldebug0','', '$(MAKE) CFLAGS="$(DEBUGFLAGS) -O0 -DRPY_ASSERT -DRPY_LL_ASSERT" debug_target'),
@@ -762,7 +760,6 @@
     srcdir = py.path.local(__file__).join('..', 'src')
     files = [
         srcdir / 'entrypoint.c',       # ifdef PYPY_STANDALONE
-        srcdir / 'allocator.c',        # ifdef PYPY_STANDALONE
         srcdir / 'mem.c',
         srcdir / 'exception.c',
         srcdir / 'rtyper.c',           # ifdef HAVE_RTYPER
diff --git a/rpython/translator/c/src/allocator.c b/rpython/translator/c/src/allocator.c
deleted file mode 100644
--- a/rpython/translator/c/src/allocator.c
+++ /dev/null
@@ -1,33 +0,0 @@
-/* allocation functions */
-#include "common_header.h"
-#ifdef PYPY_STANDALONE
-#include 
-
-#if defined(PYPY_USE_TRIVIAL_MALLOC)
-  void *PyObject_Malloc(size_t n) { return malloc(n); }
-  void *PyObject_Realloc(void *p, size_t n) { return realloc(p, n); }
-  void PyObject_Free(void *p) { if (p) { *((int*)p) = 0xDDDDDDDD; } free(p); }
-
-#elif defined(PYPY_USE_LINUXMEMCHK)
-#  include "linuxmemchk.c"
-
-#elif defined(PYPY_NO_OBMALLOC)
-  void *PyObject_Malloc(size_t n) { return malloc(n); }
-  void *PyObject_Realloc(void *p, size_t n) { return realloc(p, n); }
-  void PyObject_Free(void *p) { free(p); }
-
-#else
-#  ifndef WITH_PYMALLOC
-#    define WITH_PYMALLOC
-#  endif
-/* The same obmalloc as CPython */
-#  include "src/obmalloc.c"
-
-#endif
-#elif defined _MSC_VER
-/* link will fail without some kind of definition for the functions */  
-  void *PyObject_Malloc(size_t n) { return NULL; }
-  void *PyObject_Realloc(void *p, size_t n) { return NULL; }
-  void PyObject_Free(void *p) {  }
-
-#endif  /* PYPY_STANDALONE */
diff --git a/rpython/translator/c/src/allocator.h b/rpython/translator/c/src/allocator.h
deleted file mode 100644
--- a/rpython/translator/c/src/allocator.h
+++ /dev/null
@@ -1,4 +0,0 @@
-/* allocation functions prototypes */
-RPY_EXTERN void *PyObject_Malloc(size_t n);
-RPY_EXTERN void *PyObject_Realloc(void *p, size_t n);
-RPY_EXTERN void PyObject_Free(void *p);
diff --git a/rpython/translator/c/src/g_include.h b/rpython/translator/c/src/g_include.h
--- a/rpython/translator/c/src/g_include.h
+++ b/rpython/translator/c/src/g_include.h
@@ -33,7 +33,6 @@
 #  include "src/debug_traceback.h"
 #endif
 
-#  include "src/allocator.h"
 #ifdef PYPY_STANDALONE
 #  include "src/entrypoint.h"
 #endif
diff --git a/rpython/translator/c/src/linuxmemchk.c b/rpython/translator/c/src/linuxmemchk.c
deleted file mode 100644
--- a/rpython/translator/c/src/linuxmemchk.c
+++ /dev/null
@@ -1,101 +0,0 @@
-/* custom checking allocators a la Electric Fence */
-#include 
-#include 
-#include 
-#include 
-#include 
-#include 
-#define PAGESIZE 4096
-#ifndef MALLOC_BIGBUFFER
-# define MALLOC_BIGBUFFER   (PAGESIZE*32768)   /* 128MB */
-#endif
-
-
-struct _alloc_s {
-  void* ptr;
-  int npages;
-};
-static void* _na_start = NULL;
-static char* _na_cur;
-
-static void _na_assert(int x, char* msg)
-{
-  if (!x)
-    {
-      fprintf(stderr, "linuxmemchk: failed assertion: %s\n", msg);
-      abort();
-    }
-}
-
-static struct _alloc_s* _na_find(void* data)
-{
-  int err;
-  long data1;
-  struct _alloc_s* s;
-  _na_assert(_na_start+PAGESIZE <= data &&
-             data < _na_start+MALLOC_BIGBUFFER-PAGESIZE,
-             "corrupted na_start");
-  data1 = (long) data;
-  data1 &= ~(PAGESIZE-1);
-  data1 -= PAGESIZE;
-  err = mprotect((void*) data1, PAGESIZE, PROT_READ|PROT_WRITE);
-  _na_assert(!err, "mprotect[1] failed");
-  s = (struct _alloc_s*) data1;
-  _na_assert(s->npages > 0, "corrupted s->npages");
-  return s;
-}
-
-void* PyObject_Malloc(size_t size)
-{
-  int err, npages = (size + PAGESIZE-1) / PAGESIZE + 1;
-  struct _alloc_s* s;
-  char* data;
-  if (_na_start == NULL)
-    {
-      _na_start = mmap(NULL, MALLOC_BIGBUFFER, PROT_NONE,
-                       MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
-      _na_assert(_na_start != MAP_FAILED, "initial mmap failed");
-      _na_cur = (char*) _na_start;
-    }
-  s = (struct _alloc_s*) _na_cur;
-  _na_cur += npages * PAGESIZE;
-  if (_na_cur >= ((char*) _na_start) + MALLOC_BIGBUFFER)
-    {
-      fprintf(stderr, "linuxmemchk.c: Nothing wrong so far, but we are running out\nlinuxmemchk.c: of mmap'ed memory.  Increase MALLOC_BIGBUFFER.\n");
-      abort();
-    }
-  err = mprotect(s, npages * PAGESIZE, PROT_READ|PROT_WRITE|PROT_EXEC);
-  _na_assert(!err, "mprotect[2] failed");
-  s->ptr = data = _na_cur - /*((size+3)&~3)*/ size;
-  s->npages = npages;
-  err = mprotect(s, PAGESIZE, PROT_NONE);
-  _na_assert(!err, "mprotect[3] failed");
-  return data;
-}
-
-void PyObject_Free(void* data)
-{
-  int err, npages;
-  struct _alloc_s* s;
-  if (data == NULL)
-    return;
-  s = _na_find(data);
-  _na_assert(s->ptr == data, "free got a pointer not returned by malloc");
-  npages = s->npages;
-  s->npages = 0;
-  err = mprotect(s, npages * PAGESIZE, PROT_NONE);
-  _na_assert(!err, "mprotect[4] failed");
-}
-
-void* PyObject_Realloc(void* data, size_t nsize)
-{
-  size_t size;
-  struct _alloc_s* s = _na_find(data);
-  void* ndata = PyObject_Malloc(nsize);
-
-  _na_assert(s->ptr == data, "realloc got a pointer not returned by malloc");
-  size = ((char*)s) + s->npages * PAGESIZE - (char*)data;
-  memcpy(ndata, data, size
-#include 
-
-/* An object allocator for Python.
-
-   Here is an introduction to the layers of the Python memory architecture,
-   showing where the object allocator is actually used (layer +2), It is
-   called for every object allocation and deallocation (PyObject_New/Del),
-   unless the object-specific allocators implement a proprietary allocation
-   scheme (ex.: ints use a simple free list). This is also the place where
-   the cyclic garbage collector operates selectively on container objects.
-
-
-        Object-specific allocators
-    _____   ______   ______       ________
-   [ int ] [ dict ] [ list ] ... [ string ]       Python core         |
-+3 | <----- Object-specific memory -----> | <-- Non-object memory --> |
-    _______________________________       |                           |
-   [   Python's object allocator   ]      |                           |
-+2 | ####### Object memory ####### | <------ Internal buffers ------> |
-    ______________________________________________________________    |
-   [          Python's raw memory allocator (PyMem_ API)          ]   |
-+1 | <----- Python memory (under PyMem manager's control) ------> |   |
-    __________________________________________________________________
-   [    Underlying general-purpose allocator (ex: C library malloc)   ]
- 0 | <------ Virtual memory allocated for the python process -------> |
-
-   =========================================================================
-    _______________________________________________________________________
-   [                OS-specific Virtual Memory Manager (VMM)               ]
--1 | <--- Kernel dynamic storage allocation & management (page-based) ---> |
-    __________________________________   __________________________________
-   [                                  ] [                                  ]
--2 | <-- Physical memory: ROM/RAM --> | | <-- Secondary storage (swap) --> |
-
-*/
-/*==========================================================================*/
-
-/* A fast, special-purpose memory allocator for small blocks, to be used
-   on top of a general-purpose malloc -- heavily based on previous art. */
-
-/* Vladimir Marangozov -- August 2000 */
-
-/*
- * "Memory management is where the rubber meets the road -- if we do the wrong
- * thing at any level, the results will not be good. And if we don't make the
- * levels work well together, we are in serious trouble." (1)
- *
- * (1) Paul R. Wilson, Mark S. Johnstone, Michael Neely, and David Boles,
- *    "Dynamic Storage Allocation: A Survey and Critical Review",
- *    in Proc. 1995 Int'l. Workshop on Memory Management, September 1995.
- */
-

From noreply at buildbot.pypy.org  Sun Nov 30 23:12:44 2014
From: noreply at buildbot.pypy.org (pjenvey)
Date: Sun, 30 Nov 2014 23:12:44 +0100 (CET)
Subject: [pypy-commit] pypy py3k: redo atom_strlist which we still need w/ a
 smaller diff from default
Message-ID: <20141130221244.3EB0B1D24AA@cobra.cs.uni-duesseldorf.de>

Author: Philip Jenvey 
Branch: py3k
Changeset: r74768:1b7b870d25f5
Date: 2014-11-30 14:12 -0800
http://bitbucket.org/pypy/pypy/changeset/1b7b870d25f5/

Log:	redo atom_strlist which we still need w/ a smaller diff from default

diff --git a/pypy/objspace/std/marshal_impl.py b/pypy/objspace/std/marshal_impl.py
--- a/pypy/objspace/std/marshal_impl.py
+++ b/pypy/objspace/std/marshal_impl.py
@@ -311,6 +311,12 @@
     return None
 
 
+def _put_str_list(space, m, strlist):
+    m.atom_int(TYPE_TUPLE, len(strlist))
+    atom_str = m.atom_str
+    for item in strlist:
+        atom_str(TYPE_STRING, item)
+
 @marshaller(PyCode)
 def marshal_pycode(space, w_pycode, m):
     m.start(TYPE_CODE)
@@ -324,9 +330,9 @@
     m.atom_str(TYPE_STRING, x.co_code)
     m.put_tuple_w(TYPE_TUPLE, x.co_consts_w)
     m.put_tuple_w(TYPE_TUPLE, x.co_names_w)
-    m.atom_strlist(TYPE_TUPLE, TYPE_STRING, x.co_varnames)
-    m.atom_strlist(TYPE_TUPLE, TYPE_STRING, x.co_freevars)
-    m.atom_strlist(TYPE_TUPLE, TYPE_STRING, x.co_cellvars)
+    _put_str_list(space, m, x.co_varnames)
+    _put_str_list(space, m, x.co_freevars)
+    _put_str_list(space, m, x.co_cellvars)
     m.atom_str(TYPE_STRING, x.co_filename)
     m.atom_str(TYPE_STRING, x.co_name)
     m.put_int(x.co_firstlineno)

From noreply at buildbot.pypy.org  Sun Nov 30 23:12:47 2014
From: noreply at buildbot.pypy.org (pjenvey)
Date: Sun, 30 Nov 2014 23:12:47 +0100 (CET)
Subject: [pypy-commit] pypy py3k: merge default
Message-ID: <20141130221247.6C71A1D24AA@cobra.cs.uni-duesseldorf.de>

Author: Philip Jenvey 
Branch: py3k
Changeset: r74769:f682ccbce64f
Date: 2014-11-30 14:12 -0800
http://bitbucket.org/pypy/pypy/changeset/f682ccbce64f/

Log:	merge default

diff too long, truncating to 2000 out of 3147 lines

diff --git a/pypy/module/_cffi_backend/ctypeprim.py b/pypy/module/_cffi_backend/ctypeprim.py
--- a/pypy/module/_cffi_backend/ctypeprim.py
+++ b/pypy/module/_cffi_backend/ctypeprim.py
@@ -158,21 +158,14 @@
 
 
 class W_CTypePrimitiveSigned(W_CTypePrimitive):
-    _attrs_            = ['value_fits_long', 'vmin', 'vrangemax']
-    _immutable_fields_ = ['value_fits_long', 'vmin', 'vrangemax']
+    _attrs_            = ['value_fits_long', 'value_smaller_than_long']
+    _immutable_fields_ = ['value_fits_long', 'value_smaller_than_long']
     is_primitive_integer = True
 
     def __init__(self, *args):
         W_CTypePrimitive.__init__(self, *args)
         self.value_fits_long = self.size <= rffi.sizeof(lltype.Signed)
-        if self.size < rffi.sizeof(lltype.Signed):
-            assert self.value_fits_long
-            sh = self.size * 8
-            self.vmin = r_uint(-1) << (sh - 1)
-            self.vrangemax = (r_uint(1) << sh) - 1
-        else:
-            self.vmin = r_uint(0)
-            self.vrangemax = r_uint(-1)
+        self.value_smaller_than_long = self.size < rffi.sizeof(lltype.Signed)
 
     def cast_to_int(self, cdata):
         return self.convert_to_object(cdata)
@@ -192,8 +185,17 @@
     def convert_from_object(self, cdata, w_ob):
         if self.value_fits_long:
             value = misc.as_long(self.space, w_ob)
-            if self.size < rffi.sizeof(lltype.Signed):
-                if r_uint(value) - self.vmin > self.vrangemax:
+            if self.value_smaller_than_long:
+                size = self.size
+                if size == 1:
+                    signextended = misc.signext(value, 1)
+                elif size == 2:
+                    signextended = misc.signext(value, 2)
+                elif size == 4:
+                    signextended = misc.signext(value, 4)
+                else:
+                    raise AssertionError("unsupported size")
+                if value != signextended:
                     self._overflow(w_ob)
             misc.write_raw_signed_data(cdata, value, self.size)
         else:
@@ -221,7 +223,7 @@
             length = w_cdata.get_array_length()
             populate_list_from_raw_array(res, buf, length)
             return res
-        elif self.value_fits_long:
+        elif self.value_smaller_than_long:
             res = [0] * w_cdata.get_array_length()
             misc.unpack_list_from_raw_array(res, w_cdata._cdata, self.size)
             return res
@@ -235,8 +237,8 @@
                 cdata = rffi.cast(rffi.LONGP, cdata)
                 copy_list_to_raw_array(int_list, cdata)
             else:
-                overflowed = misc.pack_list_to_raw_array_bounds(
-                    int_list, cdata, self.size, self.vmin, self.vrangemax)
+                overflowed = misc.pack_list_to_raw_array_bounds_signed(
+                    int_list, cdata, self.size)
                 if overflowed != 0:
                     self._overflow(self.space.wrap(overflowed))
             return True
@@ -314,8 +316,8 @@
     def pack_list_of_items(self, cdata, w_ob):
         int_list = self.space.listview_int(w_ob)
         if int_list is not None:
-            overflowed = misc.pack_list_to_raw_array_bounds(
-                int_list, cdata, self.size, r_uint(0), self.vrangemax)
+            overflowed = misc.pack_list_to_raw_array_bounds_unsigned(
+                int_list, cdata, self.size, self.vrangemax)
             if overflowed != 0:
                 self._overflow(self.space.wrap(overflowed))
             return True
diff --git a/pypy/module/_cffi_backend/misc.py b/pypy/module/_cffi_backend/misc.py
--- a/pypy/module/_cffi_backend/misc.py
+++ b/pypy/module/_cffi_backend/misc.py
@@ -222,6 +222,19 @@
 neg_msg = "can't convert negative number to unsigned"
 ovf_msg = "long too big to convert"
 
+ at specialize.arg(1)
+def signext(value, size):
+    # 'value' is sign-extended from 'size' bytes to a full integer.
+    # 'size' should be a constant smaller than a full integer size.
+    if size == rffi.sizeof(rffi.SIGNEDCHAR):
+        return rffi.cast(lltype.Signed, rffi.cast(rffi.SIGNEDCHAR, value))
+    elif size == rffi.sizeof(rffi.SHORT):
+        return rffi.cast(lltype.Signed, rffi.cast(rffi.SHORT, value))
+    elif size == rffi.sizeof(rffi.INT):
+        return rffi.cast(lltype.Signed, rffi.cast(rffi.INT, value))
+    else:
+        raise AssertionError("unsupported size")
+
 # ____________________________________________________________
 
 class _NotStandardObject(Exception):
@@ -339,13 +352,26 @@
 
 # ____________________________________________________________
 
-def pack_list_to_raw_array_bounds(int_list, target, size, vmin, vrangemax):
+def pack_list_to_raw_array_bounds_signed(int_list, target, size):
     for TP, TPP in _prim_signed_types:
         if size == rffi.sizeof(TP):
             ptr = rffi.cast(TPP, target)
             for i in range(len(int_list)):
                 x = int_list[i]
-                if r_uint(x) - vmin > vrangemax:
+                y = rffi.cast(TP, x)
+                if x != rffi.cast(lltype.Signed, y):
+                    return x      # overflow
+                ptr[i] = y
+            return 0
+    raise NotImplementedError("bad integer size")
+
+def pack_list_to_raw_array_bounds_unsigned(int_list, target, size, vrangemax):
+    for TP, TPP in _prim_signed_types:
+        if size == rffi.sizeof(TP):
+            ptr = rffi.cast(TPP, target)
+            for i in range(len(int_list)):
+                x = int_list[i]
+                if r_uint(x) > vrangemax:
                     return x      # overflow
                 ptr[i] = rffi.cast(TP, x)
             return 0
diff --git a/pypy/module/_ssl/thread_lock.py b/pypy/module/_ssl/thread_lock.py
--- a/pypy/module/_ssl/thread_lock.py
+++ b/pypy/module/_ssl/thread_lock.py
@@ -24,12 +24,19 @@
 
 separate_module_source = """
 #include 
+#ifndef _WIN32
+# include 
+#endif
 
 static unsigned int _ssl_locks_count = 0;
 static struct RPyOpaque_ThreadLock *_ssl_locks;
 
 static unsigned long _ssl_thread_id_function(void) {
-    return RPyThreadGetIdent();
+#ifdef _WIN32
+    return (unsigned long)GetCurrentThreadId();
+#else
+    return (unsigned long)pthread_self();
+#endif
 }
 
 static void _ssl_thread_locking_function(int mode, int n, const char *file,
diff --git a/pypy/module/cpyext/src/pythread.c b/pypy/module/cpyext/src/pythread.c
--- a/pypy/module/cpyext/src/pythread.c
+++ b/pypy/module/cpyext/src/pythread.c
@@ -1,11 +1,18 @@
 #include 
+#ifndef _WIN32
+# include 
+#endif
 #include "pythread.h"
 #include "src/thread.h"
 
 long
 PyThread_get_thread_ident(void)
 {
-    return RPyThreadGetIdent();
+#ifdef _WIN32
+    return (long)GetCurrentThreadId();
+#else
+    return (long)pthread_self();
+#endif
 }
 
 PyThread_type_lock
diff --git a/pypy/module/micronumpy/concrete.py b/pypy/module/micronumpy/concrete.py
--- a/pypy/module/micronumpy/concrete.py
+++ b/pypy/module/micronumpy/concrete.py
@@ -47,6 +47,7 @@
     def setitem(self, index, value):
         self.dtype.itemtype.store(self, index, 0, value)
 
+    @jit.unroll_safe
     def setslice(self, space, arr):
         if len(arr.get_shape()) > 0 and len(self.get_shape()) == 0:
             raise oefmt(space.w_ValueError,
diff --git a/pypy/module/micronumpy/iterators.py b/pypy/module/micronumpy/iterators.py
--- a/pypy/module/micronumpy/iterators.py
+++ b/pypy/module/micronumpy/iterators.py
@@ -154,7 +154,7 @@
         index = state.index
         if self.track_index:
             index += 1
-        indices = state.indices
+        indices = state.indices[:]
         offset = state.offset
         if self.contiguous:
             offset += self.array.dtype.elsize
diff --git a/pypy/module/micronumpy/loop.py b/pypy/module/micronumpy/loop.py
--- a/pypy/module/micronumpy/loop.py
+++ b/pypy/module/micronumpy/loop.py
@@ -88,6 +88,21 @@
                                 reds = 'auto')
 
 def setslice(space, shape, target, source):
+    if not shape:
+        # XXX - simplify
+        target_iter, target_state = target.create_iter(shape)
+        source_iter, source_state = source.create_iter(shape)
+        dtype = target.dtype
+        val = source_iter.getitem(source_state)
+        if dtype.is_str_or_unicode():
+            val = dtype.coerce(space, val)
+        else:
+            val = val.convert_to(space, dtype)
+        target_iter.setitem(target_state, val)
+        return target        
+    return _setslice(space, shape, target, source)
+
+def _setslice(space, shape, target, source):
     # note that unlike everything else, target and source here are
     # array implementations, not arrays
     target_iter, target_state = target.create_iter(shape)
diff --git a/pypy/module/micronumpy/nditer.py b/pypy/module/micronumpy/nditer.py
--- a/pypy/module/micronumpy/nditer.py
+++ b/pypy/module/micronumpy/nditer.py
@@ -1,3 +1,4 @@
+from rpython.rlib import jit
 from pypy.interpreter.baseobjspace import W_Root
 from pypy.interpreter.typedef import TypeDef, GetSetProperty
 from pypy.interpreter.gateway import interp2app, unwrap_spec, WrappedDefault
@@ -264,8 +265,8 @@
         self.index = [0] * len(shape)
         self.backward = backward
 
+    @jit.unroll_safe
     def next(self):
-        # TODO It's probably possible to refactor all the "next" method from each iterator
         for i in range(len(self.shape) - 1, -1, -1):
             if self.index[i] < self.shape[i] - 1:
                 self.index[i] += 1
diff --git a/pypy/module/micronumpy/test/test_zjit.py b/pypy/module/micronumpy/test/test_zjit.py
--- a/pypy/module/micronumpy/test/test_zjit.py
+++ b/pypy/module/micronumpy/test/test_zjit.py
@@ -127,12 +127,13 @@
         assert result == 3 ** 2
         self.check_trace_count(1)
         self.check_simple_loop({
-            'call': 3,
+            'call': 1,
             'float_add': 1,
             'float_eq': 3,
             'float_mul': 2,
             'float_ne': 1,
             'getarrayitem_gc': 1,
+            'getarrayitem_raw': 1,     # read the errno
             'guard_false': 4,
             'guard_not_invalidated': 1,
             'guard_true': 3,
@@ -144,6 +145,7 @@
             'raw_load': 2,
             'raw_store': 1,
             'setarrayitem_gc': 1,
+            'setarrayitem_raw': 1,     # write the errno
         })
 
     def define_pow_int():
diff --git a/pypy/module/pypyjit/test_pypy_c/model.py b/pypy/module/pypyjit/test_pypy_c/model.py
--- a/pypy/module/pypyjit/test_pypy_c/model.py
+++ b/pypy/module/pypyjit/test_pypy_c/model.py
@@ -184,10 +184,10 @@
         matcher = OpMatcher(ops)
         return matcher.match(expected_src, **kwds)
 
-    def match_by_id(self, id, expected_src, **kwds):
+    def match_by_id(self, id, expected_src, ignore_ops=[], **kwds):
         ops = list(self.ops_by_id(id, **kwds))
         matcher = OpMatcher(ops, id)
-        return matcher.match(expected_src)
+        return matcher.match(expected_src, ignore_ops=ignore_ops)
 
 class PartialTraceWithIds(TraceWithIds):
     def __init__(self, trace, is_entry_bridge=False):
diff --git a/pypy/module/pypyjit/test_pypy_c/test_call.py b/pypy/module/pypyjit/test_pypy_c/test_call.py
--- a/pypy/module/pypyjit/test_pypy_c/test_call.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_call.py
@@ -82,7 +82,7 @@
         assert log.opnames(ops) == []
         #
         assert entry_bridge.match_by_id('call', """
-            p38 = call(ConstClass(_ll_0_threadlocalref_getter___), descr=)
+            p38 = call(ConstClass(_ll_1_threadlocalref_get__Ptr_GcStruct_objectLlT_Signed), #, descr=)
             p39 = getfield_gc(p38, descr=)
             i40 = force_token()
             p41 = getfield_gc_pure(p38, descr=)
@@ -444,7 +444,7 @@
             p26 = getfield_gc(p7, descr=)
             guard_value(p26, ConstPtr(ptr27), descr=...)
             guard_not_invalidated(descr=...)
-            p29 = call(ConstClass(_ll_0_threadlocalref_getter___), descr=)
+            p29 = call(ConstClass(_ll_1_threadlocalref_get__Ptr_GcStruct_objectLlT_Signed), #, descr=)
             p30 = getfield_gc(p29, descr=)
             p31 = force_token()
             p32 = getfield_gc_pure(p29, descr=)
diff --git a/pypy/module/pypyjit/test_pypy_c/test_ffi.py b/pypy/module/pypyjit/test_pypy_c/test_ffi.py
--- a/pypy/module/pypyjit/test_pypy_c/test_ffi.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_ffi.py
@@ -199,21 +199,16 @@
         ldexp_addr, res = log.result
         assert res == 8.0 * 300
         loop, = log.loops_by_filename(self.filepath)
-        if 'ConstClass(ldexp)' in repr(loop):   # e.g. OS/X
-            ldexp_addr = 'ConstClass(ldexp)'
         assert loop.match_by_id('cfficall', """
-            ...
-            f1 = call_release_gil(..., descr=)
-            ...
-        """)
-        ops = loop.ops_by_id('cfficall')
-        for name in ['raw_malloc', 'raw_free']:
-            assert name not in str(ops)
-        for name in ['raw_load', 'raw_store', 'getarrayitem_raw', 'setarrayitem_raw']:
-            assert name not in log.opnames(ops)
-        # so far just check that call_release_gil() is produced.
-        # later, also check that the arguments to call_release_gil()
-        # are constants
+            setarrayitem_raw(i69, 0, i95, descr=)    # write 'errno'
+            p96 = force_token()
+            setfield_gc(p0, p96, descr=)
+            f97 = call_release_gil(i59, 1.0, 3, descr=)
+            guard_not_forced(descr=...)
+            guard_no_exception(descr=...)
+            i98 = getarrayitem_raw(i69, 0, descr=)   # read 'errno'
+            setfield_gc(p65, i98, descr=)
+        """, ignore_ops=['guard_not_invalidated'])
 
     def test_cffi_call_guard_not_forced_fails(self):
         # this is the test_pypy_c equivalent of
@@ -340,18 +335,16 @@
         guard_value(p166, ConstPtr(ptr72), descr=...)
         p167 = call(ConstClass(_ll_0_alloc_with_del___), descr=)
         guard_no_exception(descr=...)
-        i112 = int_sub(i160, -32768)
+        i112 = int_signext(i160, 2)
         setfield_gc(p167, ConstPtr(null), descr=)
         setfield_gc(p167, ConstPtr(ptr85), descr=)
-        i114 = uint_gt(i112, 65535)
+        i114 = int_ne(i160, i112)
         guard_false(i114, descr=...)
-        i115 = int_and(i112, 65535)
-        i116 = int_add(i115, -32768)
         --TICK--
         i119 = call(ConstClass(_ll_1_raw_malloc_varsize__Signed), 6, descr=)
-        raw_store(i119, 0, i116, descr=)
-        raw_store(i119, 2, i116, descr=)
-        raw_store(i119, 4, i116, descr=)
+        raw_store(i119, 0, i112, descr=)
+        raw_store(i119, 2, i112, descr=)
+        raw_store(i119, 4, i112, descr=)
         setfield_gc(p167, i119, descr=)
         i123 = arraylen_gc(p67, descr=)
         jump(..., descr=...)
diff --git a/rpython/annotator/bookkeeper.py b/rpython/annotator/bookkeeper.py
--- a/rpython/annotator/bookkeeper.py
+++ b/rpython/annotator/bookkeeper.py
@@ -65,6 +65,7 @@
         self.external_class_cache = {}      # cache of ExternalType classes
 
         self.needs_generic_instantiate = {}
+        self.thread_local_fields = set()
 
         delayed_imports()
 
diff --git a/rpython/jit/backend/arm/assembler.py b/rpython/jit/backend/arm/assembler.py
--- a/rpython/jit/backend/arm/assembler.py
+++ b/rpython/jit/backend/arm/assembler.py
@@ -497,9 +497,11 @@
         if self.cpu.supports_floats:
             mc.VPOP([reg.value for reg in r.callee_saved_vfp_registers],
                                                                     cond=cond)
-        # pop all callee saved registers and IP to keep the alignment
+        # pop all callee saved registers.  This pops 'pc' last.
+        # It also pops the threadlocal_addr back into 'r1', but it
+        # is not needed any more and will be discarded.
         mc.POP([reg.value for reg in r.callee_restored_registers] +
-                                                       [r.ip.value], cond=cond)
+                                                       [r.r1.value], cond=cond)
         mc.BKPT()
 
     def gen_func_prolog(self):
@@ -508,11 +510,16 @@
         if self.cpu.supports_floats:
             stack_size += len(r.callee_saved_vfp_registers) * 2 * WORD
 
-        # push all callee saved registers and IP to keep the alignment
+        # push all callee saved registers including lr; and push r1 as
+        # well, which contains the threadlocal_addr argument.  Note that
+        # we're pushing a total of 10 words, which keeps the stack aligned.
         self.mc.PUSH([reg.value for reg in r.callee_saved_registers] +
-                                                        [r.ip.value])
+                                                        [r.r1.value])
+        self.saved_threadlocal_addr = 0   # at offset 0 from location 'sp'
         if self.cpu.supports_floats:
             self.mc.VPUSH([reg.value for reg in r.callee_saved_vfp_registers])
+            self.saved_threadlocal_addr += (
+                len(r.callee_saved_vfp_registers) * 2 * WORD)
         assert stack_size % 8 == 0 # ensure we keep alignment
 
         # set fp to point to the JITFRAME
@@ -952,16 +959,11 @@
             regalloc._check_invariants()
         self.mc.mark_op(None)  # end of the loop
 
-    def regalloc_emit_llong(self, op, arglocs, fcond, regalloc):
+    def regalloc_emit_extra(self, op, arglocs, fcond, regalloc):
+        # for calls to a function with a specifically-supported OS_xxx
         effectinfo = op.getdescr().get_extra_info()
         oopspecindex = effectinfo.oopspecindex
-        asm_llong_operations[oopspecindex](self, op, arglocs, regalloc, fcond)
-        return fcond
-
-    def regalloc_emit_math(self, op, arglocs, fcond, regalloc):
-        effectinfo = op.getdescr().get_extra_info()
-        oopspecindex = effectinfo.oopspecindex
-        asm_math_operations[oopspecindex](self, op, arglocs, regalloc, fcond)
+        asm_extra_operations[oopspecindex](self, op, arglocs, regalloc, fcond)
         return fcond
 
     def patch_trace(self, faildescr, looptoken, bridge_addr, regalloc):
@@ -1150,6 +1152,14 @@
         else:
             assert 0, 'unsupported case'
 
+    def _mov_raw_sp_to_loc(self, prev_loc, loc, cond=c.AL):
+        if loc.is_core_reg():
+            # load a value from 'SP + n'
+            assert prev_loc.value <= 0xFFF     # not too far
+            self.load_reg(self.mc, loc, r.sp, prev_loc.value, cond=cond)
+        else:
+            assert 0, 'unsupported case'
+
     def regalloc_mov(self, prev_loc, loc, cond=c.AL):
         """Moves a value from a previous location to some other location"""
         if prev_loc.is_imm():
@@ -1163,7 +1173,7 @@
         elif prev_loc.is_vfp_reg():
             self._mov_vfp_reg_to_loc(prev_loc, loc, cond)
         elif prev_loc.is_raw_sp():
-            assert 0, 'raw sp locs are not supported as source loc'
+            self._mov_raw_sp_to_loc(prev_loc, loc, cond)
         else:
             assert 0, 'unsupported case'
     mov_loc_loc = regalloc_mov
@@ -1509,22 +1519,17 @@
 
 asm_operations = [notimplemented_op] * (rop._LAST + 1)
 asm_operations_with_guard = [notimplemented_op_with_guard] * (rop._LAST + 1)
-asm_llong_operations = {}
-asm_math_operations = {}
+asm_extra_operations = {}
 
 for name, value in ResOpAssembler.__dict__.iteritems():
     if name.startswith('emit_guard_'):
         opname = name[len('emit_guard_'):]
         num = getattr(rop, opname.upper())
         asm_operations_with_guard[num] = value
-    elif name.startswith('emit_op_llong_'):
-        opname = name[len('emit_op_llong_'):]
-        num = getattr(EffectInfo, 'OS_LLONG_' + opname.upper())
-        asm_llong_operations[num] = value
-    elif name.startswith('emit_op_math_'):
-        opname = name[len('emit_op_math_'):]
-        num = getattr(EffectInfo, 'OS_MATH_' + opname.upper())
-        asm_math_operations[num] = value
+    elif name.startswith('emit_opx_'):
+        opname = name[len('emit_opx_'):]
+        num = getattr(EffectInfo, 'OS_' + opname.upper())
+        asm_extra_operations[num] = value
     elif name.startswith('emit_op_'):
         opname = name[len('emit_op_'):]
         num = getattr(rop, opname.upper())
diff --git a/rpython/jit/backend/arm/codebuilder.py b/rpython/jit/backend/arm/codebuilder.py
--- a/rpython/jit/backend/arm/codebuilder.py
+++ b/rpython/jit/backend/arm/codebuilder.py
@@ -318,6 +318,18 @@
                     | (rd & 0xF) << 12
                     | imm16 & 0xFFF)
 
+    def SXTB_rr(self, rd, rm, c=cond.AL):
+        self.write32(c << 28
+                    | 0x06AF0070
+                    | (rd & 0xF) << 12
+                    | (rm & 0xF))
+
+    def SXTH_rr(self, rd, rm, c=cond.AL):
+        self.write32(c << 28
+                    | 0x06BF0070
+                    | (rd & 0xF) << 12
+                    | (rm & 0xF))
+
     def LDREX(self, rt, rn, c=cond.AL):
         self.write32(c << 28
                     | 0x01900f9f
diff --git a/rpython/jit/backend/arm/locations.py b/rpython/jit/backend/arm/locations.py
--- a/rpython/jit/backend/arm/locations.py
+++ b/rpython/jit/backend/arm/locations.py
@@ -46,7 +46,7 @@
     def is_core_reg(self):
         return True
 
-    def as_key(self):
+    def as_key(self):       # 0 <= as_key <= 15
         return self.value
 
 
@@ -64,7 +64,7 @@
     def is_vfp_reg(self):
         return True
 
-    def as_key(self):
+    def as_key(self):            # 20 <= as_key <= 35
         return self.value + 20
 
     def is_float(self):
@@ -115,8 +115,8 @@
     def is_imm_float(self):
         return True
 
-    def as_key(self):
-        return self.value
+    def as_key(self):          # a real address + 1
+        return self.value | 1
 
     def is_float(self):
         return True
@@ -148,7 +148,7 @@
     def is_stack(self):
         return True
 
-    def as_key(self):
+    def as_key(self):                # an aligned word + 10000
         return self.position + 10000
 
     def is_float(self):
@@ -174,6 +174,9 @@
     def is_float(self):
         return self.type == FLOAT
 
+    def as_key(self):            # a word >= 1000, and < 1000 + size of SP frame
+        return self.value + 1000
+
 
 def imm(i):
     return ImmLocation(i)
diff --git a/rpython/jit/backend/arm/opassembler.py b/rpython/jit/backend/arm/opassembler.py
--- a/rpython/jit/backend/arm/opassembler.py
+++ b/rpython/jit/backend/arm/opassembler.py
@@ -19,7 +19,7 @@
 from rpython.jit.backend.arm.codebuilder import InstrBuilder, OverwritingBuilder
 from rpython.jit.backend.arm.jump import remap_frame_layout
 from rpython.jit.backend.arm.regalloc import TempBox
-from rpython.jit.backend.arm.locations import imm
+from rpython.jit.backend.arm.locations import imm, RawSPStackLocation
 from rpython.jit.backend.llsupport import symbolic
 from rpython.jit.backend.llsupport.gcmap import allocate_gcmap
 from rpython.jit.backend.llsupport.descr import InteriorFieldDescr
@@ -102,6 +102,17 @@
         self.mc.MOV_rr(res.value, arg.value, cond=c.GE)
         return fcond
 
+    def emit_op_int_signext(self, op, arglocs, regalloc, fcond):
+        arg, numbytes, res = arglocs
+        assert numbytes.is_imm()
+        if numbytes.value == 1:
+            self.mc.SXTB_rr(res.value, arg.value)
+        elif numbytes.value == 2:
+            self.mc.SXTH_rr(res.value, arg.value)
+        else:
+            raise AssertionError("bad number of bytes")
+        return fcond
+
     #ref: http://blogs.arm.com/software-enablement/detecting-overflow-from-mul/
     def emit_guard_int_mul_ovf(self, op, guard, arglocs, regalloc, fcond):
         reg1 = arglocs[0]
@@ -971,7 +982,9 @@
         return fcond
 
     def _call_assembler_emit_call(self, addr, argloc, resloc):
-        self.simple_call(addr, [argloc], result_loc=resloc)
+        ofs = self.saved_threadlocal_addr
+        threadlocal_loc = RawSPStackLocation(ofs, INT)
+        self.simple_call(addr, [argloc, threadlocal_loc], result_loc=resloc)
 
     def _call_assembler_emit_helper_call(self, addr, arglocs, resloc):
         self.simple_call(addr, arglocs, result_loc=resloc)
@@ -1097,7 +1110,7 @@
 
     emit_op_float_neg = gen_emit_unary_float_op('float_neg', 'VNEG')
     emit_op_float_abs = gen_emit_unary_float_op('float_abs', 'VABS')
-    emit_op_math_sqrt = gen_emit_unary_float_op('math_sqrt', 'VSQRT')
+    emit_opx_math_sqrt = gen_emit_unary_float_op('math_sqrt', 'VSQRT')
 
     emit_op_float_lt = gen_emit_float_cmp_op('float_lt', c.VFP_LT)
     emit_op_float_le = gen_emit_float_cmp_op('float_le', c.VFP_LE)
@@ -1131,13 +1144,13 @@
 
     # the following five instructions are only ARMv7;
     # regalloc.py won't call them at all on ARMv6
-    emit_op_llong_add = gen_emit_float_op('llong_add', 'VADD_i64')
-    emit_op_llong_sub = gen_emit_float_op('llong_sub', 'VSUB_i64')
-    emit_op_llong_and = gen_emit_float_op('llong_and', 'VAND_i64')
-    emit_op_llong_or = gen_emit_float_op('llong_or', 'VORR_i64')
-    emit_op_llong_xor = gen_emit_float_op('llong_xor', 'VEOR_i64')
+    emit_opx_llong_add = gen_emit_float_op('llong_add', 'VADD_i64')
+    emit_opx_llong_sub = gen_emit_float_op('llong_sub', 'VSUB_i64')
+    emit_opx_llong_and = gen_emit_float_op('llong_and', 'VAND_i64')
+    emit_opx_llong_or = gen_emit_float_op('llong_or', 'VORR_i64')
+    emit_opx_llong_xor = gen_emit_float_op('llong_xor', 'VEOR_i64')
 
-    def emit_op_llong_to_int(self, op, arglocs, regalloc, fcond):
+    def emit_opx_llong_to_int(self, op, arglocs, regalloc, fcond):
         loc = arglocs[0]
         res = arglocs[1]
         assert loc.is_vfp_reg()
@@ -1271,3 +1284,11 @@
             regalloc.rm.possibly_free_var(length_box)
         regalloc.rm.possibly_free_var(dstaddr_box)
         return fcond
+
+    def emit_opx_threadlocalref_get(self, op, arglocs, regalloc, fcond):
+        ofs0, res = arglocs
+        assert ofs0.is_imm()
+        ofs = self.saved_threadlocal_addr
+        self.load_reg(self.mc, res, r.sp, ofs)
+        self.load_reg(self.mc, res, res, ofs0.value)
+        return fcond
diff --git a/rpython/jit/backend/arm/regalloc.py b/rpython/jit/backend/arm/regalloc.py
--- a/rpython/jit/backend/arm/regalloc.py
+++ b/rpython/jit/backend/arm/regalloc.py
@@ -373,11 +373,8 @@
         return gcmap
 
     # ------------------------------------------------------------
-    def perform_llong(self, op, args, fcond):
-        return self.assembler.regalloc_emit_llong(op, args, fcond, self)
-
-    def perform_math(self, op, args, fcond):
-        return self.assembler.regalloc_emit_math(op, args, self, fcond)
+    def perform_extra(self, op, args, fcond):
+        return self.assembler.regalloc_emit_extra(op, args, fcond, self)
 
     def force_spill_var(self, var):
         if var.type == FLOAT:
@@ -458,6 +455,12 @@
         resloc = self.force_allocate_reg(op.result, [op.getarg(0)])
         return [argloc, resloc]
 
+    def prepare_op_int_signext(self, op, fcond):
+        argloc = self.make_sure_var_in_reg(op.getarg(0))
+        numbytes = op.getarg(1).getint()
+        resloc = self.force_allocate_reg(op.result)
+        return [argloc, imm(numbytes), resloc]
+
     def prepare_guard_int_mul_ovf(self, op, guard, fcond):
         boxes = op.getarglist()
         reg1 = self.make_sure_var_in_reg(boxes[0], forbidden_vars=boxes)
@@ -552,15 +555,19 @@
                             EffectInfo.OS_LLONG_XOR):
                 if self.cpu.cpuinfo.arch_version >= 7:
                     args = self._prepare_llong_binop_xx(op, fcond)
-                    self.perform_llong(op, args, fcond)
+                    self.perform_extra(op, args, fcond)
                     return
             elif oopspecindex == EffectInfo.OS_LLONG_TO_INT:
                 args = self._prepare_llong_to_int(op, fcond)
-                self.perform_llong(op, args, fcond)
+                self.perform_extra(op, args, fcond)
                 return
             elif oopspecindex == EffectInfo.OS_MATH_SQRT:
-                args = self.prepare_op_math_sqrt(op, fcond)
-                self.perform_math(op, args, fcond)
+                args = self._prepare_op_math_sqrt(op, fcond)
+                self.perform_extra(op, args, fcond)
+                return
+            elif oopspecindex == EffectInfo.OS_THREADLOCALREF_GET:
+                args = self._prepare_threadlocalref_get(op, fcond)
+                self.perform_extra(op, args, fcond)
                 return
             #elif oopspecindex == EffectInfo.OS_MATH_READ_TIMESTAMP:
             #    ...
@@ -618,6 +625,11 @@
         res = self.force_allocate_reg(op.result)
         return [loc0, res]
 
+    def _prepare_threadlocalref_get(self, op, fcond):
+        ofs0 = imm(op.getarg(1).getint())
+        res = self.force_allocate_reg(op.result)
+        return [ofs0, res]
+
     def _prepare_guard(self, op, args=None):
         if args is None:
             args = []
@@ -1278,7 +1290,7 @@
     prepare_guard_float_ge = prepare_float_op(guard=True,
                             float_result=False, name='prepare_guard_float_ge')
 
-    def prepare_op_math_sqrt(self, op, fcond):
+    def _prepare_op_math_sqrt(self, op, fcond):
         loc = self.make_sure_var_in_reg(op.getarg(1))
         self.possibly_free_vars_for_op(op)
         self.free_temp_vars()
diff --git a/rpython/jit/backend/llsupport/llmodel.py b/rpython/jit/backend/llsupport/llmodel.py
--- a/rpython/jit/backend/llsupport/llmodel.py
+++ b/rpython/jit/backend/llsupport/llmodel.py
@@ -217,7 +217,13 @@
         return lltype.cast_opaque_ptr(llmemory.GCREF, frame)
 
     def make_execute_token(self, *ARGS):
-        FUNCPTR = lltype.Ptr(lltype.FuncType([llmemory.GCREF],
+        # The JIT backend must generate functions with the following
+        # signature: it takes the jitframe and the threadlocal_addr
+        # as arguments, and it returns the (possibly reallocated) jitframe.
+        # The backend can optimize OS_THREADLOCALREF_GET calls to return a
+        # field of this threadlocal_addr, but only if 'translate_support_code':
+        # in untranslated tests, threadlocal_addr is a dummy NULL.
+        FUNCPTR = lltype.Ptr(lltype.FuncType([llmemory.GCREF, llmemory.Address],
                                              llmemory.GCREF))
 
         lst = [(i, history.getkind(ARG)[0]) for i, ARG in enumerate(ARGS)]
@@ -249,8 +255,13 @@
                     else:
                         assert kind == history.REF
                         self.set_ref_value(ll_frame, num, arg)
+                if self.translate_support_code:
+                    ll_threadlocal_addr = llop.threadlocalref_addr(
+                        llmemory.Address)
+                else:
+                    ll_threadlocal_addr = llmemory.NULL
                 llop.gc_writebarrier(lltype.Void, ll_frame)
-                ll_frame = func(ll_frame)
+                ll_frame = func(ll_frame, ll_threadlocal_addr)
             finally:
                 if not self.translate_support_code:
                     LLInterpreter.current_interpreter = prev_interpreter
diff --git a/rpython/jit/backend/llsupport/test/ztranslation_test.py b/rpython/jit/backend/llsupport/test/ztranslation_test.py
--- a/rpython/jit/backend/llsupport/test/ztranslation_test.py
+++ b/rpython/jit/backend/llsupport/test/ztranslation_test.py
@@ -26,8 +26,6 @@
         # - profiler
         # - full optimizer
         # - floats neg and abs
-        # - threadlocalref_get
-        # - get_errno, set_errno
         # - llexternal with macro=True
 
         class Frame(object):
@@ -36,10 +34,6 @@
             def __init__(self, i):
                 self.i = i
 
-        class Foo(object):
-            pass
-        t = ThreadLocalReference(Foo)
-
         eci = ExternalCompilationInfo(post_include_bits=['''
 #define pypy_my_fabs(x)  fabs(x)
 '''])
@@ -74,9 +68,6 @@
                 k = myabs1(myabs2(j))
                 if k - abs(j):  raise ValueError
                 if k - abs(-j): raise ValueError
-                if t.get().nine != 9: raise ValueError
-                rposix.set_errno(total)
-                if rposix.get_errno() != total: raise ValueError
             return chr(total % 253)
         #
         class Virt2(object):
@@ -104,12 +95,8 @@
             return res
         #
         def main(i, j):
-            foo = Foo()
-            foo.nine = -(i + j)
-            t.set(foo)
             a_char = f(i, j)
             a_float = libffi_stuff(i, j)
-            keepalive_until_here(foo)
             return ord(a_char) * 10 + int(a_float)
         expected = main(40, -49)
         res = self.meta_interp(main, [40, -49])
@@ -121,6 +108,7 @@
 
     def test_direct_assembler_call_translates(self):
         """Test CALL_ASSEMBLER and the recursion limit"""
+        # - also tests threadlocalref_get
         from rpython.rlib.rstackovf import StackOverflow
 
         class Thing(object):
@@ -138,6 +126,10 @@
 
         somewhere_else = SomewhereElse()
 
+        class Foo(object):
+            pass
+        t = ThreadLocalReference(Foo)
+
         def change(newthing):
             somewhere_else.frame.thing = newthing
 
@@ -163,6 +155,7 @@
                     nextval = 13
                 frame.thing = Thing(nextval + 1)
                 i += 1
+                if t.get().nine != 9: raise ValueError
             return frame.thing.val
 
         driver2 = JitDriver(greens = [], reds = ['n'])
@@ -184,13 +177,24 @@
                 n = portal2(n)
         assert portal2(10) == -9
 
+        def setup(value):
+            foo = Foo()
+            foo.nine = value
+            t.set(foo)
+            return foo
+
         def mainall(codeno, bound):
-            return main(codeno) + main2(bound)
+            foo = setup(bound + 8)
+            result = main(codeno) + main2(bound)
+            keepalive_until_here(foo)
+            return result
 
+        tmp_obj = setup(9)
+        expected_1 = main(0)
         res = self.meta_interp(mainall, [0, 1], inline=True,
                                policy=StopAtXPolicy(change))
         print hex(res)
-        assert res & 255 == main(0)
+        assert res & 255 == expected_1
         bound = res & ~255
         assert 1024 <= bound <= 131072
         assert bound & (bound-1) == 0       # a power of two
diff --git a/rpython/jit/backend/test/runner_test.py b/rpython/jit/backend/test/runner_test.py
--- a/rpython/jit/backend/test/runner_test.py
+++ b/rpython/jit/backend/test/runner_test.py
@@ -3890,6 +3890,26 @@
             deadframe = self.cpu.execute_token(looptoken, inp)
             assert outp == self.cpu.get_int_value(deadframe, 0)
 
+    def test_int_signext(self):
+        numbytes_cases = [1, 2] if IS_32_BIT else [1, 2, 4]
+        for numbytes in numbytes_cases:
+            ops = """
+            [i0]
+            i1 = int_signext(i0, %d)
+            finish(i1, descr=descr)
+            """ % numbytes
+            descr = BasicFinalDescr()
+            loop = parse(ops, self.cpu, namespace=locals())
+            looptoken = JitCellToken()
+            self.cpu.compile_loop(loop.inputargs, loop.operations, looptoken)
+            test_cases = [random.randrange(-sys.maxint-1, sys.maxint+1)
+                          for _ in range(100)]
+            for test_case in test_cases:
+                deadframe = self.cpu.execute_token(looptoken, test_case)
+                got = self.cpu.get_int_value(deadframe, 0)
+                expected = heaptracker.int_signext(test_case, numbytes)
+                assert got == expected
+
     def test_compile_asmlen(self):
         from rpython.jit.backend.llsupport.llmodel import AbstractLLCPU
         if not isinstance(self.cpu, AbstractLLCPU):
diff --git a/rpython/jit/backend/x86/arch.py b/rpython/jit/backend/x86/arch.py
--- a/rpython/jit/backend/x86/arch.py
+++ b/rpython/jit/backend/x86/arch.py
@@ -34,10 +34,16 @@
     FRAME_FIXED_SIZE = 19
     PASS_ON_MY_FRAME = 15
     JITFRAME_FIXED_SIZE = 6 + 8 * 2 # 6 GPR + 8 XMM * 2 WORDS/float
+    # 'threadlocal_addr' is passed as 2nd argument on the stack,
+    # and it can be left here for when it is needed
+    THREADLOCAL_OFS = (FRAME_FIXED_SIZE + 2) * WORD
 else:
-    # rbp + rbx + r12 + r13 + r14 + r15 + 13 extra words = 19
+    # rbp + rbx + r12 + r13 + r14 + r15 + threadlocal + 12 extra words = 19
     FRAME_FIXED_SIZE = 19
-    PASS_ON_MY_FRAME = 13
+    PASS_ON_MY_FRAME = 12
     JITFRAME_FIXED_SIZE = 28 # 13 GPR + 15 XMM
+    # 'threadlocal_addr' is passed as 2nd argument in %esi,
+    # and is moved into this frame location
+    THREADLOCAL_OFS = (FRAME_FIXED_SIZE - 1) * WORD
 
 assert PASS_ON_MY_FRAME >= 12       # asmgcc needs at least JIT_USE_WORDS + 3
diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py
--- a/rpython/jit/backend/x86/assembler.py
+++ b/rpython/jit/backend/x86/assembler.py
@@ -18,7 +18,7 @@
 from rpython.jit.backend.llsupport.regalloc import (get_scale, valid_addressing_size)
 from rpython.jit.backend.x86.arch import (FRAME_FIXED_SIZE, WORD, IS_X86_64,
                                        JITFRAME_FIXED_SIZE, IS_X86_32,
-                                       PASS_ON_MY_FRAME)
+                                       PASS_ON_MY_FRAME, THREADLOCAL_OFS)
 from rpython.jit.backend.x86.regloc import (eax, ecx, edx, ebx, esp, ebp, esi,
     xmm0, xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7, r8, r9, r10, r11, edi,
     r12, r13, r14, r15, X86_64_SCRATCH_REG, X86_64_XMM_SCRATCH_REG,
@@ -730,6 +730,7 @@
         self.mc.SUB_ri(esp.value, FRAME_FIXED_SIZE * WORD)
         self.mc.MOV_sr(PASS_ON_MY_FRAME * WORD, ebp.value)
         if IS_X86_64:
+            self.mc.MOV_sr(THREADLOCAL_OFS, esi.value)
             self.mc.MOV_rr(ebp.value, edi.value)
         else:
             self.mc.MOV_rs(ebp.value, (FRAME_FIXED_SIZE + 1) * WORD)
@@ -1143,6 +1144,18 @@
     def genop_math_sqrt(self, op, arglocs, resloc):
         self.mc.SQRTSD(arglocs[0], resloc)
 
+    def genop_int_signext(self, op, arglocs, resloc):
+        argloc, numbytesloc = arglocs
+        assert isinstance(numbytesloc, ImmedLoc)
+        if numbytesloc.value == 1:
+            self.mc.MOVSX8(resloc, argloc)
+        elif numbytesloc.value == 2:
+            self.mc.MOVSX16(resloc, argloc)
+        elif IS_X86_64 and numbytesloc.value == 4:
+            self.mc.MOVSX32(resloc, argloc)
+        else:
+            raise AssertionError("bad number of bytes")
+
     def genop_guard_float_ne(self, op, guard_op, guard_token, arglocs, result_loc):
         guard_opnum = guard_op.getopnum()
         if isinstance(arglocs[0], RegLoc):
@@ -1957,7 +1970,8 @@
         self._emit_guard_not_forced(guard_token)
 
     def _call_assembler_emit_call(self, addr, argloc, _):
-        self.simple_call(addr, [argloc])
+        threadlocal_loc = RawEspLoc(THREADLOCAL_OFS, INT)
+        self.simple_call(addr, [argloc, threadlocal_loc])
 
     def _call_assembler_emit_helper_call(self, addr, arglocs, result_loc):
         self.simple_call(addr, arglocs, result_loc)
@@ -2322,48 +2336,16 @@
         assert isinstance(reg, RegLoc)
         self.mc.MOV_rr(reg.value, ebp.value)
 
-    def threadlocalref_get(self, op, resloc):
-        # this function is only called on Linux
-        from rpython.jit.codewriter.jitcode import ThreadLocalRefDescr
-        from rpython.jit.backend.x86 import stmtlocal
+    def threadlocalref_get(self, offset, resloc):
+        # This loads the stack location THREADLOCAL_OFS into a
+        # register, and then read the word at the given offset.
+        # It is only supported if 'translate_support_code' is
+        # true; otherwise, the original call to the piece of assembler
+        # was done with a dummy NULL value.
+        assert self.cpu.translate_support_code
         assert isinstance(resloc, RegLoc)
-        effectinfo = op.getdescr().get_extra_info()
-        assert effectinfo.extradescrs is not None
-        ed = effectinfo.extradescrs[0]
-        assert isinstance(ed, ThreadLocalRefDescr)
-        addr1 = rffi.cast(lltype.Signed, ed.get_tlref_addr())
-        # 'addr1' is the address is the current thread, but we assume that
-        # it is a thread-local at a constant offset from %fs/%gs.
-        addr0 = stmtlocal.threadlocal_base()
-        addr = addr1 - addr0
-        assert rx86.fits_in_32bits(addr)
-        mc = self.mc
-        mc.writechar(stmtlocal.SEGMENT_TL)     # prefix: %fs or %gs
-        mc.MOV_rj(resloc.value, addr)          # memory read
-
-    def get_set_errno(self, op, loc, issue_a_write):
-        # this function is only called on Linux
-        from rpython.jit.backend.x86 import stmtlocal
-        addr = stmtlocal.get_errno_tl()
-        assert rx86.fits_in_32bits(addr)
-        mc = self.mc
-        mc.writechar(stmtlocal.SEGMENT_TL)     # prefix: %fs or %gs
-        # !!important: the *next* instruction must be the one using 'addr'!!
-        if issue_a_write:
-            if isinstance(loc, RegLoc):
-                mc.MOV32_jr(addr, loc.value)       # memory write from reg
-            else:
-                assert isinstance(loc, ImmedLoc)
-                newvalue = loc.value
-                newvalue = rffi.cast(rffi.INT, newvalue)
-                newvalue = rffi.cast(lltype.Signed, newvalue)
-                mc.MOV32_ji(addr, newvalue)        # memory write immediate
-        else:
-            assert isinstance(loc, RegLoc)
-            if IS_X86_32:
-                mc.MOV_rj(loc.value, addr)         # memory read
-            elif IS_X86_64:
-                mc.MOVSX32_rj(loc.value, addr)     # memory read, sign-extend
+        self.mc.MOV_rs(resloc.value, THREADLOCAL_OFS)
+        self.mc.MOV_rm(resloc.value, (resloc.value, offset))
 
     def genop_discard_zero_array(self, op, arglocs):
         (base_loc, startindex_loc, bytes_loc,
diff --git a/rpython/jit/backend/x86/regalloc.py b/rpython/jit/backend/x86/regalloc.py
--- a/rpython/jit/backend/x86/regalloc.py
+++ b/rpython/jit/backend/x86/regalloc.py
@@ -474,6 +474,12 @@
 
     consider_int_invert = consider_int_neg
 
+    def consider_int_signext(self, op):
+        argloc = self.loc(op.getarg(0))
+        numbytesloc = self.loc(op.getarg(1))
+        resloc = self.force_allocate_reg(op.result)
+        self.perform(op, [argloc, numbytesloc], resloc)
+
     def consider_int_lshift(self, op):
         if isinstance(op.getarg(1), Const):
             loc2 = self.rm.convert_to_imm(op.getarg(1))
@@ -693,29 +699,11 @@
         loc0 = self.xrm.force_result_in_reg(op.result, op.getarg(1))
         self.perform_math(op, [loc0], loc0)
 
-    TLREF_SUPPORT = sys.platform.startswith('linux')
-    ERRNO_SUPPORT = sys.platform.startswith('linux')
-
     def _consider_threadlocalref_get(self, op):
-        if self.TLREF_SUPPORT:
+        if self.translate_support_code:
+            offset = op.getarg(1).getint()   # getarg(0) == 'threadlocalref_get'
             resloc = self.force_allocate_reg(op.result)
-            self.assembler.threadlocalref_get(op, resloc)
-        else:
-            self._consider_call(op)
-
-    def _consider_get_errno(self, op):
-        if self.ERRNO_SUPPORT:
-            resloc = self.force_allocate_reg(op.result)
-            self.assembler.get_set_errno(op, resloc, issue_a_write=False)
-        else:
-            self._consider_call(op)
-
-    def _consider_set_errno(self, op):
-        if self.ERRNO_SUPPORT:
-            # op.getarg(0) is the function set_errno; op.getarg(1) is
-            # the new errno value
-            loc0 = self.rm.make_sure_var_in_reg(op.getarg(1))
-            self.assembler.get_set_errno(op, loc0, issue_a_write=True)
+            self.assembler.threadlocalref_get(offset, resloc)
         else:
             self._consider_call(op)
 
@@ -798,10 +786,6 @@
                 return self._consider_math_sqrt(op)
             if oopspecindex == EffectInfo.OS_THREADLOCALREF_GET:
                 return self._consider_threadlocalref_get(op)
-            if oopspecindex == EffectInfo.OS_GET_ERRNO:
-                return self._consider_get_errno(op)
-            if oopspecindex == EffectInfo.OS_SET_ERRNO:
-                return self._consider_set_errno(op)
             if oopspecindex == EffectInfo.OS_MATH_READ_TIMESTAMP:
                 return self._consider_math_read_timestamp(op)
         self._consider_call(op)
diff --git a/rpython/jit/backend/x86/stmtlocal.py b/rpython/jit/backend/x86/stmtlocal.py
deleted file mode 100644
--- a/rpython/jit/backend/x86/stmtlocal.py
+++ /dev/null
@@ -1,43 +0,0 @@
-from rpython.rtyper.lltypesystem import lltype, rffi
-from rpython.translator.tool.cbuild import ExternalCompilationInfo
-from rpython.jit.backend.x86.arch import WORD
-
-SEGMENT_FS = '\x64'
-SEGMENT_GS = '\x65'
-
-if WORD == 4:
-    SEGMENT_TL = SEGMENT_GS
-    _instruction = "movl %%gs:0, %0"
-else:
-    SEGMENT_TL = SEGMENT_FS
-    _instruction = "movq %%fs:0, %0"
-
-eci = ExternalCompilationInfo(post_include_bits=['''
-#define RPY_STM_JIT  1
-static long pypy__threadlocal_base(void)
-{
-    /* XXX ONLY LINUX WITH GCC/CLANG FOR NOW XXX */
-    long result;
-    asm("%s" : "=r"(result));
-    return result;
-}
-static long pypy__get_errno_tl(void)
-{
-    return ((long)&errno) - pypy__threadlocal_base();
-}
-''' % _instruction])
-
-
-threadlocal_base = rffi.llexternal(
-    'pypy__threadlocal_base',
-    [], lltype.Signed,
-    compilation_info=eci,
-    _nowrapper=True,
-    ) #transactionsafe=True)
-
-get_errno_tl = rffi.llexternal(
-    'pypy__get_errno_tl',
-    [], lltype.Signed,
-    compilation_info=eci,
-    _nowrapper=True,
-    ) #transactionsafe=True)
diff --git a/rpython/jit/codewriter/assembler.py b/rpython/jit/codewriter/assembler.py
--- a/rpython/jit/codewriter/assembler.py
+++ b/rpython/jit/codewriter/assembler.py
@@ -216,10 +216,11 @@
             self.code[pos  ] = chr(target & 0xFF)
             self.code[pos+1] = chr(target >> 8)
         for descr in self.switchdictdescrs:
-            descr.dict = {}
+            as_dict = {}
             for key, switchlabel in descr._labels:
                 target = self.label_positions[switchlabel.name]
-                descr.dict[key] = target
+                as_dict[key] = target
+            descr.attach(as_dict)
 
     def check_result(self):
         # Limitation of the number of registers, from the single-byte encoding
diff --git a/rpython/jit/codewriter/effectinfo.py b/rpython/jit/codewriter/effectinfo.py
--- a/rpython/jit/codewriter/effectinfo.py
+++ b/rpython/jit/codewriter/effectinfo.py
@@ -23,8 +23,6 @@
     OS_SHRINK_ARRAY             = 3    # rgc.ll_shrink_array
     OS_DICT_LOOKUP              = 4    # ll_dict_lookup
     OS_THREADLOCALREF_GET       = 5    # llop.threadlocalref_get
-    OS_GET_ERRNO                = 6    # rposix.get_errno
-    OS_SET_ERRNO                = 7    # rposix.set_errno
     OS_NOT_IN_TRACE             = 8    # for calls not recorded in the jit trace
     #
     OS_STR_CONCAT               = 22   # "stroruni.concat"
diff --git a/rpython/jit/codewriter/flatten.py b/rpython/jit/codewriter/flatten.py
--- a/rpython/jit/codewriter/flatten.py
+++ b/rpython/jit/codewriter/flatten.py
@@ -243,55 +243,39 @@
         else:
             # A switch.
             #
-            def emitdefaultpath():
-                if block.exits[-1].exitcase == 'default':
-                    self.make_link(block.exits[-1])
-                else:
-                    self.emitline("unreachable")
-                    self.emitline("---")
-            #
-            self.emitline('-live-')
             switches = [link for link in block.exits
                         if link.exitcase != 'default']
             switches.sort(key=lambda link: link.llexitcase)
             kind = getkind(block.exitswitch.concretetype)
-            if len(switches) >= 5 and kind == 'int':
-                # A large switch on an integer, implementable efficiently
-                # with the help of a SwitchDictDescr
-                from rpython.jit.codewriter.jitcode import SwitchDictDescr
-                switchdict = SwitchDictDescr()
-                switchdict._labels = []
-                self.emitline('switch', self.getcolor(block.exitswitch),
-                                        switchdict)
-                emitdefaultpath()
-                #
-                for switch in switches:
-                    key = lltype.cast_primitive(lltype.Signed,
-                                                switch.llexitcase)
-                    switchdict._labels.append((key, TLabel(switch)))
-                    # emit code for that path
-                    self.emitline(Label(switch))
-                    self.make_link(switch)
+            assert kind == 'int'    # XXX
             #
+            # A switch on an integer, implementable efficiently with the
+            # help of a SwitchDictDescr.  We use this even if there are
+            # very few cases: in pyjitpl.py, opimpl_switch() will promote
+            # the int only if it matches one of the cases.
+            from rpython.jit.codewriter.jitcode import SwitchDictDescr
+            switchdict = SwitchDictDescr()
+            switchdict._labels = []
+            self.emitline('-live-')    # for 'guard_value'
+            self.emitline('switch', self.getcolor(block.exitswitch),
+                                    switchdict)
+            # emit the default path
+            if block.exits[-1].exitcase == 'default':
+                self.make_link(block.exits[-1])
             else:
-                # A switch with several possible answers, though not too
-                # many of them -- a chain of int_eq comparisons is fine
-                assert kind == 'int'    # XXX
-                color = self.getcolor(block.exitswitch)
-                self.emitline('int_guard_value', color)
-                for switch in switches:
-                    # make the case described by 'switch'
-                    self.emitline('goto_if_not_int_eq',
-                                  color,
-                                  Constant(switch.llexitcase,
-                                           block.exitswitch.concretetype),
-                                  TLabel(switch))
-                    # emit code for the "taken" path
-                    self.make_link(switch)
-                    # finally, emit the label for the "non-taken" path
-                    self.emitline(Label(switch))
-                #
-                emitdefaultpath()
+                self.emitline("unreachable")
+                self.emitline("---")
+            #
+            for switch in switches:
+                key = lltype.cast_primitive(lltype.Signed,
+                                            switch.llexitcase)
+                switchdict._labels.append((key, TLabel(switch)))
+                # emit code for that path
+                # note: we need a -live- for all the 'guard_false' we produce
+                # if the switched value doesn't match any case.
+                self.emitline(Label(switch))
+                self.emitline('-live-')
+                self.make_link(switch)
 
     def insert_renamings(self, link):
         renamings = {}
diff --git a/rpython/jit/codewriter/heaptracker.py b/rpython/jit/codewriter/heaptracker.py
--- a/rpython/jit/codewriter/heaptracker.py
+++ b/rpython/jit/codewriter/heaptracker.py
@@ -1,6 +1,7 @@
 from rpython.rtyper.lltypesystem import lltype, llmemory
 from rpython.rtyper import rclass
 from rpython.rlib.objectmodel import we_are_translated
+from rpython.rlib.rarithmetic import r_uint, intmask
 
 
 def adr2int(addr):
@@ -11,6 +12,14 @@
 def int2adr(int):
     return llmemory.cast_int_to_adr(int)
 
+def int_signext(value, numbytes):
+    b8 = numbytes * 8
+    a = r_uint(value)
+    a += r_uint(1 << (b8 - 1))     # a += 128
+    a &= r_uint((1 << b8) - 1)     # a &= 255
+    a -= r_uint(1 << (b8 - 1))     # a -= 128
+    return intmask(a)
+
 def count_fields_if_immutable(STRUCT):
     assert isinstance(STRUCT, lltype.GcStruct)
     if STRUCT._hints.get('immutable', False):
diff --git a/rpython/jit/codewriter/jitcode.py b/rpython/jit/codewriter/jitcode.py
--- a/rpython/jit/codewriter/jitcode.py
+++ b/rpython/jit/codewriter/jitcode.py
@@ -1,4 +1,4 @@
-from rpython.jit.metainterp.history import AbstractDescr
+from rpython.jit.metainterp.history import AbstractDescr, ConstInt
 from rpython.jit.codewriter import heaptracker
 from rpython.rlib.objectmodel import we_are_translated
 
@@ -109,6 +109,10 @@
 class SwitchDictDescr(AbstractDescr):
     "Get a 'dict' attribute mapping integer values to bytecode positions."
 
+    def attach(self, as_dict):
+        self.dict = as_dict
+        self.const_keys_in_order = map(ConstInt, sorted(as_dict.keys()))
+
     def __repr__(self):
         dict = getattr(self, 'dict', '?')
         return '' % (dict,)
@@ -117,26 +121,6 @@
         raise NotImplementedError
 
 
-class ThreadLocalRefDescr(AbstractDescr):
-    # A special descr used as the extradescr in a call to a
-    # threadlocalref_get function.  If the backend supports it,
-    # it can use this 'get_tlref_addr()' to get the address *in the
-    # current thread* of the thread-local variable.  If, on the current
-    # platform, the "__thread" variables are implemented as an offset
-    # from some base register (e.g. %fs on x86-64), then the backend will
-    # immediately substract the current value of the base register.
-    # This gives an offset from the base register, and this can be
-    # written down in an assembler instruction to load the "__thread"
-    # variable from anywhere.
-
-    def __init__(self, opaque_id):
-        from rpython.rtyper.lltypesystem.lloperation import llop
-        from rpython.rtyper.lltypesystem import llmemory
-        def get_tlref_addr():
-            return llop.threadlocalref_getaddr(llmemory.Address, opaque_id)
-        self.get_tlref_addr = get_tlref_addr
-
-
 class LiveVarsInfo(object):
     def __init__(self, live_i, live_r, live_f):
         self.live_i = live_i
diff --git a/rpython/jit/codewriter/jtransform.py b/rpython/jit/codewriter/jtransform.py
--- a/rpython/jit/codewriter/jtransform.py
+++ b/rpython/jit/codewriter/jtransform.py
@@ -439,8 +439,6 @@
         elif oopspec_name.endswith('dict.lookup'):
             # also ordereddict.lookup
             prepare = self._handle_dict_lookup_call
-        elif oopspec_name.startswith('rposix.'):
-            prepare = self._handle_rposix_call
         else:
             prepare = self.prepare_builtin_call
         try:
@@ -1267,19 +1265,12 @@
 
         result = []
         if min2:
-            c_min2 = Constant(min2, lltype.Signed)
-            v2 = varoftype(lltype.Signed)
-            result.append(SpaceOperation('int_sub', [v_arg, c_min2], v2))
+            c_bytes = Constant(size2, lltype.Signed)
+            result.append(SpaceOperation('int_signext', [v_arg, c_bytes],
+                                         v_result))
         else:
-            v2 = v_arg
-        c_mask = Constant(int((1 << (8 * size2)) - 1), lltype.Signed)
-        if min2:
-            v3 = varoftype(lltype.Signed)
-        else:
-            v3 = v_result
-        result.append(SpaceOperation('int_and', [v2, c_mask], v3))
-        if min2:
-            result.append(SpaceOperation('int_add', [v3, c_min2], v_result))
+            c_mask = Constant(int((1 << (8 * size2)) - 1), lltype.Signed)
+            result.append(SpaceOperation('int_and', [v_arg, c_mask], v_result))
         return result
 
     def _float_to_float_cast(self, v_arg, v_result):
@@ -1986,16 +1977,6 @@
         else:
             raise NotImplementedError(oopspec_name)
 
-    def _handle_rposix_call(self, op, oopspec_name, args):
-        if oopspec_name == 'rposix.get_errno':
-            return self._handle_oopspec_call(op, args, EffectInfo.OS_GET_ERRNO,
-                                             EffectInfo.EF_CANNOT_RAISE)
-        elif oopspec_name == 'rposix.set_errno':
-            return self._handle_oopspec_call(op, args, EffectInfo.OS_SET_ERRNO,
-                                             EffectInfo.EF_CANNOT_RAISE)
-        else:
-            raise NotImplementedError(oopspec_name)
-
     def rewrite_op_ll_read_timestamp(self, op):
         op1 = self.prepare_builtin_call(op, "ll_read_timestamp", [])
         return self.handle_residual_call(op1,
@@ -2012,16 +1993,15 @@
         return [op0, op1]
 
     def rewrite_op_threadlocalref_get(self, op):
-        from rpython.jit.codewriter.jitcode import ThreadLocalRefDescr
-        opaqueid = op.args[0].value
-        op1 = self.prepare_builtin_call(op, 'threadlocalref_getter', [],
-                                        extra=(opaqueid,),
-                                        extrakey=opaqueid._obj)
-        extradescr = ThreadLocalRefDescr(opaqueid)
+        # only supports RESTYPE being exactly one word.
+        RESTYPE = op.result.concretetype
+        assert (RESTYPE in (lltype.Signed, lltype.Unsigned, llmemory.Address)
+                or isinstance(RESTYPE, lltype.Ptr))
+        c_offset, = op.args
+        op1 = self.prepare_builtin_call(op, 'threadlocalref_get', [c_offset])
         return self.handle_residual_call(op1,
             oopspecindex=EffectInfo.OS_THREADLOCALREF_GET,
-            extraeffect=EffectInfo.EF_LOOPINVARIANT,
-            extradescr=[extradescr])
+            extraeffect=EffectInfo.EF_LOOPINVARIANT)
 
 # ____________________________________________________________
 
diff --git a/rpython/jit/codewriter/support.py b/rpython/jit/codewriter/support.py
--- a/rpython/jit/codewriter/support.py
+++ b/rpython/jit/codewriter/support.py
@@ -702,10 +702,9 @@
     build_ll_1_raw_free_no_track_allocation = (
         build_raw_free_builder(track_allocation=False))
 
-    def build_ll_0_threadlocalref_getter(opaqueid):
-        def _ll_0_threadlocalref_getter():
-            return llop.threadlocalref_get(rclass.OBJECTPTR, opaqueid)
-        return _ll_0_threadlocalref_getter
+    def _ll_1_threadlocalref_get(TP, offset):
+        return llop.threadlocalref_get(TP, offset)
+    _ll_1_threadlocalref_get.need_result_type = 'exact'   # don't deref
 
     def _ll_1_weakref_create(obj):
         return llop.weakref_create(llmemory.WeakRefPtr, obj)
@@ -818,8 +817,18 @@
     s_result = lltype_to_annotation(ll_res)
     impl = setup_extra_builtin(rtyper, oopspec_name, len(args_s), extra)
     if getattr(impl, 'need_result_type', False):
-        bk = rtyper.annotator.bookkeeper
-        args_s.insert(0, annmodel.SomePBC([bk.getdesc(deref(ll_res))]))
+        if hasattr(rtyper, 'annotator'):
+            bk = rtyper.annotator.bookkeeper
+            ll_restype = ll_res
+            if impl.need_result_type != 'exact':
+                ll_restype = deref(ll_restype)
+            desc = bk.getdesc(ll_restype)
+        else:
+            class TestingDesc(object):
+                knowntype = int
+                pyobj = None
+            desc = TestingDesc()
+        args_s.insert(0, annmodel.SomePBC([desc]))
     #
     if hasattr(rtyper, 'annotator'):  # regular case
         mixlevelann = MixLevelHelperAnnotator(rtyper)
diff --git a/rpython/jit/codewriter/test/test_flatten.py b/rpython/jit/codewriter/test/test_flatten.py
--- a/rpython/jit/codewriter/test/test_flatten.py
+++ b/rpython/jit/codewriter/test/test_flatten.py
@@ -1,5 +1,6 @@
 import py, sys
 from rpython.jit.codewriter import support
+from rpython.jit.codewriter.heaptracker import int_signext
 from rpython.jit.codewriter.flatten import flatten_graph, reorder_renaming_list
 from rpython.jit.codewriter.flatten import GraphFlattener, ListOfKind, Register
 from rpython.jit.codewriter.format import assert_format
@@ -281,30 +282,6 @@
             foobar hi_there!
         """)
 
-    def test_switch(self):
-        def f(n):
-            if n == -5:  return 12
-            elif n == 2: return 51
-            elif n == 7: return 1212
-            else:        return 42
-        self.encoding_test(f, [65], """
-            -live-
-            int_guard_value %i0
-            goto_if_not_int_eq %i0, $-5, L1
-            int_return $12
-            ---
-            L1:
-            goto_if_not_int_eq %i0, $2, L2
-            int_return $51
-            ---
-            L2:
-            goto_if_not_int_eq %i0, $7, L3
-            int_return $1212
-            ---
-            L3:
-            int_return $42
-        """)
-
     def test_switch_dict(self):
         def f(x):
             if   x == 1: return 61
@@ -320,21 +297,27 @@
             int_return $-1
             ---
             L1:
+            -live-
             int_return $61
             ---
             L2:
+            -live-
             int_return $511
             ---
             L3:
+            -live-
             int_return $-22
             ---
             L4:
+            -live-
             int_return $81
             ---
             L5:
+            -live-
             int_return $17
             ---
             L6:
+            -live-
             int_return $54
         """)
 
@@ -780,53 +763,37 @@
             (rffi.SIGNEDCHAR, rffi.LONG, ""),
             (rffi.SIGNEDCHAR, rffi.ULONG, ""),
 
-            (rffi.UCHAR, rffi.SIGNEDCHAR, """int_sub %i0, $-128 -> %i1
-                                             int_and %i1, $255 -> %i2
-                                             int_add %i2, $-128 -> %i3"""),
+            (rffi.UCHAR, rffi.SIGNEDCHAR, "int_signext %i0, $1 -> %i1"),
             (rffi.UCHAR, rffi.UCHAR, ""),
             (rffi.UCHAR, rffi.SHORT, ""),
             (rffi.UCHAR, rffi.USHORT, ""),
             (rffi.UCHAR, rffi.LONG, ""),
             (rffi.UCHAR, rffi.ULONG, ""),
 
-            (rffi.SHORT, rffi.SIGNEDCHAR, """int_sub %i0, $-128 -> %i1
-                                             int_and %i1, $255 -> %i2
-                                             int_add %i2, $-128 -> %i3"""),
+            (rffi.SHORT, rffi.SIGNEDCHAR, "int_signext %i0, $1 -> %i1"),
             (rffi.SHORT, rffi.UCHAR, "int_and %i0, $255 -> %i1"),
             (rffi.SHORT, rffi.SHORT, ""),
             (rffi.SHORT, rffi.USHORT, "int_and %i0, $65535 -> %i1"),
             (rffi.SHORT, rffi.LONG, ""),
             (rffi.SHORT, rffi.ULONG, ""),
 
-            (rffi.USHORT, rffi.SIGNEDCHAR, """int_sub %i0, $-128 -> %i1
-                                              int_and %i1, $255 -> %i2
-                                              int_add %i2, $-128 -> %i3"""),
+            (rffi.USHORT, rffi.SIGNEDCHAR, "int_signext %i0, $1 -> %i1"),
             (rffi.USHORT, rffi.UCHAR, "int_and %i0, $255 -> %i1"),
-            (rffi.USHORT, rffi.SHORT, """int_sub %i0, $-32768 -> %i1
-                                         int_and %i1, $65535 -> %i2
-                                         int_add %i2, $-32768 -> %i3"""),
+            (rffi.USHORT, rffi.SHORT, "int_signext %i0, $2 -> %i1"),
             (rffi.USHORT, rffi.USHORT, ""),
             (rffi.USHORT, rffi.LONG, ""),
             (rffi.USHORT, rffi.ULONG, ""),
 
-            (rffi.LONG, rffi.SIGNEDCHAR, """int_sub %i0, $-128 -> %i1
-                                            int_and %i1, $255 -> %i2
-                                            int_add %i2, $-128 -> %i3"""),
+            (rffi.LONG, rffi.SIGNEDCHAR, "int_signext %i0, $1 -> %i1"),
             (rffi.LONG, rffi.UCHAR, "int_and %i0, $255 -> %i1"),
-            (rffi.LONG, rffi.SHORT, """int_sub %i0, $-32768 -> %i1
-                                       int_and %i1, $65535 -> %i2
-                                       int_add %i2, $-32768 -> %i3"""),
+            (rffi.LONG, rffi.SHORT, "int_signext %i0, $2 -> %i1"),
             (rffi.LONG, rffi.USHORT, "int_and %i0, $65535 -> %i1"),
             (rffi.LONG, rffi.LONG, ""),
             (rffi.LONG, rffi.ULONG, ""),
 
-            (rffi.ULONG, rffi.SIGNEDCHAR, """int_sub %i0, $-128 -> %i1
-                                             int_and %i1, $255 -> %i2
-                                             int_add %i2, $-128 -> %i3"""),
+            (rffi.ULONG, rffi.SIGNEDCHAR, "int_signext %i0, $1 -> %i1"),
             (rffi.ULONG, rffi.UCHAR, "int_and %i0, $255 -> %i1"),
-            (rffi.ULONG, rffi.SHORT, """int_sub %i0, $-32768 -> %i1
-                                        int_and %i1, $65535 -> %i2
-                                        int_add %i2, $-32768 -> %i3"""),
+            (rffi.ULONG, rffi.SHORT, "int_signext %i0, $2 -> %i1"),
             (rffi.ULONG, rffi.USHORT, "int_and %i0, $65535 -> %i1"),
             (rffi.ULONG, rffi.LONG, ""),
             (rffi.ULONG, rffi.ULONG, ""),
@@ -910,18 +877,14 @@
             return rffi.cast(rffi.SIGNEDCHAR, n)
         self.encoding_test(f, [12.456], """
             cast_float_to_int %f0 -> %i0
-            int_sub %i0, $-128 -> %i1
-            int_and %i1, $255 -> %i2
-            int_add %i2, $-128 -> %i3
-            int_return %i3
+            int_signext %i0, $1 -> %i1
+            int_return %i1
         """, transform=True)
         self.encoding_test(f, [rffi.cast(lltype.SingleFloat, 12.456)], """
             cast_singlefloat_to_float %i0 -> %f0
             cast_float_to_int %f0 -> %i1
-            int_sub %i1, $-128 -> %i2
-            int_and %i2, $255 -> %i3
-            int_add %i3, $-128 -> %i4
-            int_return %i4
+            int_signext %i1, $1 -> %i2
+            int_return %i2
         """, transform=True)
 
         def f(dbl):
@@ -1068,9 +1031,12 @@
         match = r.match(op)
         assert match, "line %r does not match regexp" % (op,)
         opname = match.group(1)
-        if   opname == 'int_add': value += int(match.group(2))
-        elif opname == 'int_sub': value -= int(match.group(2))
-        elif opname == 'int_and': value &= int(match.group(2))
-        else: assert 0, opname
+        if opname == 'int_and':
+            value &= int(match.group(2))
+        elif opname == 'int_signext':
+            numbytes = int(match.group(2))
+            value = int_signext(value, numbytes)
+        else:
+            assert 0, opname
     #
     assert rffi.cast(lltype.Signed, value) == expected_value
diff --git a/rpython/jit/codewriter/test/test_jtransform.py b/rpython/jit/codewriter/test/test_jtransform.py
--- a/rpython/jit/codewriter/test/test_jtransform.py
+++ b/rpython/jit/codewriter/test/test_jtransform.py
@@ -148,9 +148,7 @@
              EI.OS_UNIEQ_LENGTHOK:       ([PUNICODE, PUNICODE], INT),
              EI.OS_RAW_MALLOC_VARSIZE_CHAR: ([INT], ARRAYPTR),
              EI.OS_RAW_FREE:             ([ARRAYPTR], lltype.Void),
-             EI.OS_THREADLOCALREF_GET:   ([], rclass.OBJECTPTR),
-             EI.OS_GET_ERRNO:            ([], INT),
-             EI.OS_SET_ERRNO:            ([INT], lltype.Void),
+             EI.OS_THREADLOCALREF_GET:   ([INT], INT),   # for example
             }
             argtypes = argtypes[oopspecindex]
             assert argtypes[0] == [v.concretetype for v in op.args[1:]]
@@ -159,9 +157,7 @@
                 assert extraeffect == EI.EF_ELIDABLE_CAN_RAISE
             elif oopspecindex == EI.OS_RAW_MALLOC_VARSIZE_CHAR:
                 assert extraeffect == EI.EF_CAN_RAISE
-            elif oopspecindex in (EI.OS_RAW_FREE,
-                                  EI.OS_GET_ERRNO,
-                                  EI.OS_SET_ERRNO):
+            elif oopspecindex == EI.OS_RAW_FREE:
                 assert extraeffect == EI.EF_CANNOT_RAISE
             elif oopspecindex == EI.OS_THREADLOCALREF_GET:
                 assert extraeffect == EI.EF_LOOPINVARIANT
@@ -1347,53 +1343,20 @@
     assert op2 is None
 
 def test_threadlocalref_get():
-    from rpython.rtyper import rclass
-    from rpython.rlib.rthread import ThreadLocalReference
+    from rpython.rlib.rthread import ThreadLocalField
+    tlfield = ThreadLocalField(lltype.Signed, 'foobar_test_')
     OS_THREADLOCALREF_GET = effectinfo.EffectInfo.OS_THREADLOCALREF_GET
-    class Foo: pass
-    t = ThreadLocalReference(Foo)
-    v2 = varoftype(rclass.OBJECTPTR)
-    c_opaqueid = const(t.opaque_id)
-    op = SpaceOperation('threadlocalref_get', [c_opaqueid], v2)
+    c = const(tlfield.offset)
+    v = varoftype(lltype.Signed)
+    op = SpaceOperation('threadlocalref_get', [c], v)
     tr = Transformer(FakeCPU(), FakeBuiltinCallControl())
     op0 = tr.rewrite_operation(op)
-    assert op0.opname == 'residual_call_r_r'
-    assert op0.args[0].value == 'threadlocalref_getter' # pseudo-function as str
-    assert op0.args[1] == ListOfKind("ref", [])
-    assert op0.args[2] == 'calldescr-%d' % OS_THREADLOCALREF_GET
-    assert op0.result == v2
-
-def test_get_errno():
-    # test that the oopspec is present and correctly transformed
-    from rpython.rlib import rposix
-    FUNC = lltype.FuncType([], lltype.Signed)
-    func = lltype.functionptr(FUNC, 'get_errno', _callable=rposix.get_errno)
-    v3 = varoftype(lltype.Signed)
-    op = SpaceOperation('direct_call', [const(func)], v3)
-    tr = Transformer(FakeCPU(), FakeBuiltinCallControl())
-    op1 = tr.rewrite_operation(op)
-    assert op1.opname == 'residual_call_r_i'
-    assert op1.args[0].value == func
-    assert op1.args[1] == ListOfKind('ref', [])
-    assert op1.args[2] == 'calldescr-%d' % effectinfo.EffectInfo.OS_GET_ERRNO
-    assert op1.result == v3
-
-def test_set_errno():
-    # test that the oopspec is present and correctly transformed
-    from rpython.rlib import rposix
-    FUNC = lltype.FuncType([lltype.Signed], lltype.Void)
-    func = lltype.functionptr(FUNC, 'set_errno', _callable=rposix.set_errno)
-    v1 = varoftype(lltype.Signed)
-    v3 = varoftype(lltype.Void)
-    op = SpaceOperation('direct_call', [const(func), v1], v3)
-    tr = Transformer(FakeCPU(), FakeBuiltinCallControl())
-    op1 = tr.rewrite_operation(op)
-    assert op1.opname == 'residual_call_ir_v'
-    assert op1.args[0].value == func
-    assert op1.args[1] == ListOfKind('int', [v1])
-    assert op1.args[2] == ListOfKind('ref', [])
-    assert op1.args[3] == 'calldescr-%d' % effectinfo.EffectInfo.OS_SET_ERRNO
-    assert op1.result == v3
+    assert op0.opname == 'residual_call_ir_i'
+    assert op0.args[0].value == 'threadlocalref_get' # pseudo-function as str
+    assert op0.args[1] == ListOfKind("int", [c])
+    assert op0.args[2] == ListOfKind("ref", [])
+    assert op0.args[3] == 'calldescr-%d' % OS_THREADLOCALREF_GET
+    assert op0.result == v
 
 def test_unknown_operation():
     op = SpaceOperation('foobar', [], varoftype(lltype.Void))
diff --git a/rpython/jit/metainterp/blackhole.py b/rpython/jit/metainterp/blackhole.py
--- a/rpython/jit/metainterp/blackhole.py
+++ b/rpython/jit/metainterp/blackhole.py
@@ -489,6 +489,9 @@
         if i < 0:
             return 0
         return i
+    @arguments("i", "i", returns="i")
+    def bhimpl_int_signext(a, b):
+        return heaptracker.int_signext(a, b)
 
     @arguments("i", "i", returns="i")
     def bhimpl_uint_lt(a, b):
diff --git a/rpython/jit/metainterp/optimizeopt/intbounds.py b/rpython/jit/metainterp/optimizeopt/intbounds.py
--- a/rpython/jit/metainterp/optimizeopt/intbounds.py
+++ b/rpython/jit/metainterp/optimizeopt/intbounds.py
@@ -342,6 +342,19 @@
         else:
             self.emit_operation(op)
 
+    def optimize_INT_SIGNEXT(self, op):
+        value = self.getvalue(op.getarg(0))
+        numbits = op.getarg(1).getint() * 8
+        start = -(1 << (numbits - 1))
+        stop = 1 << (numbits - 1)
+        bounds = IntBound(start, stop - 1)
+        if bounds.contains_bound(value.intbound):
+            self.make_equal_to(op.result, value)
+        else:
+            self.emit_operation(op)
+            vres = self.getvalue(op.result)
+            vres.intbound.intersect(bounds)
+
     def optimize_ARRAYLEN_GC(self, op):
         self.emit_operation(op)
         array = self.getvalue(op.getarg(0))
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py
@@ -5494,6 +5494,41 @@
         """
         self.optimize_loop(ops, expected)
 
+    def test_int_signext_already_in_bounds(self):
+        ops = """
+        [i0]
+        i1 = int_signext(i0, 1)
+        i2 = int_signext(i1, 2)
+        jump(i2)
+        """
+        expected = """
+        [i0]
+        i1 = int_signext(i0, 1)
+        jump(i1)
+        """
+        self.optimize_loop(ops, expected)
+        #
+        ops = """
+        [i0]
+        i1 = int_signext(i0, 1)
+        i2 = int_signext(i1, 1)
+        jump(i2)
+        """
+        expected = """
+        [i0]
+        i1 = int_signext(i0, 1)
+        jump(i1)
+        """
+        self.optimize_loop(ops, expected)
+        #
+        ops = """
+        [i0]
+        i1 = int_signext(i0, 2)
+        i2 = int_signext(i1, 1)
+        jump(i2)
+        """
+        self.optimize_loop(ops, ops)
+
 
 class TestLLtype(BaseTestOptimizeBasic, LLtypeMixin):
     pass
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
@@ -1,4 +1,4 @@
-import py
+import py, sys
 from rpython.rlib.objectmodel import instantiate
 from rpython.jit.metainterp import compile, resume
 from rpython.jit.metainterp.history import AbstractDescr, ConstInt, BoxInt, TreeLoop
@@ -190,6 +190,11 @@
             args = []
             for _ in range(oparity[opnum]):
                 args.append(random.randrange(1, 20))
+            if opnum == rop.INT_SIGNEXT:
+                # 2nd arg is number of bytes to extend from ---
+                # must not be too random
+                args[-1] = random.choice([1, 2] if sys.maxint < 2**32 else
+                                         [1, 2, 4])
             ops = """
             []
             i1 = %s(%s)
@@ -5607,6 +5612,44 @@
         """
         self.optimize_loop(ops, ops, ops)
 
+    def test_bound_backpropagate_int_signext(self):
+        ops = """
+        []
+        i0 = escape()
+        i1 = int_signext(i0, 1)
+        i2 = int_eq(i0, i1)
+        guard_true(i2) []
+        i3 = int_le(i0, 127)    # implied by equality with int_signext
+        guard_true(i3) []
+        i5 = int_gt(i0, -129)   # implied by equality with int_signext
+        guard_true(i5) []
+        jump()
+        """
+        expected = """
+        []
+        i0 = escape()
+        i1 = int_signext(i0, 1)
+        i2 = int_eq(i0, i1)
+        guard_true(i2) []
+        jump()
+        """
+        self.optimize_loop(ops, expected)
+
+    def test_bound_backpropagate_int_signext_2(self):
+        ops = """
+        []
+        i0 = escape()
+        i1 = int_signext(i0, 1)
+        i2 = int_eq(i0, i1)
+        guard_true(i2) []
+        i3 = int_le(i0, 126)    # false for i1 == 127
+        guard_true(i3) []
+        i5 = int_gt(i0, -128)   # false for i1 == -128
+        guard_true(i5) []
+        jump()
+        """
+        self.optimize_loop(ops, ops)
+
     def test_mul_ovf(self):
         ops = """
         [i0, i1]
diff --git a/rpython/jit/metainterp/pyjitpl.py b/rpython/jit/metainterp/pyjitpl.py
--- a/rpython/jit/metainterp/pyjitpl.py
+++ b/rpython/jit/metainterp/pyjitpl.py
@@ -197,7 +197,7 @@
     # ------------------------------
 
     for _opimpl in ['int_add', 'int_sub', 'int_mul', 'int_floordiv', 'int_mod',
-                    'int_and', 'int_or', 'int_xor',
+                    'int_and', 'int_or', 'int_xor', 'int_signext',
                     'int_rshift', 'int_lshift', 'uint_rshift',
                     'uint_lt', 'uint_le', 'uint_gt', 'uint_ge',
                     'uint_floordiv',
@@ -402,13 +402,26 @@
 
     @arguments("box", "descr", "orgpc")
     def opimpl_switch(self, valuebox, switchdict, orgpc):
-        box = self.implement_guard_value(valuebox, orgpc)
-        search_value = box.getint()
+        search_value = valuebox.getint()
         assert isinstance(switchdict, SwitchDictDescr)
         try:
-            self.pc = switchdict.dict[search_value]
+            target = switchdict.dict[search_value]
         except KeyError:
-            pass
+            # None of the cases match.  Fall back to generating a chain
+            # of 'int_eq'.
+            # xxx as a minor optimization, if that's a bridge, then we would
+            # not need the cases that we already tested (and failed) with
+            # 'guard_value'.  How to do it is not very clear though.
+            for const1 in switchdict.const_keys_in_order:
+                box = self.execute(rop.INT_EQ, valuebox, const1)
+                assert box.getint() == 0
+                target = switchdict.dict[const1.getint()]
+                self.metainterp.generate_guard(rop.GUARD_FALSE, box,
+                                               resumepc=target)
+        else:
+            # found one of the cases
+            self.implement_guard_value(valuebox, orgpc)
+            self.pc = target
 
     @arguments()
     def opimpl_unreachable(self):
@@ -2270,8 +2283,8 @@
         if opnum == rop.GUARD_TRUE:     # a goto_if_not that jumps only now
             if not dont_change_position:
                 frame.pc = frame.jitcode.follow_jump(frame.pc)
-        elif opnum == rop.GUARD_FALSE:     # a goto_if_not that stops jumping
-            pass
+        elif opnum == rop.GUARD_FALSE:     # a goto_if_not that stops jumping;
+            pass                  # or a switch that was in its "default" case
         elif opnum == rop.GUARD_VALUE or opnum == rop.GUARD_CLASS:
             pass        # the pc is already set to the *start* of the opcode
         elif (opnum == rop.GUARD_NONNULL or
diff --git a/rpython/jit/metainterp/resoperation.py b/rpython/jit/metainterp/resoperation.py
--- a/rpython/jit/metainterp/resoperation.py
+++ b/rpython/jit/metainterp/resoperation.py
@@ -419,6 +419,7 @@
     'INT_RSHIFT/2',
     'INT_LSHIFT/2',
     'UINT_RSHIFT/2',
+    'INT_SIGNEXT/2',
     'FLOAT_ADD/2',
     'FLOAT_SUB/2',
     'FLOAT_MUL/2',
diff --git a/rpython/jit/metainterp/test/test_ajit.py b/rpython/jit/metainterp/test/test_ajit.py
--- a/rpython/jit/metainterp/test/test_ajit.py
+++ b/rpython/jit/metainterp/test/test_ajit.py
@@ -698,6 +698,40 @@
         res = self.interp_operations(f, [12311])
         assert res == 42
 
+    def test_switch_bridges(self):
+        from rpython.rlib.rarithmetic import intmask
+        myjitdriver = JitDriver(greens = [], reds = 'auto')
+        lsts = [[-5, 2, 20] * 6,
+                [7, 123, 2] * 6,
+                [12311, -5, 7] * 6,
+                [7, 123, 2] * 4 + [-5, -5, -5] * 2,
+                [7, 123, 2] * 4 + [-5, -5, -5] * 2 + [12311, 12311, 12311],
+                ]
+        def f(case):
+            x = 0
+            i = 0
+            lst = lsts[case]
+            while i < len(lst):
+                myjitdriver.jit_merge_point()
+                n = lst[i]
+                if n == -5:  a = 5
+                elif n == 2: a = 4
+                elif n == 7: a = 3
+                else:        a = 2
+                x = intmask(x * 10 + a)
+                i += 1
+            return x
+        res = self.meta_interp(f, [0], backendopt=True)
+        assert res == intmask(542 * 1001001001001001)
+        res = self.meta_interp(f, [1], backendopt=True)
+        assert res == intmask(324 * 1001001001001001)
+        res = self.meta_interp(f, [2], backendopt=True)
+        assert res == intmask(253 * 1001001001001001)
+        res = self.meta_interp(f, [3], backendopt=True)
+        assert res == intmask(324324324324555555)
+        res = self.meta_interp(f, [4], backendopt=True)
+        assert res == intmask(324324324324555555222)
+
     def test_r_uint(self):
         from rpython.rlib.rarithmetic import r_uint
         myjitdriver = JitDriver(greens = [], reds = ['y'])
@@ -833,23 +867,6 @@
         assert type(res) == bool
         assert not res
 
-    def test_switch_dict(self):
-        def f(x):
-            if   x == 1: return 61
-            elif x == 2: return 511
-            elif x == 3: return -22
-            elif x == 4: return 81
-            elif x == 5: return 17
-            elif x == 6: return 54
-            elif x == 7: return 987
-            elif x == 8: return -12
-            elif x == 9: return 321
-            return -1
-        res = self.interp_operations(f, [5])
-        assert res == 17
-        res = self.interp_operations(f, [15])
-        assert res == -1
-
     def test_int_add_ovf(self):
         def f(x, y):
             try:
@@ -3048,6 +3065,16 @@
         res = self.meta_interp(f, [32])
         assert res == f(32)
 
+    def test_int_signext(self):
+        def f(n):
+            return rffi.cast(rffi.SIGNEDCHAR, n)
+        res = self.interp_operations(f, [128])
+        assert res == -128
+        res = self.interp_operations(f, [-35 + 256 * 29])
+        assert res == -35
+        res = self.interp_operations(f, [127 - 256 * 29])
+        assert res == 127
+
 class BaseLLtypeTests(BasicTests):
 
     def test_identityhash(self):
diff --git a/rpython/jit/metainterp/test/test_threadlocal.py b/rpython/jit/metainterp/test/test_threadlocal.py
--- a/rpython/jit/metainterp/test/test_threadlocal.py
+++ b/rpython/jit/metainterp/test/test_threadlocal.py
@@ -1,29 +1,21 @@
 import py
+from rpython.rlib import rthread
 from rpython.jit.metainterp.test.support import LLJitMixin
-from rpython.rlib.rthread import ThreadLocalReference
-from rpython.rlib.jit import dont_look_inside
+from rpython.rtyper.lltypesystem import lltype
+from rpython.rtyper.lltypesystem.lloperation import llop
 
 
 class ThreadLocalTest(object):
 
     def test_threadlocalref_get(self):
-        class Foo:
-            pass
-        t = ThreadLocalReference(Foo)
-        x = Foo()
-
-        @dont_look_inside
-        def setup():
-            t.set(x)

From noreply at buildbot.pypy.org  Sun Nov 30 23:43:26 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Sun, 30 Nov 2014 23:43:26 +0100 (CET)
Subject: [pypy-commit] creflect default: progress
Message-ID: <20141130224326.CCF7C1C0296@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r131:f96272960996
Date: 2014-11-30 23:15 +0100
http://bitbucket.org/cffi/creflect/changeset/f96272960996/

Log:	progress

diff --git a/creflect/cparser.py b/creflect/cparser.py
--- a/creflect/cparser.py
+++ b/creflect/cparser.py
@@ -121,16 +121,15 @@
             assert isinstance(qualtp.type, model.FunctionType)
             self.declarations.append(model.FuncDecl(decl.name, qualtp))
         else:
-            const = 'const' in decl.quals
             if isinstance(node, pycparser.c_ast.Struct):
                 if node.decls is not None:
-                    self.get_struct_union_enum_type('struct', node, const)
+                    self.get_struct_union_enum_type('struct', node)
             elif isinstance(node, pycparser.c_ast.Union):
                 if node.decls is not None:
-                    self.get_struct_union_enum_type('union', node, const)
+                    self.get_struct_union_enum_type('union', node)
             elif isinstance(node, pycparser.c_ast.Enum):
                 if node.values is not None:
-                    self.get_struct_union_enum_type('enum', node, const)
+                    self.get_struct_union_enum_type('enum', node)
             elif not decl.name:
                 raise api.CDefError("construct does not declare any variable",
                                     decl)
@@ -184,27 +183,29 @@
                                   quals_num(typenode.quals))
         #
         if isinstance(typenode, pycparser.c_ast.TypeDecl):
-            const = 'const' in typenode.quals
             type = typenode.type
+            realtype = None
             if isinstance(type, pycparser.c_ast.IdentifierType):
                 # assume a primitive type.
                 realtype = resolve_common_type(type.names)
-                return model.QualType(realtype, quals_num(typenode.quals))
             #
             if isinstance(type, pycparser.c_ast.Struct):
                 # 'struct foobar'
-                return self.get_struct_union_enum_type('struct', type, const,
-                                                       approx_name)
+                realtype = self.get_struct_union_enum_type('struct', type,
+                                                           approx_name)
             #
             if isinstance(type, pycparser.c_ast.Union):
                 # 'union foobar'
-                return self.get_struct_union_enum_type('union', type, const,
-                                                       approx_name)
+                realtype = self.get_struct_union_enum_type('union', type,
+                                                           approx_name)
             #
             if isinstance(type, pycparser.c_ast.Enum):
                 # 'enum foobar'
-                return self.get_struct_union_enum_type('enum', type, const,
-                                                       approx_name)
+                realtype = self.get_struct_union_enum_type('enum', type,
+                                                           approx_name)
+
+            if realtype is not None:
+                return model.QualType(realtype, quals_num(typenode.quals))
         #
         if isinstance(typenode, pycparser.c_ast.FuncDecl):
             # a function type (it is never qualified)
@@ -213,10 +214,10 @@
         #
         # nested anonymous structs or unions end up here
         if isinstance(typenode, pycparser.c_ast.Struct):
-            return self.get_struct_union_enum_type('struct', typenode, const,
+            return self.get_struct_union_enum_type('struct', typenode,
                                                    name, nested=True)
         if isinstance(typenode, pycparser.c_ast.Union):
-            return self.get_struct_union_enum_type('union', typenode, const,
+            return self.get_struct_union_enum_type('union', typenode,
                                                    name, nested=True)
         #
         raise api.FFIError(":%d: bad or unsupported type declaration" %
@@ -253,13 +254,13 @@
             return 'const' in typenode.quals
         return False
 
-    def get_struct_union_enum_type(self, kind, type, const, approx_name=None):
+    def get_struct_union_enum_type(self, kind, type, approx_name=None):
         name = type.name or approx_name
         if not name or name.startswith('$$$'):
             self.parse_error("not implemented: anonymous 'struct' elsewhere "
                              "than in 'typedef struct { ... } typename;' or "
                              "'typedef struct { ... } *typename;'", type)
-        result = model.StructOrUnionOrEnum(kind, name, const)
+        result = model.StructOrUnionOrEnum(kind, name)
         #
         # get the type declaration or create it if needed
         key = '%s %s' % (kind, name)
@@ -289,7 +290,7 @@
         if typedecl.fldnames is not None:
             raise api.CDefError("duplicate declaration of struct %s" % name)
         fldnames = []
-        fldtypes = []
+        fldqualtypes = []
         fldbitsize = []
         if len(fields) == 1 and fields[0].name == '__crx_empty__':
             fields = []
@@ -298,17 +299,12 @@
                 bitsize = -1
             else:
                 bitsize = self.parse_constant(decl.bitsize)
-            self.partial_length = False
-            type = self.get_type(decl.type)
-            if self.partial_length:
-                self.make_partial(tp, nested)
-            #if isinstance(type, model.StructType) and type.partial:
-            #    self.make_partial(tp, nested)
+            qualtype = self.get_qualtype(decl.type)
             fldnames.append(decl.name or '')
-            fldtypes.append(type)
+            fldqualtypes.append(qualtype)
             fldbitsize.append(bitsize)
         typedecl.fldnames = tuple(fldnames)
-        typedecl.fldtypes = tuple(fldtypes)
+        typedecl.fldqualtypes = tuple(fldqualtypes)
         typedecl.fldbitsize = tuple(fldbitsize)
         if fldbitsize != [-1] * len(fldbitsize):
             xxxx
diff --git a/creflect/model.py b/creflect/model.py
--- a/creflect/model.py
+++ b/creflect/model.py
@@ -472,8 +472,6 @@
         self.kind = kind
         self.name = name
         self.c_name_with_marker = '%s %s &' % (self.kind, self.name)
-        if const:
-            self.c_name_with_marker = 'const ' + self.c_name_with_marker
 
     def get_type_var(self, block):
         return block.write_crx_type_var('cb->get_%s_type(cb, "%s")' % (
@@ -643,7 +641,7 @@
     def __init__(self, type):
         self.type = type
         self.fldnames = None
-        self.fldtypes = None
+        self.fldqualtypes = None
         self.fldbitsize = None
 
     def write_declaration(self, funcblock):
@@ -677,14 +675,14 @@
             d1 = 'NULL'
         t1 = self.type.get_type_var(funcblock)
         #
-        for i, (fldname, fldtype) in enumerate(
-                zip(self.fldnames, self.fldtypes)):
+        for i, (fldname, fldqualtype) in enumerate(
+                zip(self.fldnames, self.fldqualtypes)):
             block = CodeBlock(funcblock)
             inspect = TypeInspector(block, insptp, fldname)
             inspect.start()
             # get the offset of the field
             arraylevels = 0
-            checktype = fldtype
+            checktype = fldqualtype.type
             while isinstance(checktype, ArrayType):
                 arraylevels += 1
                 checktype = checktype.item
@@ -700,7 +698,7 @@
                 o_decl = ("size_t o = ((char *)&((%s)0)->%s)"
                           " - (char *)0;%s" % (ptrtp, fldname, comment))
             #
-            t2, q2 = fldtype.inspect_type(block, inspect)
+            t2, q2 = fldqualtype.inspect_qualtype(block, inspect)
             inspect.stop()
             block.writedecl(o_decl)
             block.writeline('%s[%d].name = "%s";' % (d1, i, fldname))

From noreply at buildbot.pypy.org  Sun Nov 30 23:43:28 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Sun, 30 Nov 2014 23:43:28 +0100 (CET)
Subject: [pypy-commit] creflect default: fix fix done
Message-ID: <20141130224328.0D5501C0296@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r132:6ed79bee3ecc
Date: 2014-11-30 23:43 +0100
http://bitbucket.org/cffi/creflect/changeset/6ed79bee3ecc/

Log:	fix fix done

diff --git a/creflect/creflect_cdecl.c b/creflect/creflect_cdecl.c
--- a/creflect/creflect_cdecl.c
+++ b/creflect/creflect_cdecl.c
@@ -35,6 +35,7 @@
     TOK_UNION,
     TOK_UNSIGNED,
     TOK_VOID,
+    TOK_VOLATILE,
 };
 
 #define NUM_DELAY_SLOTS   5000
@@ -152,6 +153,7 @@
         break;
     case 'v':
         if (tok->size == 4 && !memcmp(p, "void", 4))   tok->kind = TOK_VOID;
+        if (tok->size == 8 && !memcmp(p,"volatile",8)) tok->kind = TOK_VOLATILE;
         break;
     }
 }
@@ -178,12 +180,13 @@
     return result;
 }
 
-static _crx_type_t *parse_complete(token_t *tok);
+static void parse_complete(token_t *tok, _crx_qual_type *result);
 
 static void parse_sequel(token_t *tok, intptr_t ds_end)
 {
     intptr_t *ds;
-    while (tok->kind == TOK_STAR || tok->kind == TOK_CONST) {
+    while (tok->kind == TOK_STAR || tok->kind == TOK_CONST ||
+           tok->kind == TOK_VOLATILE) {
         ds = alloc_ds(tok, 1);
         if (ds == NULL)
             return;
@@ -212,6 +215,7 @@
 
         if (check_for_grouping == 0 && (tok->kind == TOK_STAR ||
                                         tok->kind == TOK_CONST ||
+                                        tok->kind == TOK_VOLATILE ||
                                         tok->kind == TOK_OPEN_BRACKET)) {
             /* just parentheses for grouping */
             ds = tok->delay_slots;
@@ -235,12 +239,12 @@
                         next_token(tok);
                         break;
                     }
-                    _crx_type_t *t1 = parse_complete(tok);
-                    intptr_t *ds_type = alloc_ds(tok, 1);
+                    intptr_t *ds_type = alloc_ds(tok, 2);
                     if (ds_type == NULL)
                         return;
-                    assert(ds_type == ds + 2 + ds[1]);
-                    *ds_type = (intptr_t)t1;
+                    assert(ds_type == ds + 2 + 2 * ds[1]);
+                    assert(2 * sizeof(intptr_t) >= sizeof(_crx_qual_type));
+                    parse_complete(tok, (_crx_qual_type *)ds_type);
                     ds[1]++;
                     if (tok->kind != TOK_COMMA)
                         break;
@@ -250,7 +254,7 @@
             intptr_t *ds_next = alloc_ds(tok, 1);
             if (ds_next == NULL)
                 return;
-            assert(ds_next == ds + 2 + ds[1]);
+            assert(ds_next == ds + 2 + 2 * ds[1]);
             *ds_next = *jump_slot;
             *jump_slot = -(ds - tok->all_delay_slots);
         }
@@ -303,11 +307,11 @@
     }
 }
 
-static _crx_type_t *fetch_delay_slots(token_t *tok, _crx_type_t *t1,
-                                     intptr_t *delay_slot)
+static void fetch_delay_slots(token_t *tok, _crx_qual_type *result,
+                              intptr_t *delay_slot)
 {
     if (tok->kind == TOK_ERROR)
-        return NULL;
+        return;
     tok->delay_slots = delay_slot;
     while (1) {
         intptr_t tok_kind = *delay_slot++;
@@ -317,34 +321,46 @@
         }
         switch (tok_kind) {
         case TOK_END:
-            return t1;
+            return;    /* done */
         case TOK_STAR:
-            t1 = tok->cb->get_pointer_type(tok->cb, t1);
+            result->type = tok->cb->get_pointer_type(tok->cb, result->type,
+                                                     result->qualifiers);
+            result->qualifiers = 0;
             break;
         case TOK_CONST:
-            t1 = tok->cb->get_const_type(tok->cb, t1);
+            result->qualifiers |= _CRX_CONST;
+            break;
+        case TOK_VOLATILE:
+            result->qualifiers |= _CRX_VOLATILE;
             break;
         case TOK_OPEN_BRACKET:   /* array */
             {
                 uintptr_t length = (uintptr_t)*delay_slot++;
                 if (length != (uintptr_t)-1)
-                    t1 = tok->cb->get_array_type(tok->cb, t1, length);
+                    result->type = tok->cb->get_array_type(
+                        tok->cb, result->type, length);
                 else
-                    t1 = tok->cb->get_incomplete_array_type(tok->cb, t1);
+                    result->type = tok->cb->get_incomplete_array_type(
+                        tok->cb, result->type);
+                /* result->qualifiers remains unmodified */
                 break;
             }
         case TOK_OPEN_PAREN:   /* function */
         case TOK_DOTDOTDOT:    /* function ending with a '...' */
             {
                 intptr_t nbargs = *delay_slot++;
-                _crx_type_t **argtypes = (_crx_type_t **)delay_slot;
-                delay_slot += nbargs;
+                _crx_type_t *t1;
+                _crx_qual_type *argtypes = (_crx_qual_type *)delay_slot;
+                delay_slot += 2 * nbargs;
                 if (tok_kind == TOK_DOTDOTDOT)
-                    t1 = tok->cb->get_ellipsis_function_type(tok->cb, t1,
+                    t1 = tok->cb->get_ellipsis_function_type(tok->cb,
+                                                             result->type,
                                                              argtypes, nbargs);
                 else
-                    t1 = tok->cb->get_function_type(tok->cb, t1, argtypes,
-                                                    nbargs, NULL);
+                    t1 = tok->cb->get_function_type(tok->cb, result->type,
+                                                    argtypes, nbargs, NULL);
+                result->type = t1;
+                result->qualifiers = 0; /* drop qualifiers on the return type */
                 break;
             }
         default:
@@ -353,12 +369,23 @@
     }
 }
 
-static _crx_type_t *parse_complete(token_t *tok)
+static void parse_complete(token_t *tok, _crx_qual_type *result)
 {
     _crx_type_t *t1;
-    int is_const = (tok->kind == TOK_CONST);
-    if (is_const) {
+
+    result->qualifiers = 0;
+ qualifiers:
+    switch (tok->kind) {
+    case TOK_CONST:
+        result->qualifiers |= _CRX_CONST;
         next_token(tok);
+        goto qualifiers;
+    case TOK_VOLATILE:
+        result->qualifiers |= _CRX_VOLATILE;
+        next_token(tok);
+        goto qualifiers;
+    default:
+        ;
     }
 
     int modifiers_length = 0;
@@ -367,31 +394,41 @@
     switch (tok->kind) {
 
     case TOK_SHORT:
-        if (modifiers_length != 0)
-            return parse_error(tok, "'short' after another 'short' or 'long'");
+        if (modifiers_length != 0) {
+            parse_error(tok, "'short' after another 'short' or 'long'");
+            return;
+        }
         modifiers_length--;
         next_token(tok);
         goto modifiers;
 
     case TOK_LONG:
-        if (modifiers_length < 0)
-            return parse_error(tok, "'long' after 'short'");
-        if (modifiers_length >= 2)
-            return parse_error(tok, "'long long long' is too long");
+        if (modifiers_length < 0) {
+            parse_error(tok, "'long' after 'short'");
+            return;
+        }
+        if (modifiers_length >= 2) {
+            parse_error(tok, "'long long long' is too long");
+            return;
+        }
         modifiers_length++;
         next_token(tok);
         goto modifiers;
 
     case TOK_SIGNED:
-        if (modifiers_sign)
-            return parse_error(tok, "multiple 'signed' or 'unsigned'");
+        if (modifiers_sign) {
+            parse_error(tok, "multiple 'signed' or 'unsigned'");
+            return;
+        }
         modifiers_sign++;
         next_token(tok);
         goto modifiers;
 
     case TOK_UNSIGNED:
-        if (modifiers_sign)
-            return parse_error(tok, "multiple 'signed' or 'unsigned'");
+        if (modifiers_sign) {
+            parse_error(tok, "multiple 'signed' or 'unsigned'");
+            return;
+        }
         modifiers_sign--;
         next_token(tok);
         goto modifiers;
@@ -410,19 +447,24 @@
         case TOK_IDENTIFIER:
         case TOK_STRUCT:
         case TOK_UNION:
-            return parse_error(tok, "invalid combination of types");
+            parse_error(tok, "invalid combination of types");
+            return;
 
         case TOK_DOUBLE:
-            if (modifiers_sign != 0 || modifiers_length != 1)
-                return parse_error(tok, "invalid combination of types");
+            if (modifiers_sign != 0 || modifiers_length != 1) {
+                parse_error(tok, "invalid combination of types");
+                return;
+            }
             next_token(tok);
             t1 = tok->cb->get_float_type(tok->cb, sizeof(long double),
                                          "long double");
             break;
 
         case TOK_CHAR:
-            if (modifiers_length != 0)
-                return parse_error(tok, "invalid combination of types");
+            if (modifiers_length != 0) {
+                parse_error(tok, "invalid combination of types");
+                return;
+            }
             modifiers_length = -2;
             /* fall-through */
         case TOK_INT:
@@ -468,12 +510,17 @@
             break;
         case TOK_IDENTIFIER:
         {
+            _crx_qual_type qt2;
             char identifier[1024];
-            if (tok->size >= 1024)
-                return parse_error(tok, "identifier name too long");
+            if (tok->size >= 1024) {
+                parse_error(tok, "identifier name too long");
+                return;
+            }
             memcpy(identifier, tok->p, tok->size);
             identifier[tok->size] = 0;
-            t1 = tok->cb->get_user_type(tok->cb, identifier);
+            qt2 = tok->cb->get_user_type(tok->cb, identifier);
+            t1 = qt2.type;
+            result->qualifiers |= qt2.qualifiers;
             break;
         }
         case TOK_STRUCT:
@@ -482,10 +529,14 @@
             char identifier[1024];
             int kind = tok->kind;
             next_token(tok);
-            if (tok->kind != TOK_IDENTIFIER)
-                return parse_error(tok, "struct or union name expected");
-            if (tok->size >= 1024)
-                return parse_error(tok, "struct or union name too long");
+            if (tok->kind != TOK_IDENTIFIER) {
+                parse_error(tok, "struct or union name expected");
+                return;
+            }
+            if (tok->size >= 1024) {
+                parse_error(tok, "struct or union name too long");
+                return;
+            }
             memcpy(identifier, tok->p, tok->size);
             identifier[tok->size] = 0;
             if (kind == TOK_STRUCT)
@@ -495,41 +546,40 @@
             break;
         }
         default:
-            return parse_error(tok, "identifier expected");
+            parse_error(tok, "identifier expected");
+            return;
         }
         next_token(tok);
     }
-
-    if (is_const) {
-        t1 = tok->cb->get_const_type(tok->cb, t1);
-    }
+    result->type = t1;
 
     intptr_t *orig = tok->delay_slots;
     parse_sequel(tok, TOK_END);
-    return fetch_delay_slots(tok, t1, orig);
+    fetch_delay_slots(tok, result, orig);
 }
 
-_crx_type_t *creflect_decl_parser(_crx_builder_t *cb, const char *input,
-                                  const char **error_location)
+const char *creflect_decl_parser(_crx_builder_t *cb, const char *input,
+                                 _crx_qual_type *result)
 {
-    _crx_type_t *result;
+    _crx_qual_type qt1;
+    const char *error_location;
     token_t token;
+
     token.kind = TOK_START;
     token.cb = cb;
     token.p = input;
-    token.error_location = error_location;
+    token.error_location = &error_location;
     token.size = 0;
     token.delay_slots = token.all_delay_slots;
     next_token(&token);
-    result = parse_complete(&token);
+    parse_complete(&token, &qt1);
 
     if (token.kind == TOK_END) {
-        if (error_location)
-            *error_location = NULL;
-        return result;
+        *result = qt1;
+        return NULL;
     }
     else {
         parse_error(&token, "unexpected symbol");
-        return NULL;
+        return error_location;
     }
 }
diff --git a/creflect/creflect_cdecl.h b/creflect/creflect_cdecl.h
--- a/creflect/creflect_cdecl.h
+++ b/creflect/creflect_cdecl.h
@@ -4,8 +4,8 @@
 #include "creflect.h"
 
 
-_crx_type_t *creflect_decl_parser(_crx_builder_t *cb, const char *input,
-                                  const char **error_location);
+const char *creflect_decl_parser(_crx_builder_t *cb, const char *input,
+                                 _crx_qual_type *result);
 
 
 #endif  /* _CRX_CDECL_H_ */
diff --git a/creflect/creflect_cdecl_main.c b/creflect/creflect_cdecl_main.c
--- a/creflect/creflect_cdecl_main.c
+++ b/creflect/creflect_cdecl_main.c
@@ -3,22 +3,27 @@
 #include "creflect_cdecl.h"
 #include "creflect_debug_print.h"
 
+#define ALL_QUAL  (_CRX_CONST | _CRX_VOLATILE)
+
+
 int main(int argc, char *argv[])
 {
-    _crx_type_t *t1;
+    _crx_qual_type qt1;
     const char *error;
 
     if (argc != 2) {
         fprintf(stderr, "Usage: %s 'c_type_declaration'\n", argv[0]);
         return 2;
     }
-    t1 = creflect_decl_parser(&creflect_debug_builder, argv[1], &error);
+    qt1.type = NULL;
+    qt1.qualifiers = -1;
+    error = creflect_decl_parser(&creflect_debug_builder, argv[1], &qt1);
     if (error == NULL) {
-        assert(t1 != NULL);
-        printf("%s\n", creflect_type_text(t1));
+        assert(qt1.type != NULL);
+        assert((qt1.qualifiers & ~ALL_QUAL) == 0);
+        printf("%s\n", show_qualtype1(qt1));
     }
     else {
-        assert(t1 == NULL);
         printf("%s\n", argv[1]);
         while (error > argv[1]) {
             printf(" ");
diff --git a/creflect/creflect_debug_print.c b/creflect/creflect_debug_print.c
--- a/creflect/creflect_debug_print.c
+++ b/creflect/creflect_debug_print.c
@@ -10,11 +10,6 @@
 };
 
 
-const char *creflect_type_text(_crx_type_t *t1)
-{
-    return t1->text;
-}
-
 static _crx_type_t *newtype(const char *a)
 {
     size_t la = strlen(a);
@@ -32,7 +27,7 @@
     return t;
 }
 
-static const char *show_qualtype2(_crx_type_t *type, int qualifiers)
+const char *show_qualtype2(_crx_type_t *type, int qualifiers)
 {
     size_t la = strlen(type->text);
     size_t extra = 1;
@@ -52,7 +47,7 @@
     return p;
 }
 
-static const char *show_qualtype1(_crx_qual_type qualtype)
+const char *show_qualtype1(_crx_qual_type qualtype)
 {
     return show_qualtype2(qualtype.type, qualtype.qualifiers);
 }
diff --git a/creflect/creflect_debug_print.h b/creflect/creflect_debug_print.h
--- a/creflect/creflect_debug_print.h
+++ b/creflect/creflect_debug_print.h
@@ -5,7 +5,8 @@
 
 
 extern _crx_builder_t creflect_debug_builder;
-const char *creflect_type_text(_crx_type_t *);
+const char *show_qualtype2(_crx_type_t *type, int qualifiers);
+const char *show_qualtype1(_crx_qual_type qualtype);
 
 
 #endif  /* _CRX_DEBUG_PRINT_H_ */
diff --git a/creflect/test/test_c_decl_parser.py b/creflect/test/test_c_decl_parser.py
--- a/creflect/test/test_c_decl_parser.py
+++ b/creflect/test/test_c_decl_parser.py
@@ -5,7 +5,7 @@
 def setup_module(mod):
     creflect_dir = os.path.join(os.path.dirname(__file__), '..')
     executable = str(udir.join('c_decl_parser_test'))
-    err = os.system("gcc -g -Werror -o '%s'"
+    err = os.system("gcc -g -Wall -Werror -o '%s'"
                     " '%s/creflect_cdecl.c'"
                     " '%s/creflect_debug_print.c'"
                     " '%s/creflect_cdecl_main.c'" % (

From noreply at buildbot.pypy.org  Sun Nov 30 23:48:14 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Sun, 30 Nov 2014 23:48:14 +0100 (CET)
Subject: [pypy-commit] creflect default: fix
Message-ID: <20141130224814.036F21C0296@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r133:352276036901
Date: 2014-11-30 23:48 +0100
http://bitbucket.org/cffi/creflect/changeset/352276036901/

Log:	fix

diff --git a/zeffir/zef_builder.c b/zeffir/zef_builder.c
--- a/zeffir/zef_builder.c
+++ b/zeffir/zef_builder.c
@@ -71,7 +71,7 @@
 }
 
 static _crx_type_t *zef_get_function_type(_crx_builder_t *cb, _crx_type_t *ret,
-                                          _crx_type_t *args[], int nargs,
+                                          _crx_qual_type args[], int nargs,
                                           _crx_trampoline1_fn trampl)
 {
     abort();
@@ -79,18 +79,14 @@
 
 static _crx_type_t *zef_get_ellipsis_function_type(_crx_builder_t *cb,
                                                    _crx_type_t *ret,
-                                                   _crx_type_t *args[],
+                                                   _crx_qual_type args[],
                                                    int nargs)
 {
     abort();
 }
 
-static _crx_type_t *zef_get_pointer_type(_crx_builder_t *cb, _crx_type_t *t)
-{
-    abort();
-}
-
-static _crx_type_t *zef_get_const_type(_crx_builder_t *cb, _crx_type_t *t)
+static _crx_type_t *zef_get_pointer_type(_crx_builder_t *cb,
+                                         _crx_type_t *totype, int toquals)
 {
     abort();
 }
@@ -122,7 +118,7 @@
     abort();
 }
 
-static _crx_type_t *zef_get_user_type(_crx_builder_t *cb, const char *name)
+static _crx_qual_type zef_get_user_type(_crx_builder_t *cb, const char *name)
 {
     abort();
 }
@@ -146,19 +142,19 @@
 }
 
 static void zef_define_type(_crx_builder_t *cb, const char *name,
-                            _crx_type_t *t)
+                            _crx_type_t *type, int quals)
 {
     abort();
 }
 
 static void zef_define_var(_crx_builder_t *cb, const char *name,
-                           _crx_type_t *t, void *addr)
+                           _crx_type_t *type, int quals, const void *addr)
 {
     abort();
 }
 
 static void zef_define_func(_crx_builder_t *cb, const char *name,
-                            _crx_type_t *ret, _crx_type_t *args[], int nargs,
+                            _crx_type_t *ret, _crx_qual_type args[], int nargs,
                             _crx_trampoline0_fn trampl, void *directcall)
 {
     abort();
@@ -219,7 +215,6 @@
             zef_get_function_type,
             zef_get_ellipsis_function_type,
             zef_get_pointer_type,
-            zef_get_const_type,
             zef_get_array_type,
             zef_get_incomplete_array_type,
             zef_get_struct_type,