From pypy.commits at gmail.com Fri Jul 1 01:41:38 2016 From: pypy.commits at gmail.com (mattip) Date: Thu, 30 Jun 2016 22:41:38 -0700 (PDT) Subject: [pypy-commit] pypy default: add a base type for ArrayType, make sure child nb_multiply is called Message-ID: <57760292.8323c20a.76358.2c23@mx.google.com> Author: Matti Picus Branch: Changeset: r85478:45fce19205cd Date: 2016-06-30 23:43 +0300 http://bitbucket.org/pypy/pypy/changeset/45fce19205cd/ Log: add a base type for ArrayType, make sure child nb_multiply is called diff --git a/pypy/module/cpyext/test/array.c b/pypy/module/cpyext/test/array.c --- a/pypy/module/cpyext/test/array.c +++ b/pypy/module/cpyext/test/array.c @@ -1908,6 +1908,60 @@ (binaryfunc)NULL, /* nb_divide */ }; +static PyObject* +array_base_multiply(PyObject* obj1, PyObject* obj2) +{ + if (PyList_Check(obj1) && ((arrayobject*)obj2)->ob_descr->typecode == 'i' && Py_SIZE(obj2) == 1) + { + int nn; + int n = PyList_Size(obj1); + PyObject *v = getarrayitem(obj2, 0); + int i = ((PyIntObject*)v)->ob_ival; + PyObject * ret = PyList_New(n); + for (nn = 0; nn < n; nn++) + { + v = PyList_GetItem(obj1, nn); + if (PyInt_Check(v)) + PyList_SetItem(ret, nn, PyLong_FromLong(i * ((PyIntObject*)v)->ob_ival)); + else + PyList_SetItem(ret, nn, v); + } + return ret; + } + else if (PyList_Check(obj2) && ((arrayobject*)obj1)->ob_descr->typecode == 'i' && Py_SIZE(obj1) == 1) + { + int nn; + int n = PyList_Size(obj2); + PyObject *v = getarrayitem(obj1, 0); + int i = ((PyIntObject*)v)->ob_ival; + PyObject * ret = PyList_New(n); + for (nn = 0; nn < n; nn++) + { + v = PyList_GetItem(obj2, nn); + if (PyInt_Check(v)) + PyList_SetItem(ret, nn, PyLong_FromLong(i * ((PyIntObject*)v)->ob_ival)); + else + PyList_SetItem(ret, nn, v); + } + return ret; + } + else if(obj1->ob_type == &Arraytype) + fprintf(stderr, "\nCannot multiply array of type %c and %s\n", + ((arrayobject*)obj1)->ob_descr->typecode, obj2->ob_type->tp_name); + else if(obj2->ob_type == &Arraytype) + fprintf(stderr, "\nCannot multiply array of type %c and %s\n", + ((arrayobject*)obj2)->ob_descr->typecode, obj1->ob_type->tp_name); + Py_INCREF(Py_NotImplemented); + return Py_NotImplemented; +} + +static PyNumberMethods array_base_as_number = { + (binaryfunc)NULL, /* nb_add*/ + (binaryfunc)NULL, /* nb_subtract */ + (binaryfunc)array_base_multiply, /* nb_multiply */ + (binaryfunc)NULL, /* nb_divide */ +}; + static PyMappingMethods array_as_mapping = { (lenfunc)array_length, (binaryfunc)array_subscr, @@ -2156,6 +2210,49 @@ static PyObject *array_iter(arrayobject *ao); +static PyTypeObject ArrayBasetype = { + PyVarObject_HEAD_INIT(NULL, 0) + "array.basearray", + sizeof(arrayobject), + 0, + (destructor)array_dealloc, /* tp_dealloc */ + 0, /* tp_print */ + 0, /* tp_getattr */ + 0, /* tp_setattr */ + 0, /* tp_compare */ + (reprfunc)array_repr, /* tp_repr */ + &array_base_as_number, /* tp_as_number*/ + &array_as_sequence, /* tp_as_sequence*/ + &array_as_mapping, /* tp_as_mapping*/ + 0, /* tp_hash */ + 0, /* tp_call */ + 0, /* tp_str */ + PyObject_GenericGetAttr, /* tp_getattro */ + 0, /* tp_setattro */ + &array_as_buffer, /* tp_as_buffer*/ + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | + Py_TPFLAGS_HAVE_WEAKREFS | Py_TPFLAGS_CHECKTYPES, /* tp_flags */ + arraytype_doc, /* tp_doc */ + 0, /* tp_traverse */ + 0, /* tp_clear */ + array_richcompare, /* tp_richcompare */ + offsetof(arrayobject, weakreflist), /* tp_weaklistoffset */ + (getiterfunc)array_iter, /* tp_iter */ + 0, /* tp_iternext */ + array_methods, /* tp_methods */ + 0, /* tp_members */ + array_getsets, /* tp_getset */ + 0, /* tp_base */ + 0, /* tp_dict */ + 0, /* tp_descr_get */ + 0, /* tp_descr_set */ + 0, /* tp_dictoffset */ + 0, /* tp_init */ + PyType_GenericAlloc, /* tp_alloc */ + array_new, /* tp_new */ + PyObject_Del, /* tp_free */ +}; + static PyTypeObject Arraytype = { PyVarObject_HEAD_INIT(NULL, 0) "array.array", @@ -2311,17 +2408,19 @@ { PyObject *m; + ArrayBasetype.ob_type = &PyType_Type; + Arraytype.tp_base = &ArrayBasetype; Arraytype.ob_type = &PyType_Type; PyArrayIter_Type.ob_type = &PyType_Type; m = Py_InitModule3("array", a_methods, module_doc); if (m == NULL) return; - Py_INCREF((PyObject *)&Arraytype); + if (PyType_Ready(&ArrayBasetype) < 0) + return; if (PyType_Ready(&Arraytype) < 0) return; PyModule_AddObject(m, "ArrayType", (PyObject *)&Arraytype); - Py_INCREF((PyObject *)&Arraytype); PyModule_AddObject(m, "array", (PyObject *)&Arraytype); - /* No need to check the error here, the caller will do that */ + PyModule_AddObject(m, "arraybase", (PyObject *)&ArrayBasetype); } From pypy.commits at gmail.com Fri Jul 1 01:41:40 2016 From: pypy.commits at gmail.com (mattip) Date: Thu, 30 Jun 2016 22:41:40 -0700 (PDT) Subject: [pypy-commit] pypy default: trivial test fixes Message-ID: <57760294.921f1c0a.b2d12.ffffad21@mx.google.com> Author: Matti Picus Branch: Changeset: r85479:f99e58901198 Date: 2016-07-01 08:39 +0300 http://bitbucket.org/pypy/pypy/changeset/f99e58901198/ Log: trivial test fixes diff --git a/pypy/objspace/fake/objspace.py b/pypy/objspace/fake/objspace.py --- a/pypy/objspace/fake/objspace.py +++ b/pypy/objspace/fake/objspace.py @@ -199,6 +199,9 @@ def newbytes(self, x): return w_some_obj() + def newunicode(self, x): + return w_some_obj() + def wrap(self, x): if not we_are_translated(): if isinstance(x, gateway.interp2app): diff --git a/rpython/rlib/test/test_rsocket.py b/rpython/rlib/test/test_rsocket.py --- a/rpython/rlib/test/test_rsocket.py +++ b/rpython/rlib/test/test_rsocket.py @@ -313,8 +313,10 @@ assert isinstance(lst, list) found = False for family, socktype, protocol, canonname, addr in lst: - if addr.get_host() == '104.130.43.121': + if addr.get_host() in ('104.130.43.121', '23.253.135.79'): found = True + elif family == AF_INET: + print 'pydotorg changed to', addr.get_host() result[i] += found def test_getaddrinfo_pydotorg(): From pypy.commits at gmail.com Fri Jul 1 04:17:38 2016 From: pypy.commits at gmail.com (plan_rich) Date: Fri, 01 Jul 2016 01:17:38 -0700 (PDT) Subject: [pypy-commit] pypy default: remove a little endian check. ctypes does not assign arrays in big endian order for int128_t (s390x) Message-ID: <57762722.871a1c0a.c1eab.74c3@mx.google.com> Author: Richard Plangger Branch: Changeset: r85480:e3bd32fc0036 Date: 2016-07-01 10:16 +0200 http://bitbucket.org/pypy/pypy/changeset/e3bd32fc0036/ Log: remove a little endian check. ctypes does not assign arrays in big endian order for int128_t (s390x) diff --git a/rpython/rtyper/lltypesystem/ll2ctypes.py b/rpython/rtyper/lltypesystem/ll2ctypes.py --- a/rpython/rtyper/lltypesystem/ll2ctypes.py +++ b/rpython/rtyper/lltypesystem/ll2ctypes.py @@ -171,10 +171,7 @@ _length_ = 2 @property def value(self): - if sys.byteorder == 'little': - res = self[0] | (self[1] << 64) - else: - res = self[1] | (self[0] << 64) + res = self[0] | (self[1] << 64) if res >= (1 << 127): res -= 1 << 128 return res From pypy.commits at gmail.com Fri Jul 1 04:37:29 2016 From: pypy.commits at gmail.com (plan_rich) Date: Fri, 01 Jul 2016 01:37:29 -0700 (PDT) Subject: [pypy-commit] pypy default: removed tests that call int_floordiv, this operation does not exist anymore (arm) Message-ID: <57762bc9.24f9c20a.d1a96.ffff9c50@mx.google.com> Author: Richard Plangger Branch: Changeset: r85481:518a6a527292 Date: 2016-07-01 10:36 +0200 http://bitbucket.org/pypy/pypy/changeset/518a6a527292/ Log: removed tests that call int_floordiv, this operation does not exist anymore (arm) diff --git a/rpython/jit/backend/arm/test/test_assembler.py b/rpython/jit/backend/arm/test/test_assembler.py --- a/rpython/jit/backend/arm/test/test_assembler.py +++ b/rpython/jit/backend/arm/test/test_assembler.py @@ -1,6 +1,5 @@ from rpython.jit.backend.arm import conditions as c from rpython.jit.backend.arm import registers as r -from rpython.jit.backend.arm.support import arm_int_div from rpython.jit.backend.arm.assembler import AssemblerARM from rpython.jit.backend.arm.locations import imm from rpython.jit.backend.arm.test.support import run_asm @@ -180,19 +179,6 @@ self.a.gen_func_epilog() assert run_asm(self.a) == 133 - def test_division(self): - self.a.gen_func_prolog() - self.a.mc.MOV_ri(r.r0.value, 123) - self.a.mc.MOV_ri(r.r1.value, 2) - - # call to div - self.a.mc.PUSH(range(2, 12)) - div_addr = rffi.cast(lltype.Signed, arm_int_div) - self.a.mc.BL(div_addr) - self.a.mc.POP(range(2, 12)) - self.a.gen_func_epilog() - assert run_asm(self.a) == 61 - def test_bl_with_conditional_exec(self): functype = lltype.Ptr(lltype.FuncType([lltype.Signed], lltype.Signed)) call_addr = rffi.cast(lltype.Signed, llhelper(functype, callme)) diff --git a/rpython/jit/backend/arm/test/test_regalloc.py b/rpython/jit/backend/arm/test/test_regalloc.py --- a/rpython/jit/backend/arm/test/test_regalloc.py +++ b/rpython/jit/backend/arm/test/test_regalloc.py @@ -545,23 +545,6 @@ self.interpret(ops, [s, 1234567890]) assert s[1] == 1234567890 - def test_division_optimized(self): - ops = ''' - [i7, i6] - label(i7, i6, descr=targettoken) - i18 = int_floordiv(i7, i6) - i19 = int_xor(i7, i6) - i21 = int_lt(i19, 0) - i22 = int_mod(i7, i6) - i23 = int_is_true(i22) - i24 = int_eq(i6, 4) - guard_false(i24) [i18] - jump(i18, i6, descr=targettoken) - ''' - self.interpret(ops, [10, 4]) - assert self.getint(0) == 2 - # FIXME: Verify that i19 - i23 are removed - class TestRegallocFloats(CustomBaseTestRegalloc): def test_float_add(self): From pypy.commits at gmail.com Fri Jul 1 04:42:35 2016 From: pypy.commits at gmail.com (plan_rich) Date: Fri, 01 Jul 2016 01:42:35 -0700 (PDT) Subject: [pypy-commit] pypy ppc-vsx-support: merge default Message-ID: <57762cfb.e409c20a.b5405.7ff2@mx.google.com> Author: Richard Plangger Branch: ppc-vsx-support Changeset: r85482:71642c531e12 Date: 2016-07-01 10:41 +0200 http://bitbucket.org/pypy/pypy/changeset/71642c531e12/ Log: merge default diff too long, truncating to 2000 out of 6953 lines diff --git a/pypy/doc/config/commandline.txt b/pypy/doc/config/commandline.txt --- a/pypy/doc/config/commandline.txt +++ b/pypy/doc/config/commandline.txt @@ -9,7 +9,7 @@ PyPy Python interpreter options ------------------------------- -The following options can be used after ``translate.py +The following options can be used after ``rpython targetpypystandalone`` or as options to ``py.py``. .. GENERATE: objspace @@ -22,7 +22,7 @@ General translation options --------------------------- -The following are options of ``translate.py``. They must be +The following are options of ``bin/rpython``. They must be given before the ``targetxxx`` on the command line. * `--opt -O:`__ set the optimization level `[0, 1, size, mem, 2, 3]` diff --git a/pypy/doc/config/index.rst b/pypy/doc/config/index.rst --- a/pypy/doc/config/index.rst +++ b/pypy/doc/config/index.rst @@ -15,12 +15,12 @@ ./py.py <`objspace options`_> -and the ``translate.py`` translation entry +and the ``rpython/bin/rpython`` translation entry point which takes arguments of this form: .. parsed-literal:: - ./translate.py <`translation options`_> + ./rpython/bin/rpython <`translation options`_> For the common case of ```` being ``targetpypystandalone.py``, you can then pass the `object space options`_ after @@ -28,7 +28,7 @@ .. parsed-literal:: - ./translate.py <`translation options`_> targetpypystandalone.py <`objspace options`_> + ./rpython/bin/rpython <`translation options`_> targetpypystandalone.py <`objspace options`_> There is an `overview`_ of all command line arguments that can be passed in either position. diff --git a/pypy/doc/config/opt.rst b/pypy/doc/config/opt.rst --- a/pypy/doc/config/opt.rst +++ b/pypy/doc/config/opt.rst @@ -4,8 +4,8 @@ This meta-option selects a default set of optimization settings to use during a translation. Usage:: - translate.py --opt=# - translate.py -O# + bin/rpython --opt=# + bin/rpython -O# where ``#`` is the desired optimization level. The valid choices are: diff --git a/pypy/doc/config/translation.dont_write_c_files.txt b/pypy/doc/config/translation.dont_write_c_files.txt --- a/pypy/doc/config/translation.dont_write_c_files.txt +++ b/pypy/doc/config/translation.dont_write_c_files.txt @@ -1,4 +1,4 @@ write the generated C files to ``/dev/null`` instead of to the disk. Useful if -you want to use translate.py as a benchmark and don't want to access the disk. +you want to use translation as a benchmark and don't want to access the disk. .. _`translation documentation`: ../translation.html diff --git a/pypy/doc/config/translation.fork_before.txt b/pypy/doc/config/translation.fork_before.txt --- a/pypy/doc/config/translation.fork_before.txt +++ b/pypy/doc/config/translation.fork_before.txt @@ -1,4 +1,4 @@ This is an option mostly useful when working on the PyPy toolchain. If you use -it, translate.py will fork before the specified phase. If the translation +it, translation will fork before the specified phase. If the translation crashes after that fork, you can fix the bug in the toolchain, and continue translation at the fork-point. diff --git a/pypy/doc/cppyy.rst b/pypy/doc/cppyy.rst --- a/pypy/doc/cppyy.rst +++ b/pypy/doc/cppyy.rst @@ -122,7 +122,7 @@ $ hg up reflex-support # optional # This example shows python, but using pypy-c is faster and uses less memory - $ python rpython/translator/goal/translate.py --opt=jit pypy/goal/targetpypystandalone --withmod-cppyy + $ python rpython/bin/rpython --opt=jit pypy/goal/targetpypystandalone --withmod-cppyy This will build a ``pypy-c`` that includes the cppyy module, and through that, Reflex support. diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -1,6 +1,6 @@ -========================= +========================== What's new in PyPy2.7 5.3+ -========================= +========================== .. this is a revision shortly after release-pypy2.7-v5.3 .. startrev: 873218a739f1 @@ -37,3 +37,19 @@ .. branch: pyfile-tell Sync w_file with the c-level FILE* before returning FILE* in PyFile_AsFile + +.. branch: rw-PyString_AS_STRING +Allow rw access to the char* returned from PyString_AS_STRING, also refactor +PyStringObject to look like cpython's and allow subclassing PyString_Type and +PyUnicode_Type + +.. branch: save_socket_errno + +Bug fix: if ``socket.socket()`` failed, the ``socket.error`` did not show +the errno of the failing system call, but instead some random previous +errno. + +.. branch: PyTuple_Type-subclass + +Refactor PyTupleObject to look like cpython's and allow subclassing +PyTuple_Type diff --git a/pypy/interpreter/astcompiler/tools/asdl_py.py b/pypy/interpreter/astcompiler/tools/asdl_py.py --- a/pypy/interpreter/astcompiler/tools/asdl_py.py +++ b/pypy/interpreter/astcompiler/tools/asdl_py.py @@ -400,7 +400,7 @@ if not (space.isinstance_w(w_obj, space.w_str) or space.isinstance_w(w_obj, space.w_unicode)): raise oefmt(space.w_TypeError, - "AST string must be of type str or unicode") + "AST string must be of type str or unicode") return w_obj def get_field(space, w_node, name, optional): diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py --- a/pypy/interpreter/baseobjspace.py +++ b/pypy/interpreter/baseobjspace.py @@ -1034,7 +1034,7 @@ return (None, None) def newlist_bytes(self, list_s): - return self.newlist([self.wrap(s) for s in list_s]) + return self.newlist([self.newbytes(s) for s in list_s]) def newlist_unicode(self, list_u): return self.newlist([self.wrap(u) for u in list_u]) @@ -1533,7 +1533,7 @@ # unclear if there is any use at all for getting the bytes in # the unicode buffer.) try: - return self.str_w(w_obj) + return self.bytes_w(w_obj) except OperationError as e: if not e.match(self, self.w_TypeError): raise diff --git a/pypy/interpreter/pycode.py b/pypy/interpreter/pycode.py --- a/pypy/interpreter/pycode.py +++ b/pypy/interpreter/pycode.py @@ -408,14 +408,14 @@ w(self.co_nlocals), w(self.co_stacksize), w(self.co_flags), - w(self.co_code), + space.newbytes(self.co_code), space.newtuple(self.co_consts_w), space.newtuple(self.co_names_w), space.newtuple([w(v) for v in self.co_varnames]), w(self.co_filename), w(self.co_name), w(self.co_firstlineno), - w(self.co_lnotab), + space.newbytes(self.co_lnotab), space.newtuple([w(v) for v in self.co_freevars]), space.newtuple([w(v) for v in self.co_cellvars]), w(self.magic), diff --git a/pypy/interpreter/pyparser/automata.py b/pypy/interpreter/pyparser/automata.py --- a/pypy/interpreter/pyparser/automata.py +++ b/pypy/interpreter/pyparser/automata.py @@ -13,12 +13,11 @@ # PYPY Modification: removed the EMPTY class as it's not needed here -# PYPY Modification: we don't need a particuliar DEFAULT class here -# a simple None works fine. -# (Having a DefaultClass inheriting from str makes -# the annotator crash) -DEFAULT = "\00default" # XXX hack, the rtyper does not support dict of with str|None keys - # anyway using dicts doesn't seem the best final way to store these char indexed tables +# PYPY Modification: DEFAULT is a singleton, used only in the pre-RPython +# dicts (see pytokenize.py). Then DFA.__init__() turns these dicts into +# more compact strings. +DEFAULT = object() + # PYPY Modification : removed all automata functions (any, maybe, # newArcPair, etc.) diff --git a/pypy/interpreter/pyparser/genpytokenize.py b/pypy/interpreter/pyparser/genpytokenize.py --- a/pypy/interpreter/pyparser/genpytokenize.py +++ b/pypy/interpreter/pyparser/genpytokenize.py @@ -293,7 +293,7 @@ i = 0 for k, v in sorted(state.items()): i += 1 - if k == '\x00default': + if k == DEFAULT: k = "automata.DEFAULT" else: k = repr(k) diff --git a/pypy/interpreter/pyparser/parsestring.py b/pypy/interpreter/pyparser/parsestring.py --- a/pypy/interpreter/pyparser/parsestring.py +++ b/pypy/interpreter/pyparser/parsestring.py @@ -81,7 +81,7 @@ if need_encoding: enc = encoding v = PyString_DecodeEscape(space, substr, 'strict', enc) - return space.wrap(v) + return space.newbytes(v) def decode_unicode_utf8(space, s, ps, q): # ****The Python 2.7 version, producing UTF-32 escapes**** diff --git a/pypy/module/_cffi_backend/ctypeprim.py b/pypy/module/_cffi_backend/ctypeprim.py --- a/pypy/module/_cffi_backend/ctypeprim.py +++ b/pypy/module/_cffi_backend/ctypeprim.py @@ -84,7 +84,7 @@ if self.size == 1: with cdataobj as ptr: s = ptr[0] - return self.space.wrap(s) + return self.space.newbytes(s) return W_CType.string(self, cdataobj, maxlen) def unpack_ptr(self, w_ctypeptr, ptr, length): @@ -126,12 +126,12 @@ return self.space.wrap(ord(cdata[0])) def convert_to_object(self, cdata): - return self.space.wrap(cdata[0]) + return self.space.newbytes(cdata[0]) def _convert_to_char(self, w_ob): space = self.space if space.isinstance_w(w_ob, space.w_str): - s = space.str_w(w_ob) + s = space.bytes_w(w_ob) if len(s) == 1: return s[0] if (isinstance(w_ob, cdataobj.W_CData) and @@ -146,7 +146,7 @@ def unpack_ptr(self, w_ctypeptr, ptr, length): s = rffi.charpsize2str(ptr, length) - return self.space.wrapbytes(s) + return self.space.newbytes(s) # XXX explicitly use an integer type instead of lltype.UniChar here, diff --git a/pypy/module/_cffi_backend/ctypeptr.py b/pypy/module/_cffi_backend/ctypeptr.py --- a/pypy/module/_cffi_backend/ctypeptr.py +++ b/pypy/module/_cffi_backend/ctypeptr.py @@ -120,7 +120,7 @@ s = rffi.charp2str(ptr) else: s = rffi.charp2strn(ptr, length) - return space.wrapbytes(s) + return space.newbytes(s) # # pointer to a wchar_t: builds and returns a unicode if self.is_unichar_ptr_or_array(): @@ -129,7 +129,7 @@ u = rffi.wcharp2unicode(cdata) else: u = rffi.wcharp2unicoden(cdata, length) - return space.wrap(u) + return space.newunicode(u) # return W_CType.string(self, cdataobj, maxlen) diff --git a/pypy/module/_codecs/interp_codecs.py b/pypy/module/_codecs/interp_codecs.py --- a/pypy/module/_codecs/interp_codecs.py +++ b/pypy/module/_codecs/interp_codecs.py @@ -36,12 +36,14 @@ w_errorhandler = lookup_error(space, errors) if decode: w_cls = space.w_UnicodeDecodeError + w_input = space.newbytes(input) else: w_cls = space.w_UnicodeEncodeError + w_input = space.newunicode(input) w_exc = space.call_function( w_cls, space.wrap(encoding), - space.wrap(input), + w_input, space.wrap(startpos), space.wrap(endpos), space.wrap(reason)) @@ -314,7 +316,7 @@ @unwrap_spec(errors='str_or_None') def readbuffer_encode(space, w_data, errors='strict'): s = space.getarg_w('s#', w_data) - return space.newtuple([space.wrap(s), space.wrap(len(s))]) + return space.newtuple([space.newbytes(s), space.wrap(len(s))]) @unwrap_spec(errors='str_or_None') def charbuffer_encode(space, w_data, errors='strict'): @@ -377,7 +379,7 @@ state = space.fromcache(CodecState) func = getattr(runicode, rname) result = func(uni, len(uni), errors, state.encode_error_handler) - return space.newtuple([space.wrap(result), space.wrap(len(uni))]) + return space.newtuple([space.newbytes(result), space.wrap(len(uni))]) wrap_encoder.func_name = rname globals()[name] = wrap_encoder @@ -398,7 +400,7 @@ wrap_decoder.func_name = rname globals()[name] = wrap_decoder -for encoders in [ +for encoder in [ "ascii_encode", "latin_1_encode", "utf_7_encode", @@ -412,9 +414,9 @@ "raw_unicode_escape_encode", "unicode_internal_encode", ]: - make_encoder_wrapper(encoders) + make_encoder_wrapper(encoder) -for decoders in [ +for decoder in [ "ascii_decode", "latin_1_decode", "utf_7_decode", @@ -426,7 +428,7 @@ "utf_32_le_decode", "raw_unicode_escape_decode", ]: - make_decoder_wrapper(decoders) + make_decoder_wrapper(decoder) if hasattr(runicode, 'str_decode_mbcs'): make_encoder_wrapper('mbcs_encode') @@ -560,7 +562,7 @@ if space.isinstance_w(w_ch, space.w_str): # Charmap may return a string - return space.str_w(w_ch) + return space.bytes_w(w_ch) elif space.isinstance_w(w_ch, space.w_int): # Charmap may return a number x = space.int_w(w_ch) @@ -608,7 +610,7 @@ result = runicode.unicode_encode_charmap( uni, len(uni), errors, state.encode_error_handler, mapping) - return space.newtuple([space.wrap(result), space.wrap(len(uni))]) + return space.newtuple([space.newbytes(result), space.wrap(len(uni))]) @unwrap_spec(chars=unicode) @@ -696,4 +698,4 @@ def escape_decode(space, data, errors='strict'): from pypy.interpreter.pyparser.parsestring import PyString_DecodeEscape result = PyString_DecodeEscape(space, data, errors, None) - return space.newtuple([space.wrap(result), space.wrap(len(data))]) + return space.newtuple([space.newbytes(result), space.wrap(len(data))]) diff --git a/pypy/module/_hashlib/interp_hashlib.py b/pypy/module/_hashlib/interp_hashlib.py --- a/pypy/module/_hashlib/interp_hashlib.py +++ b/pypy/module/_hashlib/interp_hashlib.py @@ -111,7 +111,7 @@ def digest(self, space): "Return the digest value as a string of binary data." digest = self._digest(space) - return space.wrap(digest) + return space.newbytes(digest) def hexdigest(self, space): "Return the digest value as a string of hexadecimal digits." diff --git a/pypy/module/_io/interp_bufferedio.py b/pypy/module/_io/interp_bufferedio.py --- a/pypy/module/_io/interp_bufferedio.py +++ b/pypy/module/_io/interp_bufferedio.py @@ -343,7 +343,7 @@ self._writer_reset_buf() def _write(self, space, data): - w_data = space.wrap(data) + w_data = space.newbytes(data) while True: try: w_written = space.call_method(self.w_raw, "write", w_data) @@ -415,7 +415,7 @@ else: raise oefmt(space.w_ValueError, "read length must be positive or -1") - return space.wrap(res) + return space.newbytes(res) @unwrap_spec(size=int) def peek_w(self, space, size=0): @@ -432,7 +432,7 @@ have = self._readahead() if have > 0: data = ''.join(self.buffer[self.pos:self.pos+have]) - return space.wrap(data) + return space.newbytes(data) # Fill the buffer from the raw stream, and copy it to the result self._reader_reset_buf() @@ -442,7 +442,7 @@ size = 0 self.pos = 0 data = ''.join(self.buffer[:size]) - return space.wrap(data) + return space.newbytes(data) @unwrap_spec(size=int) def read1_w(self, space, size): @@ -452,7 +452,7 @@ if size < 0: raise oefmt(space.w_ValueError, "read length must be positive") if size == 0: - return space.wrap("") + return space.newbytes("") with self.lock: # Return up to n bytes. If at least one byte is buffered, we only @@ -480,7 +480,7 @@ endpos = self.pos + size data = ''.join(self.buffer[self.pos:endpos]) self.pos = endpos - return space.wrap(data) + return space.newbytes(data) def _read_all(self, space): "Read all the file, don't update the cache" @@ -505,7 +505,7 @@ if current_size == 0: return w_data break - data = space.str_w(w_data) + data = space.bytes_w(w_data) size = len(data) if size == 0: break @@ -513,7 +513,7 @@ current_size += size if self.abs_pos != -1: self.abs_pos += size - return space.wrap(builder.build()) + return space.newbytes(builder.build()) def _raw_read(self, space, buffer, start, length): length = intmask(length) @@ -644,11 +644,11 @@ else: pos = -1 if pos >= 0: - w_res = space.wrap(''.join(self.buffer[self.pos:pos+1])) + w_res = space.newbytes(''.join(self.buffer[self.pos:pos+1])) self.pos = pos + 1 return w_res if have == limit: - w_res = space.wrap(''.join(self.buffer[self.pos:self.pos+have])) + w_res = space.newbytes(''.join(self.buffer[self.pos:self.pos+have])) self.pos += have return w_res @@ -690,7 +690,7 @@ written += have if limit >= 0: limit -= have - return space.wrap(''.join(chunks)) + return space.newbytes(''.join(chunks)) # ____________________________________________________ # Write methods @@ -1024,7 +1024,6 @@ self._deprecated_max_buffer_size(space) self.state = STATE_ZERO - check_readable_w(space, w_raw) check_writable_w(space, w_raw) check_seekable_w(space, w_raw) diff --git a/pypy/module/_io/interp_bytesio.py b/pypy/module/_io/interp_bytesio.py --- a/pypy/module/_io/interp_bytesio.py +++ b/pypy/module/_io/interp_bytesio.py @@ -32,12 +32,12 @@ def read_w(self, space, w_size=None): self._check_closed(space) size = convert_size(space, w_size) - return space.wrap(self.read(size)) + return space.newbytes(self.read(size)) def readline_w(self, space, w_limit=None): self._check_closed(space) limit = convert_size(space, w_limit) - return space.wrap(self.readline(limit)) + return space.newbytes(self.readline(limit)) def read1_w(self, space, w_size): return self.read_w(space, w_size) @@ -81,7 +81,7 @@ def getvalue_w(self, space): self._check_closed(space) - return space.wrap(self.getvalue()) + return space.newbytes(self.getvalue()) def tell_w(self, space): self._check_closed(space) @@ -128,7 +128,7 @@ def getstate_w(self, space): self._check_closed(space) return space.newtuple([ - space.wrap(self.getvalue()), + space.newbytes(self.getvalue()), space.wrap(self.tell()), self.getdict(space)]) diff --git a/pypy/module/_io/interp_fileio.py b/pypy/module/_io/interp_fileio.py --- a/pypy/module/_io/interp_fileio.py +++ b/pypy/module/_io/interp_fileio.py @@ -361,7 +361,7 @@ raise wrap_oserror(space, e, exception_name='w_IOError') - return space.wrap(s) + return space.newbytes(s) def readinto_w(self, space, w_buffer): self._check_closed(space) @@ -405,7 +405,7 @@ break builder.append(chunk) total += len(chunk) - return space.wrap(builder.build()) + return space.newbytes(builder.build()) if sys.platform == "win32": def _truncate(self, size): diff --git a/pypy/module/_io/interp_iobase.py b/pypy/module/_io/interp_iobase.py --- a/pypy/module/_io/interp_iobase.py +++ b/pypy/module/_io/interp_iobase.py @@ -192,7 +192,7 @@ length = space.len_w(w_readahead) if length > 0: n = 0 - buf = space.str_w(w_readahead) + buf = space.bytes_w(w_readahead) if limit >= 0: while True: if n >= length or n >= limit: @@ -219,7 +219,7 @@ raise oefmt(space.w_IOError, "peek() should have returned a bytes object, not " "'%T'", w_read) - read = space.str_w(w_read) + read = space.bytes_w(w_read) if not read: break @@ -229,7 +229,7 @@ if read[-1] == '\n': break - return space.wrap(builder.build()) + return space.newbytes(builder.build()) def readlines_w(self, space, w_hint=None): hint = convert_size(space, w_hint) @@ -339,11 +339,11 @@ if not space.isinstance_w(w_data, space.w_str): raise oefmt(space.w_TypeError, "read() should return bytes") - data = space.str_w(w_data) + data = space.bytes_w(w_data) if not data: break builder.append(data) - return space.wrap(builder.build()) + return space.newbytes(builder.build()) W_RawIOBase.typedef = TypeDef( '_io._RawIOBase', W_IOBase.typedef, diff --git a/pypy/module/_io/interp_textio.py b/pypy/module/_io/interp_textio.py --- a/pypy/module/_io/interp_textio.py +++ b/pypy/module/_io/interp_textio.py @@ -160,7 +160,7 @@ w_buffer, w_flag = space.unpackiterable(w_state, 2) flag = space.r_longlong_w(w_flag) else: - w_buffer = space.wrap("") + w_buffer = space.newbytes("") flag = 0 flag <<= 1 if self.pendingcr: @@ -556,7 +556,7 @@ # Given this, we know there was a valid snapshot point # len(dec_buffer) bytes ago with decoder state (b'', dec_flags). w_dec_buffer, w_dec_flags = space.unpackiterable(w_state, 2) - dec_buffer = space.str_w(w_dec_buffer) + dec_buffer = space.bytes_w(w_dec_buffer) dec_flags = space.int_w(w_dec_flags) else: dec_buffer = None @@ -582,7 +582,7 @@ if self.telling: # At the snapshot point, len(dec_buffer) bytes before the read, # the next input to be decoded is dec_buffer + input_chunk. - next_input = dec_buffer + space.str_w(w_input) + next_input = dec_buffer + space.bytes_w(w_input) self.snapshot = PositionSnapshot(dec_flags, next_input) return not eof @@ -769,7 +769,7 @@ else: w_bytes = space.call_method(self.w_encoder, "encode", w_text) - b = space.str_w(w_bytes) + b = space.bytes_w(w_bytes) if not self.pending_bytes: self.pending_bytes = [] self.pending_bytes_count = 0 @@ -799,7 +799,8 @@ while True: try: - space.call_method(self.w_buffer, "write", space.wrap(pending_bytes)) + space.call_method(self.w_buffer, "write", + space.newbytes(pending_bytes)) except OperationError as e: if trap_eintr(space, e): continue @@ -828,7 +829,7 @@ space.call_method(self.w_decoder, "reset") else: space.call_method(self.w_decoder, "setstate", - space.newtuple([space.wrap(""), + space.newtuple([space.newbytes(""), space.wrap(cookie.dec_flags)])) def _encoder_setstate(self, space, cookie): @@ -904,7 +905,7 @@ raise oefmt(space.w_TypeError, msg, w_chunk) self.snapshot = PositionSnapshot(cookie.dec_flags, - space.str_w(w_chunk)) + space.bytes_w(w_chunk)) w_decoded = space.call_method(self.w_decoder, "decode", w_chunk, space.wrap(cookie.need_eof)) @@ -975,7 +976,7 @@ i = 0 while i < len(input): w_decoded = space.call_method(self.w_decoder, "decode", - space.wrap(input[i])) + space.newbytes(input[i])) check_decoded(space, w_decoded) chars_decoded += len(space.unicode_w(w_decoded)) diff --git a/pypy/module/_md5/interp_md5.py b/pypy/module/_md5/interp_md5.py --- a/pypy/module/_md5/interp_md5.py +++ b/pypy/module/_md5/interp_md5.py @@ -20,7 +20,7 @@ self.update(string) def digest_w(self): - return self.space.wrap(self.digest()) + return self.space.newbytes(self.digest()) def hexdigest_w(self): return self.space.wrap(self.hexdigest()) diff --git a/pypy/module/_minimal_curses/interp_curses.py b/pypy/module/_minimal_curses/interp_curses.py --- a/pypy/module/_minimal_curses/interp_curses.py +++ b/pypy/module/_minimal_curses/interp_curses.py @@ -83,12 +83,12 @@ return space.w_None except curses_error as e: raise convert_error(space, e) - return space.wrap(result) + return space.newbytes(result) @unwrap_spec(s=str) def tparm(space, s, args_w): args = [space.int_w(a) for a in args_w] try: - return space.wrap(_curses_tparm(s, args)) + return space.newbytes(_curses_tparm(s, args)) except curses_error as e: raise convert_error(space, e) diff --git a/pypy/module/_multibytecodec/interp_incremental.py b/pypy/module/_multibytecodec/interp_incremental.py --- a/pypy/module/_multibytecodec/interp_incremental.py +++ b/pypy/module/_multibytecodec/interp_incremental.py @@ -113,7 +113,7 @@ pos = c_codecs.pypy_cjk_enc_inbuf_consumed(self.encodebuf) assert 0 <= pos <= len(object) self.pending = object[pos:] - return space.wrap(output) + return space.newbytes(output) @unwrap_spec(errors="str_or_None") diff --git a/pypy/module/_multibytecodec/interp_multibytecodec.py b/pypy/module/_multibytecodec/interp_multibytecodec.py --- a/pypy/module/_multibytecodec/interp_multibytecodec.py +++ b/pypy/module/_multibytecodec/interp_multibytecodec.py @@ -40,7 +40,7 @@ raise wrap_unicodeencodeerror(space, e, input, self.name) except RuntimeError: raise wrap_runtimeerror(space) - return space.newtuple([space.wrap(output), + return space.newtuple([space.newbytes(output), space.wrap(len(input))]) @@ -66,7 +66,7 @@ space.w_UnicodeDecodeError, space.newtuple([ space.wrap(name), - space.wrap(input), + space.newbytes(input), space.wrap(e.start), space.wrap(e.end), space.wrap(e.reason)])) diff --git a/pypy/module/_multiprocessing/interp_connection.py b/pypy/module/_multiprocessing/interp_connection.py --- a/pypy/module/_multiprocessing/interp_connection.py +++ b/pypy/module/_multiprocessing/interp_connection.py @@ -122,9 +122,9 @@ space, self.BUFFER_SIZE, maxlength) try: if newbuf: - return space.wrap(rffi.charpsize2str(newbuf, res)) + return space.newbytes(rffi.charpsize2str(newbuf, res)) else: - return space.wrap(rffi.charpsize2str(self.buffer, res)) + return space.newbytes(rffi.charpsize2str(self.buffer, res)) finally: if newbuf: rffi.free_charp(newbuf) @@ -138,7 +138,7 @@ space, length - offset, PY_SSIZE_T_MAX) try: if newbuf: - raise BufferTooShort(space, space.wrap( + raise BufferTooShort(space, space.newbytes( rffi.charpsize2str(newbuf, res))) rwbuffer.setslice(offset, rffi.charpsize2str(self.buffer, res)) finally: @@ -166,9 +166,9 @@ space, self.BUFFER_SIZE, PY_SSIZE_T_MAX) try: if newbuf: - w_received = space.wrap(rffi.charpsize2str(newbuf, res)) + w_received = space.newbytes(rffi.charpsize2str(newbuf, res)) else: - w_received = space.wrap(rffi.charpsize2str(self.buffer, res)) + w_received = space.newbytes(rffi.charpsize2str(self.buffer, res)) finally: if newbuf: rffi.free_charp(newbuf) diff --git a/pypy/module/_rawffi/alt/test/test_type_converter.py b/pypy/module/_rawffi/alt/test/test_type_converter.py --- a/pypy/module/_rawffi/alt/test/test_type_converter.py +++ b/pypy/module/_rawffi/alt/test/test_type_converter.py @@ -12,14 +12,14 @@ handle_signed = handle_all handle_unsigned = handle_all handle_pointer = handle_all - handle_char = handle_all + handle_char = handle_all handle_unichar = handle_all handle_longlong = handle_all handle_char_p = handle_all handle_unichar_p = handle_all handle_float = handle_all handle_singlefloat = handle_all - + def handle_struct(self, w_ffitype, w_structinstance): self.lastval = w_structinstance @@ -119,12 +119,12 @@ def test_strings(self): # first, try automatic conversion from applevel - self.check(app_types.char_p, self.space.wrap('foo'), 'foo') - self.check(app_types.unichar_p, self.space.wrap(u'foo\u1234'), u'foo\u1234') - self.check(app_types.unichar_p, self.space.wrap('foo'), u'foo') + self.check(app_types.char_p, self.space.newbytes('foo'), 'foo') + self.check(app_types.unichar_p, self.space.wrap(u'foo\u1234'), u'foo\u1234') + self.check(app_types.unichar_p, self.space.wrap('foo'), u'foo') # then, try to pass explicit pointers self.check(app_types.char_p, self.space.wrap(42), 42) - self.check(app_types.unichar_p, self.space.wrap(42), 42) + self.check(app_types.unichar_p, self.space.wrap(42), 42) @@ -136,7 +136,7 @@ get_signed = get_all get_unsigned = get_all get_pointer = get_all - get_char = get_all + get_char = get_all get_unichar = get_all get_longlong = get_all get_char_p = get_all @@ -144,7 +144,7 @@ get_float = get_all get_singlefloat = get_all get_unsigned_which_fits_into_a_signed = get_all - + def convert(self, w_ffitype, val): self.val = val return self.do_and_wrap(w_ffitype) diff --git a/pypy/module/_rawffi/array.py b/pypy/module/_rawffi/array.py --- a/pypy/module/_rawffi/array.py +++ b/pypy/module/_rawffi/array.py @@ -181,7 +181,7 @@ start, stop = self.decodeslice(space, w_slice) ll_buffer = self.ll_buffer result = [ll_buffer[i] for i in range(start, stop)] - return space.wrap(''.join(result)) + return space.newbytes(''.join(result)) def setslice(self, space, w_slice, w_value): start, stop = self.decodeslice(space, w_slice) diff --git a/pypy/module/_rawffi/interp_rawffi.py b/pypy/module/_rawffi/interp_rawffi.py --- a/pypy/module/_rawffi/interp_rawffi.py +++ b/pypy/module/_rawffi/interp_rawffi.py @@ -570,7 +570,7 @@ s = rffi.charp2str(charp_addr) else: s = rffi.charp2strn(charp_addr, maxlength) - return space.wrap(s) + return space.newbytes(s) @unwrap_spec(address=r_uint, maxlength=int) def wcharp2unicode(space, address, maxlength=-1): @@ -588,7 +588,7 @@ if maxlength == -1: return charp2string(space, address) s = rffi.charpsize2str(rffi.cast(rffi.CCHARP, address), maxlength) - return space.wrap(s) + return space.newbytes(s) @unwrap_spec(address=r_uint, maxlength=int) def wcharp2rawunicode(space, address, maxlength=-1): diff --git a/pypy/module/_socket/interp_func.py b/pypy/module/_socket/interp_func.py --- a/pypy/module/_socket/interp_func.py +++ b/pypy/module/_socket/interp_func.py @@ -209,7 +209,7 @@ buf = rsocket.inet_aton(ip) except SocketError as e: raise converted_error(space, e) - return space.wrap(buf) + return space.newbytes(buf) @unwrap_spec(packed=str) def inet_ntoa(space, packed): @@ -234,7 +234,7 @@ buf = rsocket.inet_pton(family, ip) except SocketError as e: raise converted_error(space, e) - return space.wrap(buf) + return space.newbytes(buf) @unwrap_spec(family=int, packed=str) def inet_ntop(space, family, packed): @@ -263,10 +263,10 @@ if space.is_w(w_host, space.w_None): host = None elif space.isinstance_w(w_host, space.w_str): - host = space.str_w(w_host) + host = space.bytes_w(w_host) elif space.isinstance_w(w_host, space.w_unicode): w_shost = space.call_method(w_host, "encode", space.wrap("idna")) - host = space.str_w(w_shost) + host = space.bytes_w(w_shost) else: raise oefmt(space.w_TypeError, "getaddrinfo() argument 1 must be string or None") @@ -277,7 +277,7 @@ elif space.isinstance_w(w_port, space.w_int) or space.isinstance_w(w_port, space.w_long): port = str(space.int_w(w_port)) elif space.isinstance_w(w_port, space.w_str): - port = space.str_w(w_port) + port = space.bytes_w(w_port) else: raise oefmt(space.w_TypeError, "getaddrinfo() argument 2 must be integer or string") diff --git a/pypy/module/_socket/interp_socket.py b/pypy/module/_socket/interp_socket.py --- a/pypy/module/_socket/interp_socket.py +++ b/pypy/module/_socket/interp_socket.py @@ -296,7 +296,7 @@ except SocketError as e: raise converted_error(space, e) buflen = space.int_w(w_buflen) - return space.wrap(self.sock.getsockopt(level, optname, buflen)) + return space.newbytes(self.sock.getsockopt(level, optname, buflen)) def gettimeout_w(self, space): """gettimeout() -> timeout @@ -345,7 +345,7 @@ data = self.sock.recv(buffersize, flags) except SocketError as e: raise converted_error(space, e) - return space.wrap(data) + return space.newbytes(data) @unwrap_spec(buffersize='nonnegint', flags=int) def recvfrom_w(self, space, buffersize, flags=0): @@ -359,7 +359,7 @@ w_addr = addr_as_object(addr, self.sock.fd, space) else: w_addr = space.w_None - return space.newtuple([space.wrap(data), w_addr]) + return space.newtuple([space.newbytes(data), w_addr]) except SocketError as e: raise converted_error(space, e) @@ -436,7 +436,7 @@ except OperationError as e: if e.async(space): raise - optval = space.str_w(w_optval) + optval = space.bytes_w(w_optval) try: self.sock.setsockopt(level, optname, optval) except SocketError as e: diff --git a/pypy/module/_sre/interp_sre.py b/pypy/module/_sre/interp_sre.py --- a/pypy/module/_sre/interp_sre.py +++ b/pypy/module/_sre/interp_sre.py @@ -36,11 +36,12 @@ def slice_w(space, ctx, start, end, w_default): if 0 <= start <= end: if isinstance(ctx, rsre_core.BufMatchContext): - return space.wrap(ctx._buffer.getslice(start, end, 1, end-start)) + return space.newbytes(ctx._buffer.getslice(start, end, 1, + end-start)) if isinstance(ctx, rsre_core.StrMatchContext): - return space.wrap(ctx._string[start:end]) + return space.newbytes(ctx._string[start:end]) elif isinstance(ctx, rsre_core.UnicodeMatchContext): - return space.wrap(ctx._unicodestr[start:end]) + return space.newunicode(ctx._unicodestr[start:end]) else: # unreachable raise SystemError @@ -242,7 +243,7 @@ space.isinstance_w(w_string, space.w_unicode) and literal) else: try: - filter_as_string = space.str_w(w_ptemplate) + filter_as_string = space.bytes_w(w_ptemplate) except OperationError as e: if e.async(space): raise @@ -331,15 +332,15 @@ strbuilder, unicodebuilder, last_pos, ctx.end) if use_builder: if strbuilder is not None: - return space.wrap(strbuilder.build()), n + return space.newbytes(strbuilder.build()), n else: assert unicodebuilder is not None - return space.wrap(unicodebuilder.build()), n + return space.newunicode(unicodebuilder.build()), n else: if space.isinstance_w(w_string, space.w_unicode): - w_emptystr = space.wrap(u'') + w_emptystr = space.newunicode(u'') else: - w_emptystr = space.wrap('') + w_emptystr = space.newbytes('') w_item = space.call_method(w_emptystr, 'join', space.newlist(sublist_w)) return w_item, n @@ -565,11 +566,11 @@ def fget_string(self, space): ctx = self.ctx if isinstance(ctx, rsre_core.BufMatchContext): - return space.wrap(ctx._buffer.as_str()) + return space.newbytes(ctx._buffer.as_str()) elif isinstance(ctx, rsre_core.StrMatchContext): - return space.wrap(ctx._string) + return space.newbytes(ctx._string) elif isinstance(ctx, rsre_core.UnicodeMatchContext): - return space.wrap(ctx._unicodestr) + return space.newunicode(ctx._unicodestr) else: raise SystemError diff --git a/pypy/module/_ssl/interp_ssl.py b/pypy/module/_ssl/interp_ssl.py --- a/pypy/module/_ssl/interp_ssl.py +++ b/pypy/module/_ssl/interp_ssl.py @@ -666,7 +666,7 @@ length = libssl_SSL_get_peer_finished(self.ssl, buf, CB_MAXLEN) if length > 0: - return space.wrap(rffi.charpsize2str(buf, intmask(length))) + return space.newbytes(rffi.charpsize2str(buf, intmask(length))) def descr_get_context(self, space): return self.w_ctx @@ -707,7 +707,7 @@ if length < 0: raise _ssl_seterror(space, None, 0) try: - return space.wrap(rffi.charpsize2str(buf_ptr[0], length)) + return space.newbytes(rffi.charpsize2str(buf_ptr[0], length)) finally: libssl_OPENSSL_free(buf_ptr[0]) @@ -926,7 +926,7 @@ if length < 0: raise _ssl_seterror(space, None, 0) try: - w_value = space.wrap(rffi.charpsize2str(buf_ptr[0], length)) + w_value = space.newbytes(rffi.charpsize2str(buf_ptr[0], length)) w_value = space.call_method(w_value, "decode", space.wrap("utf-8")) finally: libssl_OPENSSL_free(buf_ptr[0]) @@ -1232,7 +1232,7 @@ w_ssl_socket, space.w_None, w_ctx) else: - w_servername = space.wrapbytes(rffi.charp2str(servername)) + w_servername = space.newbytes(rffi.charp2str(servername)) try: w_servername_idna = space.call_method( w_servername, 'decode', space.wrap('idna')) @@ -1778,7 +1778,7 @@ if not path: return space.w_None else: - return space.wrapbytes(rffi.charp2str(path)) + return space.newbytes(rffi.charp2str(path)) def get_default_verify_paths(space): return space.newtuple([ diff --git a/pypy/module/_ssl/interp_win32.py b/pypy/module/_ssl/interp_win32.py --- a/pypy/module/_ssl/interp_win32.py +++ b/pypy/module/_ssl/interp_win32.py @@ -74,7 +74,7 @@ def w_parseKeyUsage(space, pCertCtx, flags): with lltype.scoped_alloc(rwin32.LPDWORD.TO, 1) as size_ptr: - if not CertGetEnhancedKeyUsage(pCertCtx, flags, + if not CertGetEnhancedKeyUsage(pCertCtx, flags, lltype.nullptr(CERT_ENHKEY_USAGE), size_ptr): last_error = rwin32.lastSavedWindowsError() if last_error.winerror == CRYPT_E_NOT_FOUND: @@ -120,7 +120,7 @@ pCertCtx = CertEnumCertificatesInStore(hStore, pCertCtx) if not pCertCtx: break - w_cert = space.wrapbytes( + w_cert = space.newbytes( rffi.charpsize2str(pCertCtx.c_pbCertEncoded, intmask(pCertCtx.c_cbCertEncoded))) w_enc = w_certEncodingType(space, pCertCtx.c_dwCertEncodingType) @@ -162,7 +162,7 @@ pCrlCtx = CertEnumCRLsInStore(hStore, pCrlCtx) if not pCrlCtx: break - w_crl = space.wrapbytes( + w_crl = space.newbytes( rffi.charpsize2str(pCrlCtx.c_pbCrlEncoded, intmask(pCrlCtx.c_cbCrlEncoded))) w_enc = w_certEncodingType(space, pCrlCtx.c_dwCertEncodingType) diff --git a/pypy/module/_winreg/interp_winreg.py b/pypy/module/_winreg/interp_winreg.py --- a/pypy/module/_winreg/interp_winreg.py +++ b/pypy/module/_winreg/interp_winreg.py @@ -380,7 +380,7 @@ return space.newlist(l) else: # REG_BINARY and all other types - return space.wrap(rffi.charpsize2str(buf, buflen)) + return space.newbytes(rffi.charpsize2str(buf, buflen)) @unwrap_spec(value_name=str, typ=int) def SetValueEx(space, w_hkey, value_name, w_reserved, typ, w_value): diff --git a/pypy/module/array/interp_array.py b/pypy/module/array/interp_array.py --- a/pypy/module/array/interp_array.py +++ b/pypy/module/array/interp_array.py @@ -225,11 +225,11 @@ """ size = self.len if size == 0: - return space.wrap('') + return space.newbytes('') cbuf = self._charbuf_start() s = rffi.charpsize2str(cbuf, size * self.itemsize) self._charbuf_stop() - return self.space.wrap(s) + return self.space.newbytes(s) def descr_fromstring(self, space, w_s): """ fromstring(string) @@ -263,7 +263,7 @@ except OverflowError: raise MemoryError w_item = space.call_method(w_f, 'read', space.wrap(size)) - item = space.str_w(w_item) + item = space.bytes_w(w_item) if len(item) < size: n = len(item) % self.itemsize elems = max(0, len(item) - (len(item) % self.itemsize)) @@ -338,10 +338,10 @@ else: args = [space.wrap(self.typecode)] try: - dct = space.getattr(self, space.wrap('__dict__')) + w_dict = space.getattr(self, space.wrap('__dict__')) except OperationError: - dct = space.w_None - return space.newtuple([space.type(self), space.newtuple(args), dct]) + w_dict = space.w_None + return space.newtuple([space.type(self), space.newtuple(args), w_dict]) def descr_copy(self, space): """ copy(array) diff --git a/pypy/module/binascii/interp_base64.py b/pypy/module/binascii/interp_base64.py --- a/pypy/module/binascii/interp_base64.py +++ b/pypy/module/binascii/interp_base64.py @@ -71,7 +71,7 @@ if leftbits != 0: raise_Error(space, "Incorrect padding") - return space.wrap(res.build()) + return space.newbytes(res.build()) # ____________________________________________________________ @@ -110,4 +110,4 @@ res.append(table_b2a_base64[(leftchar & 0xf) << 2]) res.append(PAD) res.append('\n') - return space.wrap(res.build()) + return space.newbytes(res.build()) diff --git a/pypy/module/binascii/interp_hexlify.py b/pypy/module/binascii/interp_hexlify.py --- a/pypy/module/binascii/interp_hexlify.py +++ b/pypy/module/binascii/interp_hexlify.py @@ -24,7 +24,7 @@ for c in data: res.append(_value2char(ord(c) >> 4)) res.append(_value2char(ord(c) & 0xf)) - return space.wrap(res.build()) + return space.newbytes(res.build()) # ____________________________________________________________ @@ -53,4 +53,4 @@ a = _char2value(space, hexstr[i]) b = _char2value(space, hexstr[i+1]) res.append(chr((a << 4) | b)) - return space.wrap(res.build()) + return space.newbytes(res.build()) diff --git a/pypy/module/binascii/interp_hqx.py b/pypy/module/binascii/interp_hqx.py --- a/pypy/module/binascii/interp_hqx.py +++ b/pypy/module/binascii/interp_hqx.py @@ -11,37 +11,37 @@ FAIL = 0x7d table_a2b_hqx = [ - #^@ ^A ^B ^C ^D ^E ^F ^G + #^@ ^A ^B ^C ^D ^E ^F ^G FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, - #\b \t \n ^K ^L \r ^N ^O + #\b \t \n ^K ^L \r ^N ^O FAIL, FAIL, SKIP, FAIL, FAIL, SKIP, FAIL, FAIL, - #^P ^Q ^R ^S ^T ^U ^V ^W + #^P ^Q ^R ^S ^T ^U ^V ^W FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, - #^X ^Y ^Z ^[ ^\ ^] ^^ ^_ + #^X ^Y ^Z ^[ ^\ ^] ^^ ^_ FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, - # ! " # $ % & ' + # ! " # $ % & ' FAIL, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, - #( ) * + , - . / + #( ) * + , - . / 0x07, 0x08, 0x09, 0x0A, 0x0B, 0x0C, FAIL, FAIL, - #0 1 2 3 4 5 6 7 + #0 1 2 3 4 5 6 7 0x0D, 0x0E, 0x0F, 0x10, 0x11, 0x12, 0x13, FAIL, - #8 9 : ; < = > ? + #8 9 : ; < = > ? 0x14, 0x15, DONE, FAIL, FAIL, FAIL, FAIL, FAIL, - #@ A B C D E F G + #@ A B C D E F G 0x16, 0x17, 0x18, 0x19, 0x1A, 0x1B, 0x1C, 0x1D, - #H I J K L M N O + #H I J K L M N O 0x1E, 0x1F, 0x20, 0x21, 0x22, 0x23, 0x24, FAIL, - #P Q R S T U V W + #P Q R S T U V W 0x25, 0x26, 0x27, 0x28, 0x29, 0x2A, 0x2B, FAIL, - #X Y Z [ \ ] ^ _ + #X Y Z [ \ ] ^ _ 0x2C, 0x2D, 0x2E, 0x2F, FAIL, FAIL, FAIL, FAIL, - #` a b c d e f g + #` a b c d e f g 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, FAIL, - #h i j k l m n o + #h i j k l m n o 0x37, 0x38, 0x39, 0x3A, 0x3B, 0x3C, FAIL, FAIL, - #p q r s t u v w + #p q r s t u v w 0x3D, 0x3E, 0x3F, FAIL, FAIL, FAIL, FAIL, FAIL, - #x y z { | } ~ ^? + #x y z { | } ~ ^? FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, @@ -97,7 +97,7 @@ else: if pending_bits > 0: raise_Incomplete(space, 'String has incomplete number of bytes') - return space.newtuple([space.wrap(res.build()), space.wrap(done)]) + return space.newtuple([space.newbytes(res.build()), space.wrap(done)]) # ____________________________________________________________ @@ -128,7 +128,7 @@ if leftbits > 0: leftchar <<= (6 - leftbits) res.append(hqx_encoding[leftchar & 0x3f]) - return space.wrap(res.build()) + return space.newbytes(res.build()) # ____________________________________________________________ @@ -150,7 +150,7 @@ lastpushed = ord(c) else: if i == end: - raise_Incomplete(space, 'String ends with the RLE code \x90') + raise_Incomplete(space, 'String ends with the RLE code \\x90') count = ord(hexbin[i]) - 1 i += 1 if count < 0: @@ -158,9 +158,9 @@ lastpushed = 0x90 else: if lastpushed < 0: - raise_Error(space, 'String starts with the RLE code \x90') + raise_Error(space, 'String starts with the RLE code \\x90') res.append_multiple_char(chr(lastpushed), count) - return space.wrap(res.build()) + return space.newbytes(res.build()) # ____________________________________________________________ @@ -197,7 +197,7 @@ # string that rledecode_hqx() would expand back to 'data', there are # some programs somewhere that would start failing obscurely in rare # cases. - return space.wrap(res.build()) + return space.newbytes(res.build()) # ____________________________________________________________ diff --git a/pypy/module/binascii/interp_qp.py b/pypy/module/binascii/interp_qp.py --- a/pypy/module/binascii/interp_qp.py +++ b/pypy/module/binascii/interp_qp.py @@ -56,7 +56,7 @@ if header and c == '_': c = ' ' odata.append(c) - return space.wrap(odata.build()) + return space.newbytes(odata.build()) # ____________________________________________________________ @@ -159,4 +159,4 @@ odata.append(c) inp += 1 - return space.wrap(odata.build()) + return space.newbytes(odata.build()) diff --git a/pypy/module/binascii/interp_uu.py b/pypy/module/binascii/interp_uu.py --- a/pypy/module/binascii/interp_uu.py +++ b/pypy/module/binascii/interp_uu.py @@ -54,7 +54,7 @@ remaining = length - res.getlength() if remaining > 0: res.append_multiple_char('\x00', remaining) - return space.wrap(res.build()) + return space.newbytes(res.build()) # ____________________________________________________________ @@ -86,4 +86,4 @@ res.append(chr(0x20 + (C & 0x3F))) res.append('\n') - return space.wrap(res.build()) + return space.newbytes(res.build()) diff --git a/pypy/module/bz2/interp_bz2.py b/pypy/module/bz2/interp_bz2.py --- a/pypy/module/bz2/interp_bz2.py +++ b/pypy/module/bz2/interp_bz2.py @@ -557,7 +557,7 @@ datasize = len(data) if datasize == 0: - return self.space.wrap("") + return self.space.newbytes("") if not self.running: raise oefmt(self.space.w_ValueError, @@ -582,7 +582,7 @@ out.prepare_next_chunk() res = out.make_result_string() - return self.space.wrap(res) + return self.space.newbytes(res) def flush(self): if not self.running: @@ -602,7 +602,7 @@ out.prepare_next_chunk() res = out.make_result_string() - return self.space.wrap(res) + return self.space.newbytes(res) W_BZ2Compressor.typedef = TypeDef("BZ2Compressor", __doc__ = W_BZ2Compressor.__doc__, @@ -669,7 +669,7 @@ raise oefmt(self.space.w_EOFError, "end of stream was already found") if data == '': - return self.space.wrap('') + return self.space.newbytes('') in_bufsize = len(data) @@ -698,7 +698,7 @@ out.prepare_next_chunk() res = out.make_result_string() - return self.space.wrap(res) + return self.space.newbytes(res) W_BZ2Decompressor.typedef = TypeDef("BZ2Decompressor", diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py --- a/pypy/module/cpyext/api.py +++ b/pypy/module/cpyext/api.py @@ -800,6 +800,21 @@ pypy_debug_catch_fatal_exception() assert False +def _restore_gil_state(pygilstate_release, gilstate, gil_release, _gil_auto, tid): + from rpython.rlib import rgil + # see "Handling of the GIL" above + assert cpyext_glob_tid_ptr[0] == 0 + if pygilstate_release: + from pypy.module.cpyext import pystate + unlock = (gilstate == pystate.PyGILState_UNLOCKED) + else: + unlock = gil_release or _gil_auto + if unlock: + rgil.release() + else: + cpyext_glob_tid_ptr[0] = tid + + def make_wrapper_second_level(space, argtypesw, restype, result_kind, error_value, gil): from rpython.rlib import rgil @@ -827,6 +842,7 @@ def wrapper_second_level(callable, pname, *args): from pypy.module.cpyext.pyobject import make_ref, from_ref, is_pyobj from pypy.module.cpyext.pyobject import as_pyobj + from pypy.module.cpyext import pystate # we hope that malloc removal removes the newtuple() that is # inserted exactly here by the varargs specializer @@ -839,7 +855,6 @@ rgil.acquire() assert cpyext_glob_tid_ptr[0] == 0 elif pygilstate_ensure: - from pypy.module.cpyext import pystate if cpyext_glob_tid_ptr[0] == tid: cpyext_glob_tid_ptr[0] = 0 args += (pystate.PyGILState_LOCKED,) @@ -850,6 +865,10 @@ if cpyext_glob_tid_ptr[0] != tid: no_gil_error(pname) cpyext_glob_tid_ptr[0] = 0 + if pygilstate_release: + gilstate = rffi.cast(lltype.Signed, args[-1]) + else: + gilstate = pystate.PyGILState_IGNORE rffi.stackcounter.stacks_counter += 1 llop.gc_stack_bottom(lltype.Void) # marker for trackgcroot.py @@ -919,24 +938,13 @@ except Exception as e: unexpected_exception(pname, e, tb) + _restore_gil_state(pygilstate_release, gilstate, gil_release, _gil_auto, tid) return fatal_value assert lltype.typeOf(retval) == restype rffi.stackcounter.stacks_counter -= 1 - # see "Handling of the GIL" above - assert cpyext_glob_tid_ptr[0] == 0 - if pygilstate_release: - from pypy.module.cpyext import pystate - arg = rffi.cast(lltype.Signed, args[-1]) - unlock = (arg == pystate.PyGILState_UNLOCKED) - else: - unlock = gil_release or _gil_auto - if unlock: - rgil.release() - else: - cpyext_glob_tid_ptr[0] = tid - + _restore_gil_state(pygilstate_release, gilstate, gil_release, _gil_auto, tid) return retval wrapper_second_level._dont_inline_ = True @@ -1202,8 +1210,6 @@ cpyext_type_init = self.cpyext_type_init self.cpyext_type_init = None for pto, w_type in cpyext_type_init: - if space.is_w(w_type, space.w_str): - pto.c_tp_itemsize = 1 finish_type_1(space, pto) finish_type_2(space, pto, w_type) diff --git a/pypy/module/cpyext/bufferobject.py b/pypy/module/cpyext/bufferobject.py --- a/pypy/module/cpyext/bufferobject.py +++ b/pypy/module/cpyext/bufferobject.py @@ -79,5 +79,5 @@ Py_DecRef(space, py_buf.c_b_base) else: rffi.free_charp(rffi.cast(rffi.CCHARP, py_buf.c_b_ptr)) - from pypy.module.cpyext.object import PyObject_dealloc - PyObject_dealloc(space, py_obj) + from pypy.module.cpyext.object import _dealloc + _dealloc(space, py_obj) diff --git a/pypy/module/cpyext/bytearrayobject.py b/pypy/module/cpyext/bytearrayobject.py --- a/pypy/module/cpyext/bytearrayobject.py +++ b/pypy/module/cpyext/bytearrayobject.py @@ -25,55 +25,11 @@ PyByteArrayObjectStruct = lltype.ForwardReference() PyByteArrayObject = lltype.Ptr(PyByteArrayObjectStruct) -PyByteArrayObjectFields = PyVarObjectFields -# (("ob_exports", rffi.INT), ("ob_alloc", rffi.LONG), ("ob_bytes", rffi.CCHARP)) +PyByteArrayObjectFields = PyVarObjectFields cpython_struct("PyByteArrayObject", PyByteArrayObjectFields, PyByteArrayObjectStruct) - at bootstrap_function -def init_bytearrayobject(space): - "Type description of PyByteArrayObject" - #make_typedescr(space.w_bytearray.layout.typedef, - # basestruct=PyByteArrayObject.TO, - # attach=bytearray_attach, - # dealloc=bytearray_dealloc, - # realize=bytearray_realize) - PyByteArray_Check, PyByteArray_CheckExact = build_type_checkers("ByteArray", "w_bytearray") -# XXX dead code to be removed -#def bytearray_attach(space, py_obj, w_obj): -# """ -# Fills a newly allocated PyByteArrayObject with the given bytearray object -# """ -# py_ba = rffi.cast(PyByteArrayObject, py_obj) -# py_ba.c_ob_size = len(space.str_w(w_obj)) -# py_ba.c_ob_bytes = lltype.nullptr(rffi.CCHARP.TO) -# py_ba.c_ob_exports = rffi.cast(rffi.INT, 0) - -#def bytearray_realize(space, py_obj): -# """ -# Creates the bytearray in the interpreter. -# """ -# py_ba = rffi.cast(PyByteArrayObject, py_obj) -# if not py_ba.c_ob_bytes: -# py_ba.c_buffer = lltype.malloc(rffi.CCHARP.TO, py_ba.c_ob_size + 1, -# flavor='raw', zero=True) -# s = rffi.charpsize2str(py_ba.c_ob_bytes, py_ba.c_ob_size) -# w_obj = space.wrap(s) -# py_ba.c_ob_exports = rffi.cast(rffi.INT, 0) -# track_reference(space, py_obj, w_obj) -# return w_obj - -#@cpython_api([PyObject], lltype.Void, header=None) -#def bytearray_dealloc(space, py_obj): -# """Frees allocated PyByteArrayObject resources. -# """ -# py_ba = rffi.cast(PyByteArrayObject, py_obj) -# if py_ba.c_ob_bytes: -# lltype.free(py_ba.c_ob_bytes, flavor="raw") -# from pypy.module.cpyext.object import PyObject_dealloc -# PyObject_dealloc(space, py_obj) - #_______________________________________________________________________ @cpython_api([PyObject], PyObject, result_is_ll=True) @@ -90,9 +46,9 @@ """Create a new bytearray object from string and its length, len. On failure, NULL is returned.""" if char_p: - w_s = space.wrap(rffi.charpsize2str(char_p, length)) + w_s = space.newbytes(rffi.charpsize2str(char_p, length)) else: - w_s = space.wrap(length) + w_s = space.newint(length) w_buffer = space.call_function(space.w_bytearray, w_s) return make_ref(space, w_buffer) @@ -124,7 +80,7 @@ if space.isinstance_w(w_obj, space.w_bytearray): oldlen = space.len_w(w_obj) if newlen > oldlen: - space.call_method(w_obj, 'extend', space.wrap('\x00' * (newlen - oldlen))) + space.call_method(w_obj, 'extend', space.newbytes('\x00' * (newlen - oldlen))) elif oldlen > newlen: assert newlen >= 0 space.delslice(w_obj, space.wrap(newlen), space.wrap(oldlen)) diff --git a/pypy/module/cpyext/bytesobject.py b/pypy/module/cpyext/bytesobject.py --- a/pypy/module/cpyext/bytesobject.py +++ b/pypy/module/cpyext/bytesobject.py @@ -6,17 +6,19 @@ from pypy.module.cpyext.pyerrors import PyErr_BadArgument from pypy.module.cpyext.pyobject import ( PyObject, PyObjectP, Py_DecRef, make_ref, from_ref, track_reference, - make_typedescr, get_typedescr, as_pyobj, Py_IncRef, get_w_obj_and_decref) + make_typedescr, get_typedescr, as_pyobj, Py_IncRef, get_w_obj_and_decref, + pyobj_has_w_obj) +from pypy.objspace.std.bytesobject import W_BytesObject ## -## Implementation of PyStringObject +## Implementation of PyBytesObject ## ================================ ## ## The problem ## ----------- ## ## PyString_AsString() must return a (non-movable) pointer to the underlying -## buffer, whereas pypy strings are movable. C code may temporarily store +## ob_sval, whereas pypy strings are movable. C code may temporarily store ## this address and use it, as long as it owns a reference to the PyObject. ## There is no "release" function to specify that the pointer is not needed ## any more. @@ -27,20 +29,23 @@ ## Solution ## -------- ## -## PyStringObject contains two additional members: the ob_size and a pointer to a -## char buffer; it may be NULL. +## PyBytesObject contains two additional members: the ob_size and a pointer to a +## char ob_sval; it may be NULL. ## -## - A string allocated by pypy will be converted into a PyStringObject with a +## - A string allocated by pypy will be converted into a PyBytesObject with a ## NULL buffer. The first time PyString_AsString() is called, memory is ## allocated (with flavor='raw') and content is copied. ## ## - A string allocated with PyString_FromStringAndSize(NULL, size) will -## allocate a PyStringObject structure, and a buffer with the specified +## allocate a PyBytesObject structure, and a buffer with the specified ## size+1, but the reference won't be stored in the global map; there is no ## corresponding object in pypy. When from_ref() or Py_INCREF() is called, ## the pypy string is created, and added to the global map of tracked ## objects. The buffer is then supposed to be immutable. ## +##- A buffer obtained from PyString_AS_STRING() could be mutable iff +## there is no corresponding pypy object for the string +## ## - _PyString_Resize() works only on not-yet-pypy'd strings, and returns a ## similar object. ## @@ -50,77 +55,73 @@ ## corresponds to the pypy gc-managed string. ## -PyStringObjectStruct = lltype.ForwardReference() -PyStringObject = lltype.Ptr(PyStringObjectStruct) -PyStringObjectFields = PyVarObjectFields + \ - (("ob_shash", rffi.LONG), ("ob_sstate", rffi.INT), ("buffer", rffi.CCHARP)) -cpython_struct("PyStringObject", PyStringObjectFields, PyStringObjectStruct) +PyBytesObjectStruct = lltype.ForwardReference() +PyBytesObject = lltype.Ptr(PyBytesObjectStruct) +PyBytesObjectFields = PyVarObjectFields + \ + (("ob_shash", rffi.LONG), ("ob_sstate", rffi.INT), ("ob_sval", rffi.CArray(lltype.Char))) +cpython_struct("PyStringObject", PyBytesObjectFields, PyBytesObjectStruct) @bootstrap_function -def init_stringobject(space): - "Type description of PyStringObject" +def init_bytesobject(space): + "Type description of PyBytesObject" make_typedescr(space.w_str.layout.typedef, - basestruct=PyStringObject.TO, - attach=string_attach, - dealloc=string_dealloc, - realize=string_realize) + basestruct=PyBytesObject.TO, + attach=bytes_attach, + dealloc=bytes_dealloc, + realize=bytes_realize) PyString_Check, PyString_CheckExact = build_type_checkers("String", "w_str") def new_empty_str(space, length): """ - Allocate a PyStringObject and its buffer, but without a corresponding - interpreter object. The buffer may be mutated, until string_realize() is + Allocate a PyBytesObject and its ob_sval, but without a corresponding + interpreter object. The ob_sval may be mutated, until bytes_realize() is called. Refcount of the result is 1. """ typedescr = get_typedescr(space.w_str.layout.typedef) - py_obj = typedescr.allocate(space, space.w_str) - py_str = rffi.cast(PyStringObject, py_obj) - - buflen = length + 1 - py_str.c_ob_size = length - py_str.c_buffer = lltype.malloc(rffi.CCHARP.TO, buflen, - flavor='raw', zero=True, - add_memory_pressure=True) + py_obj = typedescr.allocate(space, space.w_str, length) + py_str = rffi.cast(PyBytesObject, py_obj) + py_str.c_ob_shash = -1 py_str.c_ob_sstate = rffi.cast(rffi.INT, 0) # SSTATE_NOT_INTERNED return py_str -def string_attach(space, py_obj, w_obj): +def bytes_attach(space, py_obj, w_obj): """ - Fills a newly allocated PyStringObject with the given string object. The - buffer must not be modified. + Copy RPython string object contents to a PyBytesObject. The + c_ob_sval must not be modified. """ - py_str = rffi.cast(PyStringObject, py_obj) - py_str.c_ob_size = len(space.str_w(w_obj)) - py_str.c_buffer = lltype.nullptr(rffi.CCHARP.TO) + py_str = rffi.cast(PyBytesObject, py_obj) + s = space.str_w(w_obj) + if py_str.c_ob_size < len(s): + raise oefmt(space.w_ValueError, + "bytes_attach called on object with ob_size %d but trying to store %d", + py_str.c_ob_size, len(s)) + rffi.c_memcpy(py_str.c_ob_sval, rffi.str2charp(s), len(s)) + py_str.c_ob_sval[len(s)] = '\0' py_str.c_ob_shash = space.hash_w(w_obj) py_str.c_ob_sstate = rffi.cast(rffi.INT, 1) # SSTATE_INTERNED_MORTAL -def string_realize(space, py_obj): +def bytes_realize(space, py_obj): """ - Creates the string in the interpreter. The PyStringObject buffer must not + Creates the string in the interpreter. The PyBytesObject ob_sval must not be modified after this call. """ - py_str = rffi.cast(PyStringObject, py_obj) - if not py_str.c_buffer: - py_str.c_buffer = lltype.malloc(rffi.CCHARP.TO, py_str.c_ob_size + 1, - flavor='raw', zero=True) - s = rffi.charpsize2str(py_str.c_buffer, py_str.c_ob_size) - w_obj = space.wrap(s) + py_str = rffi.cast(PyBytesObject, py_obj) + s = rffi.charpsize2str(py_str.c_ob_sval, py_str.c_ob_size) + w_type = from_ref(space, rffi.cast(PyObject, py_obj.c_ob_type)) + w_obj = space.allocate_instance(W_BytesObject, w_type) + w_obj.__init__(s) py_str.c_ob_shash = space.hash_w(w_obj) py_str.c_ob_sstate = rffi.cast(rffi.INT, 1) # SSTATE_INTERNED_MORTAL track_reference(space, py_obj, w_obj) return w_obj @cpython_api([PyObject], lltype.Void, header=None) -def string_dealloc(space, py_obj): - """Frees allocated PyStringObject resources. +def bytes_dealloc(space, py_obj): + """Frees allocated PyBytesObject resources. """ - py_str = rffi.cast(PyStringObject, py_obj) - if py_str.c_buffer: - lltype.free(py_str.c_buffer, flavor="raw") - from pypy.module.cpyext.object import PyObject_dealloc - PyObject_dealloc(space, py_obj) + from pypy.module.cpyext.object import _dealloc + _dealloc(space, py_obj) #_______________________________________________________________________ @@ -139,6 +140,9 @@ @cpython_api([PyObject], rffi.CCHARP, error=0) def PyString_AsString(space, ref): + return _PyString_AsString(space, ref) + +def _PyString_AsString(space, ref): if from_ref(space, rffi.cast(PyObject, ref.c_ob_type)) is space.w_str: pass # typecheck returned "ok" without forcing 'ref' at all elif not PyString_Check(space, ref): # otherwise, use the alternate way @@ -150,16 +154,24 @@ raise oefmt(space.w_TypeError, "expected string or Unicode object, %T found", from_ref(space, ref)) - ref_str = rffi.cast(PyStringObject, ref) - if not ref_str.c_buffer: - # copy string buffer - w_str = from_ref(space, ref) - s = space.str_w(w_str) - ref_str.c_buffer = rffi.str2charp(s) - return ref_str.c_buffer + ref_str = rffi.cast(PyBytesObject, ref) + if not pyobj_has_w_obj(ref): + # XXX Force the ref? + bytes_realize(space, ref) + return ref_str.c_ob_sval + + at cpython_api([rffi.VOIDP], rffi.CCHARP, error=0) +def PyString_AS_STRING(space, void_ref): + ref = rffi.cast(PyObject, void_ref) + # if no w_str is associated with this ref, + # return the c-level ptr as RW + if not pyobj_has_w_obj(ref): + py_str = rffi.cast(PyBytesObject, ref) + return py_str.c_ob_sval + return _PyString_AsString(space, ref) @cpython_api([PyObject, rffi.CCHARPP, rffi.CArrayPtr(Py_ssize_t)], rffi.INT_real, error=-1) -def PyString_AsStringAndSize(space, ref, buffer, length): +def PyString_AsStringAndSize(space, ref, data, length): if not PyString_Check(space, ref): from pypy.module.cpyext.unicodeobject import ( PyUnicode_Check, _PyUnicode_AsDefaultEncodedString) @@ -169,18 +181,16 @@ raise oefmt(space.w_TypeError, "expected string or Unicode object, %T found", from_ref(space, ref)) - ref_str = rffi.cast(PyStringObject, ref) - if not ref_str.c_buffer: - # copy string buffer - w_str = from_ref(space, ref) - s = space.str_w(w_str) - ref_str.c_buffer = rffi.str2charp(s) - buffer[0] = ref_str.c_buffer + if not pyobj_has_w_obj(ref): + # force the ref + bytes_realize(space, ref) + ref_str = rffi.cast(PyBytesObject, ref) + data[0] = ref_str.c_ob_sval if length: length[0] = ref_str.c_ob_size else: i = 0 - while ref_str.c_buffer[i] != '\0': + while ref_str.c_ob_sval[i] != '\0': i += 1 if i != ref_str.c_ob_size: raise oefmt(space.w_TypeError, @@ -190,7 +200,7 @@ @cpython_api([PyObject], Py_ssize_t, error=-1) def PyString_Size(space, ref): if from_ref(space, rffi.cast(PyObject, ref.c_ob_type)) is space.w_str: - ref = rffi.cast(PyStringObject, ref) + ref = rffi.cast(PyBytesObject, ref) return ref.c_ob_size else: w_obj = from_ref(space, ref) @@ -209,10 +219,10 @@ set to NULL, a memory exception is set, and -1 is returned. """ # XXX always create a new string so far - py_str = rffi.cast(PyStringObject, ref[0]) - if not py_str.c_buffer: + if pyobj_has_w_obj(ref[0]): raise oefmt(space.w_SystemError, "_PyString_Resize called on already created string") + py_str = rffi.cast(PyBytesObject, ref[0]) try: py_newstr = new_empty_str(space, newsize) except MemoryError: @@ -224,7 +234,7 @@ if oldsize < newsize: to_cp = oldsize for i in range(to_cp): - py_newstr.c_buffer[i] = py_str.c_buffer[i] + py_newstr.c_ob_sval[i] = py_str.c_ob_sval[i] Py_DecRef(space, ref[0]) ref[0] = rffi.cast(PyObject, py_newstr) return 0 diff --git a/pypy/module/cpyext/eval.py b/pypy/module/cpyext/eval.py --- a/pypy/module/cpyext/eval.py +++ b/pypy/module/cpyext/eval.py @@ -13,7 +13,7 @@ "PyCompilerFlags", (("cf_flags", rffi.INT),)) PyCompilerFlagsPtr = lltype.Ptr(PyCompilerFlags) -PyCF_MASK = (consts.CO_FUTURE_DIVISION | +PyCF_MASK = (consts.CO_FUTURE_DIVISION | consts.CO_FUTURE_ABSOLUTE_IMPORT | consts.CO_FUTURE_WITH_STATEMENT | consts.CO_FUTURE_PRINT_FUNCTION | @@ -94,7 +94,7 @@ Py_eval_input = 258 def compile_string(space, source, filename, start, flags=0): - w_source = space.wrap(source) + w_source = space.newbytes(source) start = rffi.cast(lltype.Signed, start) if start == Py_file_input: mode = 'exec' @@ -227,4 +227,4 @@ cf.c_cf_flags = rffi.cast(rffi.INT, flags) return result - + diff --git a/pypy/module/cpyext/frameobject.py b/pypy/module/cpyext/frameobject.py --- a/pypy/module/cpyext/frameobject.py +++ b/pypy/module/cpyext/frameobject.py @@ -46,8 +46,8 @@ Py_DecRef(space, py_code) Py_DecRef(space, py_frame.c_f_globals) Py_DecRef(space, py_frame.c_f_locals) - from pypy.module.cpyext.object import PyObject_dealloc - PyObject_dealloc(space, py_obj) + from pypy.module.cpyext.object import _dealloc + _dealloc(space, py_obj) def frame_realize(space, py_obj): """ diff --git a/pypy/module/cpyext/funcobject.py b/pypy/module/cpyext/funcobject.py --- a/pypy/module/cpyext/funcobject.py +++ b/pypy/module/cpyext/funcobject.py @@ -60,8 +60,8 @@ def function_dealloc(space, py_obj): py_func = rffi.cast(PyFunctionObject, py_obj) Py_DecRef(space, py_func.c_func_name) - from pypy.module.cpyext.object import PyObject_dealloc - PyObject_dealloc(space, py_obj) + from pypy.module.cpyext.object import _dealloc + _dealloc(space, py_obj) def code_attach(space, py_obj, w_obj): py_code = rffi.cast(PyCodeObject, py_obj) @@ -80,8 +80,8 @@ py_code = rffi.cast(PyCodeObject, py_obj) Py_DecRef(space, py_code.c_co_name) Py_DecRef(space, py_code.c_co_filename) - from pypy.module.cpyext.object import PyObject_dealloc - PyObject_dealloc(space, py_obj) + from pypy.module.cpyext.object import _dealloc + _dealloc(space, py_obj) @cpython_api([PyObject], PyObject, result_borrowed=True) def PyFunction_GetCode(space, w_func): diff --git a/pypy/module/cpyext/include/_numpypy/numpy/__multiarray_api.h b/pypy/module/cpyext/include/_numpypy/numpy/__multiarray_api.h --- a/pypy/module/cpyext/include/_numpypy/numpy/__multiarray_api.h +++ b/pypy/module/cpyext/include/_numpypy/numpy/__multiarray_api.h @@ -5,7 +5,12 @@ npy_bool obval; } PyBoolScalarObject; -static int import_array(){return 0;}; -static int _import_array(){return 0;}; -static int _import_math(){return 0;}; +#if PY_VERSION_HEX >= 0x03000000 +#define NUMPY_IMPORT_ARRAY_RETVAL NULL +#else +#define NUMPY_IMPORT_ARRAY_RETVAL +#endif +#define import_array() {return NUMPY_IMPORT_ARRAY_RETVAL;} + + diff --git a/pypy/module/cpyext/include/stringobject.h b/pypy/module/cpyext/include/stringobject.h --- a/pypy/module/cpyext/include/stringobject.h +++ b/pypy/module/cpyext/include/stringobject.h @@ -10,7 +10,6 @@ #include #define PyString_GET_SIZE(op) PyString_Size((PyObject*)(op)) -#define PyString_AS_STRING(op) PyString_AsString((PyObject*)(op)) /* Type PyStringObject represents a character string. An extra zero byte is reserved at the end to ensure it is zero-terminated, but a size is @@ -41,12 +40,11 @@ PyObject_VAR_HEAD long ob_shash; int ob_sstate; - char * buffer; /* change the name from cpython so all non-api c access is thwarted */ + char ob_sval[1]; /* Invariants - * (not relevant in PyPy, all stringobjects are backed by a pypy object) - * buffer contains space for 'ob_size+1' elements. - * buffer[ob_size] == 0. + * ob_sval contains space for 'ob_size+1' elements. + * ob_sval[ob_size] == 0. * ob_shash is the hash of the string or -1 if not computed yet. * ob_sstate != 0 iff the string object is in stringobject.c's * 'interned' dictionary; in this case the two references diff --git a/pypy/module/cpyext/include/tupleobject.h b/pypy/module/cpyext/include/tupleobject.h --- a/pypy/module/cpyext/include/tupleobject.h +++ b/pypy/module/cpyext/include/tupleobject.h @@ -8,9 +8,12 @@ #endif typedef struct { - PyObject_HEAD - Py_ssize_t ob_size; - PyObject **ob_item; /* XXX optimize to ob_item[] */ + PyObject_VAR_HEAD + PyObject *ob_item[1]; + /* ob_item contains space for 'ob_size' elements. + * Items must normally not be NULL, except during construction when + * the tuple is not yet visible outside the function that builds it. + */ } PyTupleObject; /* defined in varargswrapper.c */ diff --git a/pypy/module/cpyext/methodobject.py b/pypy/module/cpyext/methodobject.py --- a/pypy/module/cpyext/methodobject.py +++ b/pypy/module/cpyext/methodobject.py @@ -55,8 +55,8 @@ py_func = rffi.cast(PyCFunctionObject, py_obj) Py_DecRef(space, py_func.c_m_self) Py_DecRef(space, py_func.c_m_module) - from pypy.module.cpyext.object import PyObject_dealloc - PyObject_dealloc(space, py_obj) + from pypy.module.cpyext.object import _dealloc + _dealloc(space, py_obj) class W_PyCFunctionObject(W_Root): diff --git a/pypy/module/cpyext/object.py b/pypy/module/cpyext/object.py --- a/pypy/module/cpyext/object.py +++ b/pypy/module/cpyext/object.py @@ -54,6 +54,9 @@ @cpython_api([PyObject], lltype.Void) def PyObject_dealloc(space, obj): + return _dealloc(space, obj) + +def _dealloc(space, obj): # This frees an object after its refcount dropped to zero, so we # assert that it is really zero here. assert obj.c_ob_refcnt == 0 diff --git a/pypy/module/cpyext/pyfile.py b/pypy/module/cpyext/pyfile.py --- a/pypy/module/cpyext/pyfile.py +++ b/pypy/module/cpyext/pyfile.py @@ -23,7 +23,7 @@ try: w_readline = space.getattr(w_obj, space.wrap('readline')) except OperationError: - raise oefmt(space.w_TypeError, + raise oefmt(space.w_TypeError, "argument must be a file, or have a readline() method.") n = rffi.cast(lltype.Signed, n) @@ -41,7 +41,7 @@ On success, return a new file object that is opened on the file given by filename, with a file mode given by mode, where mode has the same semantics as the standard C routine fopen(). On failure, return NULL.""" - w_filename = space.wrap(rffi.charp2str(filename)) + w_filename = space.newbytes(rffi.charp2str(filename)) w_mode = space.wrap(rffi.charp2str(mode)) return space.call_method(space.builtin, 'file', w_filename, w_mode) diff --git a/pypy/module/cpyext/pyobject.py b/pypy/module/cpyext/pyobject.py --- a/pypy/module/cpyext/pyobject.py +++ b/pypy/module/cpyext/pyobject.py @@ -7,7 +7,7 @@ from pypy.module.cpyext.api import ( cpython_api, bootstrap_function, PyObject, PyObjectP, ADDR, CANNOT_FAIL, Py_TPFLAGS_HEAPTYPE, PyTypeObjectPtr, is_PyObject, - INTERPLEVEL_API) + INTERPLEVEL_API, PyVarObject) from pypy.module.cpyext.state import State from pypy.objspace.std.typeobject import W_TypeObject from pypy.objspace.std.objectobject import W_ObjectObject @@ -47,13 +47,16 @@ size = pytype.c_tp_basicsize else: size = rffi.sizeof(self.basestruct) - if itemcount and w_type is not space.w_str: + if pytype.c_tp_itemsize: size += itemcount * pytype.c_tp_itemsize assert size >= rffi.sizeof(PyObject.TO) From pypy.commits at gmail.com Fri Jul 1 05:54:18 2016 From: pypy.commits at gmail.com (mattip) Date: Fri, 01 Jul 2016 02:54:18 -0700 (PDT) Subject: [pypy-commit] pypy call-via-pyobj: add a failing test Message-ID: <57763dca.09f6c20a.217d4.7d0b@mx.google.com> Author: Matti Picus Branch: call-via-pyobj Changeset: r85483:143f7a33f037 Date: 2016-07-01 12:45 +0300 http://bitbucket.org/pypy/pypy/changeset/143f7a33f037/ Log: add a failing test diff --git a/pypy/module/cpyext/test/array.c b/pypy/module/cpyext/test/array.c --- a/pypy/module/cpyext/test/array.c +++ b/pypy/module/cpyext/test/array.c @@ -2144,6 +2144,15 @@ return array_new(type, args, NULL); } +static PyObject * +switch_multiply(void) +{ + fprintf(stdout, "switching nb_multiply from %p to %p\n", + Arraytype.tp_as_number->nb_multiply, array_base_multiply); + Arraytype.tp_as_number->nb_multiply = array_base_multiply; + Py_RETURN_NONE; +}; + PyDoc_STRVAR(module_doc, "This module defines an object type which can efficiently represent\n\ an array of basic values: characters, integers, floating point\n\ @@ -2394,6 +2403,7 @@ /* No functions in array module. */ static PyMethodDef a_methods[] = { {"_reconstruct", (PyCFunction)_reconstruct, METH_VARARGS, NULL}, + {"switch_multiply", (PyCFunction)switch_multiply, METH_NOARGS, NULL}, {NULL, NULL, 0, NULL} /* Sentinel */ }; diff --git a/pypy/module/cpyext/test/test_arraymodule.py b/pypy/module/cpyext/test/test_arraymodule.py --- a/pypy/module/cpyext/test/test_arraymodule.py +++ b/pypy/module/cpyext/test/test_arraymodule.py @@ -84,3 +84,7 @@ arr = module.array('i', [2]) res = [1, 2, 3] * arr assert res == [1, 2, 3, 1, 2, 3] + module.switch_multiply() + res = [1, 2, 3] * arr + assert res == [2, 4, 6] + From pypy.commits at gmail.com Fri Jul 1 05:54:20 2016 From: pypy.commits at gmail.com (mattip) Date: Fri, 01 Jul 2016 02:54:20 -0700 (PDT) Subject: [pypy-commit] pypy call-via-pyobj: proof of concept using offset-to-function instead of function Message-ID: <57763dcc.67c0c20a.b97d3.ffff9e0c@mx.google.com> Author: Matti Picus Branch: call-via-pyobj Changeset: r85484:5b38ce2c4c11 Date: 2016-07-01 12:53 +0300 http://bitbucket.org/pypy/pypy/changeset/5b38ce2c4c11/ Log: proof of concept using offset-to-function instead of function diff --git a/pypy/module/cpyext/methodobject.py b/pypy/module/cpyext/methodobject.py --- a/pypy/module/cpyext/methodobject.py +++ b/pypy/module/cpyext/methodobject.py @@ -10,9 +10,10 @@ from pypy.module.cpyext.api import ( CONST_STRING, METH_CLASS, METH_COEXIST, METH_KEYWORDS, METH_NOARGS, METH_O, METH_STATIC, METH_VARARGS, PyObject, PyObjectFields, bootstrap_function, - build_type_checkers, cpython_api, cpython_struct, generic_cpy_call) + build_type_checkers, cpython_api, cpython_struct, generic_cpy_call, + PyTypeObjectPtr) from pypy.module.cpyext.pyobject import ( - Py_DecRef, from_ref, make_ref, make_typedescr) + Py_DecRef, from_ref, make_ref, as_pyobj, make_typedescr) PyCFunction_typedef = rffi.COpaquePtr(typedef='PyCFunction') PyCFunction = lltype.Ptr(lltype.FuncType([PyObject, PyObject], PyObject)) @@ -151,13 +152,14 @@ class W_PyCWrapperObject(W_Root): def __init__(self, space, pto, method_name, wrapper_func, - wrapper_func_kwds, doc, func): + wrapper_func_kwds, doc, func, offset=-1): self.space = space self.method_name = method_name self.wrapper_func = wrapper_func self.wrapper_func_kwds = wrapper_func_kwds self.doc = doc self.func = func + self.offset = offset pyo = rffi.cast(PyObject, pto) w_type = from_ref(space, pyo) assert isinstance(w_type, W_TypeObject) @@ -172,7 +174,17 @@ raise oefmt(space.w_TypeError, "wrapper %s doesn't take any keyword arguments", self.method_name) - return self.wrapper_func(space, w_self, w_args, self.func) + if self.offset >= 0: + pto = rffi.cast(PyTypeObjectPtr, as_pyobj(space, self.w_objclass)) + pto_func_as_int = lltype.cast_ptr_to_int(pto) + self.offset + # XXX make pto_func the equivalent of this line + #lltype.cast_int_to_ptr(pto_func_as_int) + func_to_call = rffi.cast(rffi.VOIDP, pto.c_tp_as_number.c_nb_multiply) + # print '\ncalling', func_to_call, 'not', self.func + else: + # print 'calling', self.method_name,'with no offset' + func_to_call = self.func + return self.wrapper_func(space, w_self, w_args, func_to_call) def descr_method_repr(self): return self.space.wrap("" % @@ -301,12 +313,6 @@ def PyDescr_NewClassMethod(space, w_type, method): return space.wrap(W_PyCClassMethodObject(space, method, w_type)) -def PyDescr_NewWrapper(space, pto, method_name, wrapper_func, - wrapper_func_kwds, doc, func): - # not exactly the API sig - return space.wrap(W_PyCWrapperObject(space, pto, method_name, - wrapper_func, wrapper_func_kwds, doc, func)) - @cpython_api([lltype.Ptr(PyMethodDef), PyObject, CONST_STRING], PyObject) def Py_FindMethod(space, table, w_obj, name_ptr): """Return a bound method object for an extension type implemented in diff --git a/pypy/module/cpyext/typeobject.py b/pypy/module/cpyext/typeobject.py --- a/pypy/module/cpyext/typeobject.py +++ b/pypy/module/cpyext/typeobject.py @@ -17,9 +17,9 @@ generic_cpy_call, Py_TPFLAGS_READY, Py_TPFLAGS_READYING, Py_TPFLAGS_HEAPTYPE, METH_VARARGS, METH_KEYWORDS, CANNOT_FAIL, Py_TPFLAGS_HAVE_GETCHARBUFFER, build_type_checkers, StaticObjectBuilder, - PyObjectFields, Py_TPFLAGS_BASETYPE) + PyObjectFields, Py_TPFLAGS_BASETYPE, PyTypeObject, PyTypeObjectPtr) from pypy.module.cpyext.methodobject import (W_PyCClassMethodObject, - PyDescr_NewWrapper, PyCFunction_NewEx, PyCFunction_typedef, PyMethodDef, + W_PyCWrapperObject, PyCFunction_NewEx, PyCFunction_typedef, PyMethodDef, W_PyCMethodObject, W_PyCFunctionObject) from pypy.module.cpyext.modsupport import convert_method_defs from pypy.module.cpyext.pyobject import ( @@ -30,7 +30,7 @@ from pypy.module.cpyext.state import State from pypy.module.cpyext.structmember import PyMember_GetOne, PyMember_SetOne from pypy.module.cpyext.typeobjectdefs import ( - PyTypeObjectPtr, PyTypeObject, PyGetSetDef, PyMemberDef, newfunc, + PyGetSetDef, PyMemberDef, newfunc, PyNumberMethods, PyMappingMethods, PySequenceMethods, PyBufferProcs) from pypy.objspace.std.typeobject import W_TypeObject, find_best_base @@ -296,6 +296,7 @@ for method_name, slot_names, wrapper_func, wrapper_func_kwds, doc in slotdefs_for_wrappers: if method_name in dict_w: continue + offset = rffi.offsetof(pto._T, slot_names[0]) if len(slot_names) == 1: func = getattr(pto, slot_names[0]) else: @@ -303,14 +304,24 @@ struct = getattr(pto, slot_names[0]) if not struct: continue + offset += rffi.offsetof(struct._T, slot_names[1]) func = getattr(struct, slot_names[1]) func_voidp = rffi.cast(rffi.VOIDP, func) if not func: continue if wrapper_func is None and wrapper_func_kwds is None: continue - dict_w[method_name] = PyDescr_NewWrapper(space, pto, method_name, wrapper_func, - wrapper_func_kwds, doc, func_voidp) + name = rffi.charp2str(pto.c_tp_name) + if method_name in ('__mul__', '__rmul__') and 'array' in name: + # print '\nsetting', name, method_name, 'from', slot_names + # print ' pto is', pto + # print ' func_voidp is', func_voidp + pass + else: + offset = -1 + w_obj = W_PyCWrapperObject(space, pto, method_name, wrapper_func, + wrapper_func_kwds, doc, func_voidp, offset=offset) + dict_w[method_name] = space.wrap(w_obj) if pto.c_tp_new: add_tp_new_wrapper(space, dict_w, pto) @@ -714,6 +725,8 @@ def py_type_ready(space, pto): if pto.c_tp_flags & Py_TPFLAGS_READY: + name = rffi.charp2str(pto.c_tp_name) + print 'py_type_ready',name, 'but PyTP_FLAGS_READY is set' return type_realize(space, rffi.cast(PyObject, pto)) @@ -730,6 +743,8 @@ try: w_obj = _type_realize(space, py_obj) finally: + name = rffi.charp2str(pto.c_tp_name) + print '_type_realize done', name pto.c_tp_flags &= ~Py_TPFLAGS_READYING pto.c_tp_flags |= Py_TPFLAGS_READY return w_obj @@ -811,7 +826,9 @@ base = pto.c_tp_base base_pyo = rffi.cast(PyObject, pto.c_tp_base) if base and not base.c_tp_flags & Py_TPFLAGS_READY: - type_realize(space, rffi.cast(PyObject, base_pyo)) + name = rffi.charp2str(base.c_tp_name) + print 'realizing base while creating child', + type_realize(space, base_pyo) if base and not pto.c_ob_type: # will be filled later pto.c_ob_type = base.c_ob_type if not pto.c_tp_bases: From pypy.commits at gmail.com Fri Jul 1 05:58:26 2016 From: pypy.commits at gmail.com (arigo) Date: Fri, 01 Jul 2016 02:58:26 -0700 (PDT) Subject: [pypy-commit] pypy reverse-debugger: A fix that is also an optimization: when single-stepping on a loop Message-ID: <57763ec2.488e1c0a.f8ace.137d@mx.google.com> Author: Armin Rigo Branch: reverse-debugger Changeset: r85485:259a2938c2c1 Date: 2016-07-01 11:59 +0200 http://bitbucket.org/pypy/pypy/changeset/259a2938c2c1/ Log: A fix that is also an optimization: when single-stepping on a loop while foo: body then it never jumps back to the first bytecode of the "while" line (it jumps back to the 2nd bytecode of that line). But we still want to see it when stepping in the loop. diff --git a/pypy/interpreter/pyopcode.py b/pypy/interpreter/pyopcode.py --- a/pypy/interpreter/pyopcode.py +++ b/pypy/interpreter/pyopcode.py @@ -53,10 +53,14 @@ ### opcode dispatch ### def dispatch(self, pycode, next_instr, ec): + if self.space.config.translation.reverse_debugger: + from pypy.interpreter.reverse_debugging import prepare_code + prepare_code(pycode) + # # For the sequel, force 'next_instr' to be unsigned for performance next_instr = r_uint(next_instr) co_code = pycode.co_code - + # try: while True: next_instr = self.handle_bytecode(co_code, next_instr, ec) @@ -1055,6 +1059,11 @@ def jump_absolute(self, jumpto, ec): # this function is overridden by pypy.module.pypyjit.interp_jit check_nonneg(jumpto) + # + if self.space.config.translation.reverse_debugger: + from pypy.interpreter.reverse_debugging import jump_backward + jump_backward(self, jumpto) + # return jumpto def JUMP_FORWARD(self, jumpby, next_instr): @@ -1308,9 +1317,12 @@ self.space.setitem(w_dict, w_key, w_value) def LOAD_REVDB_VAR(self, oparg, next_instr): - from pypy.interpreter.reverse_debugging import load_metavar - w_var = load_metavar(oparg) - self.pushvalue(w_var) + if self.space.config.translation.reverse_debugger: + from pypy.interpreter.reverse_debugging import load_metavar + w_var = load_metavar(oparg) + self.pushvalue(w_var) + else: + self.MISSING_OPCODE(oparg, next_instr) ### ____________________________________________________________ ### diff --git a/pypy/interpreter/reverse_debugging.py b/pypy/interpreter/reverse_debugging.py --- a/pypy/interpreter/reverse_debugging.py +++ b/pypy/interpreter/reverse_debugging.py @@ -5,7 +5,7 @@ from rpython.rtyper.annlowlevel import cast_gcref_to_instance from pypy.interpreter.error import OperationError, oefmt from pypy.interpreter.baseobjspace import W_Root -from pypy.interpreter import gateway, typedef, pycode, pytraceback +from pypy.interpreter import gateway, typedef, pycode, pytraceback, pyframe from pypy.module.marshal import interp_marshal @@ -47,7 +47,11 @@ revdb.register_debug_command(revdb.CMD_WATCHVALUES, lambda_watchvalues) -pycode.PyCode.co_revdb_linestarts = None # or a string: an array of bits +pycode.PyCode.co_revdb_linestarts = None # or a string: see below + +# invariant: "f_revdb_nextline_instr" is the bytecode offset of +# the start of the line that follows "last_instr". +pyframe.PyFrame.f_revdb_nextline_instr = 0 def enter_call(caller_frame, callee_frame): @@ -64,6 +68,17 @@ if dbstate.breakpoint_stack_id == revdb.get_unique_id(caller_frame): revdb.breakpoint(-1) + +def jump_backward(frame, jumpto): + # When we see a jump backward, we set 'f_revdb_nextline_instr' in + # such a way that the next instruction, at 'jumpto', will trigger + # stop_point_at_start_of_line(). We have to trigger it even if + # 'jumpto' is not actually a start of line. For example, in a + # 'while foo: body', the body ends with a JUMP_ABSOLUTE which + # jumps back to the *second* opcode of the while. + frame.f_revdb_nextline_instr = jumpto + + def potential_stop_point(frame): if not we_are_translated(): return @@ -72,22 +87,49 @@ # Uses roughly the same algo as ExecutionContext.run_trace_func() # to know where the line starts are, but tweaked for speed, # avoiding the quadratic complexity when run N times with a large - # code object. A potential difference is that we only record - # where the line starts are; the "We jumped backwards in the same - # line" case of run_trace_func() is not fully reproduced. + # code object. # - code = frame.pycode - lstart = code.co_revdb_linestarts - if lstart is None: - lstart = build_co_revdb_linestarts(code) - index = frame.last_instr - c = lstart[index >> 3] - if ord(c) & (1 << (index & 7)): + cur = frame.last_instr + if cur < frame.f_revdb_nextline_instr: + return # fast path: we're still inside the same line as before + # + call_stop_point_at_line = True + co_revdb_linestarts = frame.pycode.co_revdb_linestarts + if cur > frame.f_revdb_nextline_instr: + # + # We jumped forward over the start of the next line. We're + # inside a different line, but we will only trigger a stop + # point if we're at the starting bytecode of that line. Fetch + # from co_revdb_linestarts the start of the line that is at or + # follows 'cur'. + ch = ord(co_revdb_linestarts[cur]) + if ch == 0: + pass # we are at the start of a line now + else: + # We are not, so don't call stop_point_at_start_of_line(). + # We still have to fill f_revdb_nextline_instr. + call_stop_point_at_line = False + # + if call_stop_point_at_line: stop_point_at_start_of_line() + cur += 1 + ch = ord(co_revdb_linestarts[cur]) + # + # Update f_revdb_nextline_instr. Check if 'ch' was greater than + # 255, in which case it was rounded down to 255 and we have to + # continue looking + nextline_instr = cur + ch + while ch == 255: + ch = ord(co_revdb_linestarts[nextline_instr]) + nextline_instr += ch + frame.f_revdb_nextline_instr = nextline_instr + def build_co_revdb_linestarts(code): - # inspired by findlinestarts() in the 'dis' standard module - bits = [False] * len(code.co_code) + # Inspired by findlinestarts() in the 'dis' standard module. + # Set up 'bits' so that it contains \x00 at line starts and \xff + # in-between. + bits = ['\xff'] * (len(code.co_code) + 1) if not code.hidden_applevel: lnotab = code.co_lnotab addr = 0 @@ -98,34 +140,35 @@ line_incr = ord(lnotab[p+1]) if byte_incr: if newline != 0: - if addr < len(bits): - bits[addr] = True + bits[addr] = '\x00' newline = 0 addr += byte_incr newline |= line_incr p += 2 if newline: - if addr < len(bits): - bits[addr] = True + bits[addr] = '\x00' + bits[len(code.co_code)] = '\x00' # - byte_list = [] - pending = 0 - nextval = 1 - for bit_is_set in bits: - if bit_is_set: - pending |= nextval - if nextval < 128: - nextval <<= 1 + # Change 'bits' so that the character at 'i', if not \x00, measures + # how far the next \x00 is + next_null = len(code.co_code) + p = next_null - 1 + while p >= 0: + if bits[p] == '\x00': + next_null = p else: - byte_list.append(chr(pending)) - pending = 0 - nextval = 1 - if nextval != 1: - byte_list.append(chr(pending)) - lstart = ''.join(byte_list) + ch = next_null - p + if ch > 255: ch = 255 + bits[p] = chr(ch) + p -= 1 + lstart = ''.join(bits) code.co_revdb_linestarts = lstart return lstart +def prepare_code(code): + if code.co_revdb_linestarts is None: + build_co_revdb_linestarts(code) + def get_final_lineno(code): lineno = code.co_firstlineno lnotab = code.co_lnotab diff --git a/pypy/interpreter/test/test_reverse_debugging.py b/pypy/interpreter/test/test_reverse_debugging.py --- a/pypy/interpreter/test/test_reverse_debugging.py +++ b/pypy/interpreter/test/test_reverse_debugging.py @@ -4,6 +4,7 @@ class FakeCode: + hidden_applevel = False def __init__(self, co_code, co_lnotab): self.co_firstlineno = 43 self.co_code = co_code @@ -26,7 +27,10 @@ for addr, lineno in dis.findlinestarts(code): expected_starts.add(addr) - for index in range(len(code.co_code)): - c = lstart[index >> 3] - found = ord(c) & (1 << (index & 7)) - assert (found != 0) == (index in expected_starts) + next_start = len(code.co_code) + for index in range(len(code.co_code), -1, -1): + if index in expected_starts: + next_start = index + assert lstart[index] == chr(next_start - index + if next_start - index <= 255 + else 255) From pypy.commits at gmail.com Fri Jul 1 06:56:38 2016 From: pypy.commits at gmail.com (arigo) Date: Fri, 01 Jul 2016 03:56:38 -0700 (PDT) Subject: [pypy-commit] pypy reverse-debugger: Use marshal version 0 to avoid new_interned_str(). Message-ID: <57764c66.2457c20a.cc61d.ffffe7cf@mx.google.com> Author: Armin Rigo Branch: reverse-debugger Changeset: r85486:0f2ae8aeae0c Date: 2016-07-01 12:57 +0200 http://bitbucket.org/pypy/pypy/changeset/0f2ae8aeae0c/ Log: Use marshal version 0 to avoid new_interned_str(). diff --git a/pypy/interpreter/reverse_debugging.py b/pypy/interpreter/reverse_debugging.py --- a/pypy/interpreter/reverse_debugging.py +++ b/pypy/interpreter/reverse_debugging.py @@ -499,9 +499,11 @@ space = dbstate.space try: code = compile(expression, 'eval') + # Note: using version 0 to marshal watchpoints, in order to + # avoid space.new_interned_str() on unmarshal. This is + # forbidden because it comes with lasting side-effects. marshalled_code = space.str_w(interp_marshal.dumps( - space, space.wrap(code), - space.wrap(interp_marshal.Py_MARSHAL_VERSION))) + space, space.wrap(code), space.wrap(0))) except OperationError as e: revdb.send_watch(e.errorstr(space), ok_flag=0) else: From pypy.commits at gmail.com Fri Jul 1 07:53:58 2016 From: pypy.commits at gmail.com (mattip) Date: Fri, 01 Jul 2016 04:53:58 -0700 (PDT) Subject: [pypy-commit] extradoc extradoc: rewording Message-ID: <577659d6.e40bc30a.d83a9.ffffce42@mx.google.com> Author: Matti Picus Branch: extradoc Changeset: r5644:90f043cebe19 Date: 2016-07-01 14:53 +0300 http://bitbucket.org/pypy/extradoc/changeset/90f043cebe19/ Log: rewording diff --git a/talk/compiler-workshop-2016/pypy.rst b/talk/compiler-workshop-2016/pypy.rst --- a/talk/compiler-workshop-2016/pypy.rst +++ b/talk/compiler-workshop-2016/pypy.rst @@ -7,7 +7,7 @@ ----------------- PyPy is a mature production-ready framework automatically speeding up pure -python code by factors of 2-5 in commercial settings. +python code by factors of 2-5, already used in commercial settings. The PyPy community offers several tools to inspect and optimize Python programs. Examples: vmprof, cffi @@ -29,8 +29,8 @@ Flow graphs -> Annotation -> RTyping -> Code generation Advantages: -* Whole program optimizations (take that C) -* Deliver features fast, without sacrificing speed +* Whole program optimizations (take that, C) +* Deliver features quickly, without sacrificing speed * Loosely coupled (JIT, Interp., RPython) Downsides: @@ -44,23 +44,24 @@ under-funded. For instance, we could be doing alot more for data science but are moving slowly forward on a volunteer basis with C-API compatibility. -Our interests lie in still providing the confort of the Python eco system, -but not sacrificing execution time. Some details (e.g. garbage collection scheme) +Our interests lie in still providing the comfort of the Python ecosystem, +without sacrificing execution time. Some details (e.g. garbage collection scheme) have some impact on user programs. We could use a lot more help in identifying and resolving -some of these issues. If programs do not run out of the box, most users will stick to CPython -because their performance problems are not that of an issue (at that point in time). +these issues. If programs do not run out of the box, most users will just stick to CPython +because their performance problems are not that big of an issue (at that point in time). If we could resolve those issues (funded, or externally contributed) we would create a much better user experience. We are also working on Micro NumPy, which provides the kernels for numerical operations. -It is very much complete, but still some features are missing. We would love to have a +It is very usable, but still some features are missing. We would love to have a partner/company that would help us to complete NumPy for PyPy. We are open to changes to our JIT scheme. We are working on both high level optimizations and backend oriented changes. Ideas would be to mitigate some issues with our tracing JIT compiler -(or even build a region based compiler) and many more. Most of these aspects are covered quite well by -our core development team, but we will eventually take another big step in near future towards the 7th version -of our JIT. +(or even build a region based compiler) and many more. Most of these aspects are covered +quite well by +our core development team, but we will eventually take another big step in near +future towards the 7th version of our JIT. Other Interesting Aspects ------------------------- From pypy.commits at gmail.com Fri Jul 1 08:01:10 2016 From: pypy.commits at gmail.com (plan_rich) Date: Fri, 01 Jul 2016 05:01:10 -0700 (PDT) Subject: [pypy-commit] pypy ppc-vsx-support: add a many details to implement the reduction pattern (ppc, partly working already) Message-ID: <57765b86.4aa71c0a.f3317.1c55@mx.google.com> Author: Richard Plangger Branch: ppc-vsx-support Changeset: r85487:1360aa62b1ed Date: 2016-07-01 14:00 +0200 http://bitbucket.org/pypy/pypy/changeset/1360aa62b1ed/ Log: add a many details to implement the reduction pattern (ppc, partly working already) diff --git a/rpython/jit/backend/ppc/codebuilder.py b/rpython/jit/backend/ppc/codebuilder.py --- a/rpython/jit/backend/ppc/codebuilder.py +++ b/rpython/jit/backend/ppc/codebuilder.py @@ -605,12 +605,14 @@ # add xvadddp = XX3(60, XO9=96) xvaddsp = XX3(60, XO9=64) + xsadddp = XX3(60, XO9=32) # sub xvsubdp = XX3(60, XO9=104) xvsubsp = XX3(60, XO9=72) # mul xvmuldp = XX3(60, XO9=112) xvmulsp = XX3(60, XO9=80) + xsmuldp = XX3(60, XO9=46) # div xvdivdp = XX3(60, XO9=102) xvdivsp = XX3(60, XO9=88) @@ -662,6 +664,12 @@ # generic splat xxspltd = XX3_splat(60, XO13=10, OE=0) + xxlxor = XX3(60, XO9=154) + xxlor = XX3(60, XO9=146) + + # vector move register is alias to vector or + xvmr = xxlor + # INTEGER # ------- diff --git a/rpython/jit/backend/ppc/ppc_assembler.py b/rpython/jit/backend/ppc/ppc_assembler.py --- a/rpython/jit/backend/ppc/ppc_assembler.py +++ b/rpython/jit/backend/ppc/ppc_assembler.py @@ -771,7 +771,7 @@ self.update_frame_depth(frame_depth_no_fixed_size + JITFRAME_FIXED_SIZE) # size_excluding_failure_stuff = self.mc.get_relative_pos() - self.write_pending_failure_recoveries() + self.write_pending_failure_recoveries(regalloc) full_size = self.mc.get_relative_pos() # self.patch_stack_checks(frame_depth_no_fixed_size + JITFRAME_FIXED_SIZE) @@ -852,10 +852,12 @@ self.reserve_gcref_table(allgcrefs) startpos = self.mc.get_relative_pos() + self._update_at_exit(arglocs, inputargs, faildescr, regalloc) + self._check_frame_depth(self.mc, regalloc.get_gcmap()) frame_depth_no_fixed_size = self._assemble(regalloc, inputargs, operations) codeendpos = self.mc.get_relative_pos() - self.write_pending_failure_recoveries() + self.write_pending_failure_recoveries(regalloc) fullsize = self.mc.get_relative_pos() # self.patch_stack_checks(frame_depth_no_fixed_size + JITFRAME_FIXED_SIZE) @@ -928,7 +930,7 @@ ofs = self.cpu.get_ofs_of_frame_field('jf_gcmap') mc.store(r.SCRATCH.value, r.SPP.value, ofs) - def break_long_loop(self): + def break_long_loop(self, regalloc): # If the loop is too long, the guards in it will jump forward # more than 32 KB. We use an approximate hack to know if we # should break the loop here with an unconditional "b" that @@ -936,15 +938,19 @@ jmp_pos = self.mc.currpos() self.mc.trap() - self.write_pending_failure_recoveries() + self.write_pending_failure_recoveries(regalloc) currpos = self.mc.currpos() pmc = OverwritingBuilder(self.mc, jmp_pos, 1) pmc.b(currpos - jmp_pos) pmc.overwrite() - def generate_quick_failure(self, guardtok): + def generate_quick_failure(self, guardtok, regalloc): startpos = self.mc.currpos() + # + self._update_at_exit(guardtok.fail_locs, guardtok.failargs, + guardtok.faildescr, regalloc) + # faildescrindex, target = self.store_info_on_descr(startpos, guardtok) assert target != 0 self.mc.load_imm(r.r2, target) @@ -957,13 +963,13 @@ self.mc.trap() return startpos - def write_pending_failure_recoveries(self): + def write_pending_failure_recoveries(self, regalloc): # for each pending guard, generate the code of the recovery stub # at the end of self.mc. for i in range(self.pending_guard_tokens_recovered, len(self.pending_guard_tokens)): tok = self.pending_guard_tokens[i] - tok.pos_recovery_stub = self.generate_quick_failure(tok) + tok.pos_recovery_stub = self.generate_quick_failure(tok, regalloc) self.pending_guard_tokens_recovered = len(self.pending_guard_tokens) def patch_pending_failure_recoveries(self, rawstart): @@ -1358,6 +1364,60 @@ self.mc.load_imm(r.SCRATCH, fail_index) self.mc.store(r.SCRATCH.value, r.SPP.value, FORCE_INDEX_OFS) + def stitch_bridge(self, faildescr, target): + """ Stitching means that one can enter a bridge with a complete different register + allocation. This needs remapping which is done here for both normal registers + and accumulation registers. + """ + import pdb; pdb.set_trace() + asminfo, bridge_faildescr, version, looptoken = target + assert isinstance(bridge_faildescr, ResumeGuardDescr) + assert isinstance(faildescr, ResumeGuardDescr) + assert asminfo.rawstart != 0 + self.mc = codebuf.MachineCodeBlockWrapper() + allblocks = self.get_asmmemmgr_blocks(looptoken) + self.datablockwrapper = MachineDataBlockWrapper(self.cpu.asmmemmgr, + allblocks) + frame_info = self.datablockwrapper.malloc_aligned( + jitframe.JITFRAMEINFO_SIZE, alignment=WORD) + + self.mc.force_frame_size(DEFAULT_FRAME_BYTES) + # if accumulation is saved at the guard, we need to update it here! + guard_locs = self.rebuild_faillocs_from_descr(faildescr, version.inputargs) + bridge_locs = self.rebuild_faillocs_from_descr(bridge_faildescr, version.inputargs) + #import pdb; pdb.set_trace() + guard_accum_info = faildescr.rd_vector_info + # O(n**2), but usually you only have at most 1 fail argument + while guard_accum_info: + bridge_accum_info = bridge_faildescr.rd_vector_info + while bridge_accum_info: + if bridge_accum_info.failargs_pos == guard_accum_info.failargs_pos: + # the mapping might be wrong! + if bridge_accum_info.location is not guard_accum_info.location: + self.mov(guard_accum_info.location, bridge_accum_info.location) + bridge_accum_info = bridge_accum_info.next() + guard_accum_info = guard_accum_info.next() + + # register mapping is most likely NOT valid, thus remap it in this + # short piece of assembler + assert len(guard_locs) == len(bridge_locs) + for i,gloc in enumerate(guard_locs): + bloc = bridge_locs[i] + bstack = bloc.location_code() == 'b' + gstack = gloc.location_code() == 'b' + if bstack and gstack: + pass + elif gloc is not bloc: + self.mov(gloc, bloc) + offset = self.mc.get_relative_pos() + self.mc.JMP_l(0) + self.mc.writeimm32(0) + self.mc.force_frame_size(DEFAULT_FRAME_BYTES) + rawstart = self.materialize_loop(looptoken) + # update the jump (above) to the real trace + self._patch_jump_to(rawstart + offset, asminfo.rawstart) + # update the guard to jump right to this custom piece of assembler + self.patch_jump_for_descr(faildescr, rawstart) def notimplemented_op(self, op, arglocs, regalloc): msg = '[PPC/asm] %s not implemented\n' % op.getopname() diff --git a/rpython/jit/backend/ppc/regalloc.py b/rpython/jit/backend/ppc/regalloc.py --- a/rpython/jit/backend/ppc/regalloc.py +++ b/rpython/jit/backend/ppc/regalloc.py @@ -10,7 +10,8 @@ from rpython.jit.backend.ppc.helper.regalloc import _check_imm_arg, check_imm_box from rpython.jit.backend.ppc.helper import regalloc as helper from rpython.jit.metainterp.history import (Const, ConstInt, ConstFloat, ConstPtr, - INT, REF, FLOAT, VOID, VECTOR) + INT, REF, FLOAT, VOID, VECTOR, + AbstractFailDescr) from rpython.jit.metainterp.history import JitCellToken, TargetToken from rpython.jit.metainterp.resoperation import rop from rpython.jit.backend.ppc import locations @@ -370,7 +371,7 @@ self.vrm._check_invariants() self.ivrm._check_invariants() if self.assembler.mc.get_relative_pos() > self.limit_loop_break: - self.assembler.break_long_loop() + self.assembler.break_long_loop(self) self.limit_loop_break = (self.assembler.mc.get_relative_pos() + LIMIT_LOOP_BREAK) i += 1 @@ -411,10 +412,16 @@ return gcmap def loc(self, var): - if var.type == FLOAT: - return self.fprm.loc(var) + if var.is_vector(): + if var.type == FLOAT: + return self.vrm.loc(var) + else: + return self.ivrm.loc(var) else: - return self.rm.loc(var) + if var.type == FLOAT: + return self.fprm.loc(var) + else: + return self.rm.loc(var) def next_instruction(self): self.rm.next_instruction() @@ -607,11 +614,24 @@ args.append(self.loc(arg)) else: args.append(None) - self.possibly_free_vars(op.getfailargs()) # # generate_quick_failure() produces up to 14 instructions per guard self.limit_loop_break -= 14 * 4 - # + # specifically for vecopt + descr = op.getdescr() + if not descr: + return args + assert isinstance(descr, AbstractFailDescr) + if descr.rd_vector_info: + accuminfo = descr.rd_vector_info + while accuminfo: + i = accuminfo.getpos_in_failargs()+1 + accuminfo.location = args[i] + loc = self.loc(accuminfo.getoriginal()) + args[i] = loc + accuminfo = accuminfo.next() + + self.possibly_free_vars(op.getfailargs()) return args def load_condition_into_cc(self, box): diff --git a/rpython/jit/backend/ppc/runner.py b/rpython/jit/backend/ppc/runner.py --- a/rpython/jit/backend/ppc/runner.py +++ b/rpython/jit/backend/ppc/runner.py @@ -51,7 +51,7 @@ if detect_vsx(): self.vector_ext = AltiVectorExt() self.vector_extension = True - # ??? self.vector_horizontal_operations = True + self.vector_horizontal_operations = True self.assembler.setup_once_vector() @rgc.no_release_gil diff --git a/rpython/jit/backend/ppc/vector_ext.py b/rpython/jit/backend/ppc/vector_ext.py --- a/rpython/jit/backend/ppc/vector_ext.py +++ b/rpython/jit/backend/ppc/vector_ext.py @@ -10,7 +10,7 @@ from rpython.rlib.objectmodel import we_are_translated from rpython.rtyper.lltypesystem.lloperation import llop from rpython.rtyper.lltypesystem import lltype -from rpython.jit.backend.ppc.locations import imm +from rpython.jit.backend.ppc.locations import imm, RegisterLocation from rpython.jit.backend.ppc.arch import IS_BIG_ENDIAN from rpython.jit.backend.llsupport.vector_ext import VectorExt from rpython.jit.backend.ppc.arch import PARAM_SAVE_AREA_OFFSET @@ -105,7 +105,6 @@ self.mc.vperm(resloc.value, Vhi, Vlo, Vp) else: self.mc.vperm(resloc.value, Vlo, Vhi, Vp) - #self.mc.trap() def _emit_vec_setitem(self, op, arglocs, regalloc): # prepares item scale (raw_store does not) @@ -318,60 +317,57 @@ # index += 1 # self.mc.PBLENDW_xxi(loc.value, temp.value, select) - #def _update_at_exit(self, fail_locs, fail_args, faildescr, regalloc): - # """ If accumulation is done in this loop, at the guard exit - # some vector registers must be adjusted to yield the correct value - # """ - # if not isinstance(faildescr, ResumeGuardDescr): - # return - # assert regalloc is not None - # accum_info = faildescr.rd_vector_info - # while accum_info: - # pos = accum_info.getpos_in_failargs() - # scalar_loc = fail_locs[pos] - # vector_loc = accum_info.location - # # the upper elements will be lost if saved to the stack! - # scalar_arg = accum_info.getoriginal() - # assert isinstance(vector_loc, RegLoc) - # if not isinstance(scalar_loc, RegLoc): - # scalar_loc = regalloc.force_allocate_reg(scalar_arg) - # assert scalar_arg is not None - # if accum_info.accum_operation == '+': - # self._accum_reduce_sum(scalar_arg, vector_loc, scalar_loc) - # elif accum_info.accum_operation == '*': - # self._accum_reduce_mul(scalar_arg, vector_loc, scalar_loc) - # else: - # not_implemented("accum operator %s not implemented" % - # (accum_info.accum_operation)) - # accum_info = accum_info.next() + def _update_at_exit(self, fail_locs, fail_args, faildescr, regalloc): + """ If accumulation is done in this loop, at the guard exit + some vector registers must be adjusted to yield the correct value + """ + if not isinstance(faildescr, ResumeGuardDescr): + return + accum_info = faildescr.rd_vector_info + while accum_info: + pos = accum_info.getpos_in_failargs() + scalar_loc = fail_locs[pos] + vector_loc = accum_info.location + # the upper elements will be lost if saved to the stack! + scalar_arg = accum_info.getoriginal() + if not scalar_loc.is_reg(): + scalar_loc = regalloc.force_allocate_reg(scalar_arg) + assert scalar_arg is not None + if accum_info.accum_operation == '+': + self._accum_reduce_sum(scalar_arg, vector_loc, scalar_loc) + elif accum_info.accum_operation == '*': + self._accum_reduce_mul(scalar_arg, vector_loc, scalar_loc) + else: + not_implemented("accum operator %s not implemented" % + (accum_info.accum_operation)) + accum_info = accum_info.next() - #def _accum_reduce_mul(self, arg, accumloc, targetloc): - # scratchloc = X86_64_XMM_SCRATCH_REG - # self.mov(accumloc, scratchloc) - # # swap the two elements - # self.mc.SHUFPD_xxi(scratchloc.value, scratchloc.value, 0x01) - # self.mc.MULSD(accumloc, scratchloc) - # if accumloc is not targetloc: - # self.mov(accumloc, targetloc) + def _accum_reduce_mul(self, arg, accumloc, targetloc): + notimplemented("[ppc reduce mul]") + #scratchloc = X86_64_XMM_SCRATCH_REG + #self.mov(accumloc, scratchloc) + ## swap the two elements + #self.mc.SHUFPD_xxi(scratchloc.value, scratchloc.value, 0x01) + #self.mc.MULSD(accumloc, scratchloc) + #if accumloc is not targetloc: + # self.mov(accumloc, targetloc) - #def _accum_reduce_sum(self, arg, accumloc, targetloc): - # # Currently the accumulator can ONLY be the biggest - # # size for X86 -> 64 bit float/int - # if arg.type == FLOAT: - # # r = (r[0]+r[1],r[0]+r[1]) - # self.mc.HADDPD(accumloc, accumloc) - # # upper bits (> 64) are dirty (but does not matter) - # if accumloc is not targetloc: - # self.mov(accumloc, targetloc) - # return - # elif arg.type == INT: - # scratchloc = X86_64_SCRATCH_REG - # self.mc.PEXTRQ_rxi(targetloc.value, accumloc.value, 0) - # self.mc.PEXTRQ_rxi(scratchloc.value, accumloc.value, 1) - # self.mc.ADD(targetloc, scratchloc) - # return + def _accum_reduce_sum(self, arg, accumloc, targetloc): + # Currently the accumulator can ONLY be the biggest + # 64 bit float/int + tgt = targetloc.value + acc = accumloc.value + if arg.type == FLOAT: + # r = (r[0]+r[1],r[0]+r[1]) + self.mc.xvmr(tgt, acc, acc) + if IS_BIG_ENDIAN: + self.mc.xxspltd(tgt, acc, acc, 0b00) + else: + self.mc.xxspltd(tgt, acc, acc, 0b01) + self.mc.xsadddp(tgt, tgt, acc) + return - # not_implemented("reduce sum for %s not impl." % arg) + not_implemented("reduce sum for %s not impl." % arg) def emit_vec_int_is_true(self, op, arglocs, regalloc): resloc, argloc, sizeloc = arglocs @@ -408,6 +404,13 @@ self.mc.lvx(resloc.value, off, r.SP.value) flush_vec_cc(self, regalloc, c.EQ, op.bytesize, resloc) + def emit_vec_float_xor(self, op, arglocs, regalloc): + resloc, l0, l1, sizeloc = arglocs + res = resloc.value + r0 = l0.value + r1 = l1.value + self.mc.xxlxor(res, r0, r1) + def emit_vec_float_ne(self, op, arglocs, regalloc): resloc, loc1, loc2, sizeloc = arglocs size = sizeloc.value @@ -565,61 +568,49 @@ #genop_vec_unpack_i = genop_vec_pack_i - #def genop_vec_pack_f(self, op, arglocs, resultloc): - # resloc, srcloc, residxloc, srcidxloc, countloc, sizeloc = arglocs - # assert isinstance(resloc, RegLoc) - # assert isinstance(srcloc, RegLoc) - # count = countloc.value - # residx = residxloc.value - # srcidx = srcidxloc.value - # size = sizeloc.value - # if size == 4: - # si = srcidx - # ri = residx - # k = count - # while k > 0: - # if resloc.is_xmm: - # src = srcloc.value - # if not srcloc.is_xmm: - # # if source is a normal register (unpack) - # assert count == 1 - # assert si == 0 - # self.mov(srcloc, X86_64_XMM_SCRATCH_REG) - # src = X86_64_XMM_SCRATCH_REG.value - # select = ((si & 0x3) << 6)|((ri & 0x3) << 4) - # self.mc.INSERTPS_xxi(resloc.value, src, select) - # else: - # self.mc.PEXTRD_rxi(resloc.value, srcloc.value, si) - # si += 1 - # ri += 1 - # k -= 1 - # elif size == 8: - # assert resloc.is_xmm - # if srcloc.is_xmm: - # if srcidx == 0: - # if residx == 0: - # # r = (s[0], r[1]) - # self.mc.MOVSD(resloc, srcloc) - # else: - # assert residx == 1 - # # r = (r[0], s[0]) - # self.mc.UNPCKLPD(resloc, srcloc) - # else: - # assert srcidx == 1 - # if residx == 0: - # # r = (s[1], r[1]) - # if resloc != srcloc: - # self.mc.UNPCKHPD(resloc, srcloc) - # self.mc.SHUFPD_xxi(resloc.value, resloc.value, 1) - # else: - # assert residx == 1 - # # r = (r[0], s[1]) - # if resloc != srcloc: - # self.mc.SHUFPD_xxi(resloc.value, resloc.value, 1) - # self.mc.UNPCKHPD(resloc, srcloc) - # # if they are equal nothing is to be done + def emit_vec_pack_f(self, op, arglocs, resultloc): + resloc, vloc, srcloc, residxloc, srcidxloc, countloc = arglocs + vec = vloc.value + res = resloc.value + src = srcloc.value + count = countloc.value + residx = residxloc.value + srcidx = srcidxloc.value + size = op.bytesize + assert size == 8 + # srcloc is always a floating point register f, this means it is + # vsr[0] == valueof(f) + if srcidx == 0: + if residx == 0: + # r = (s[0], r[1]) + if IS_BIG_ENDIAN: + self.mc.xxspltd(res, src, vec, 0b10) + else: + self.mc.xxspltd(res, src, vec, 0b01) + else: + assert residx == 1 + # r = (r[0], s[0]) + if IS_BIG_ENDIAN: + self.mc.xxspltd(res, vec, src, 0b00) + else: + self.mc.xxspltd(res, vec, src, 0b11) + else: + assert srcidx == 1 + if residx == 0: + # r = (s[1], r[1]) + if IS_BIG_ENDIAN: + self.mc.xxspltd(res, src, vec, 0b11) + else: + self.mc.xxspltd(res, src, vec, 0b00) + else: + assert residx == 1 + # r = (r[0], s[1]) + if IS_BIG_ENDIAN: + self.mc.xxspltd(res, vec, src, 0b10) + else: + self.mc.xxspltd(res, vec, src, 0b01) - #genop_vec_unpack_f = genop_vec_pack_f + emit_vec_unpack_f = emit_vec_pack_f # needed as soon as PPC's support_singlefloat is implemented! #def genop_vec_cast_float_to_int(self, op, arglocs, regalloc): @@ -627,6 +618,10 @@ #def genop_vec_cast_singlefloat_to_float(self, op, arglocs, regalloc): # self.mc.CVTPS2PD(resloc, arglocs[0]) + def emit_vec_f(self, op, arglocs, regalloc): + pass + emit_vec_i = emit_vec_f + class VectorRegalloc(object): _mixin_ = True @@ -709,9 +704,10 @@ prepare_vec_int_xor = prepare_vec_arith prepare_vec_float_eq = prepare_vec_arith - prepare_vec_float_ne = prepare_vec_float_eq - prepare_vec_int_eq = prepare_vec_float_eq - prepare_vec_int_ne = prepare_vec_float_eq + prepare_vec_float_ne = prepare_vec_arith + prepare_vec_int_eq = prepare_vec_arith + prepare_vec_int_ne = prepare_vec_arith + prepare_vec_float_xor = prepare_vec_arith del prepare_vec_arith @@ -751,24 +747,35 @@ prepare_vec_float_abs = prepare_vec_arith_unary del prepare_vec_arith_unary - #def prepare_vec_pack_i(self, op): - # # new_res = vec_pack_i(res, src, index, count) - # assert isinstance(op, VectorOp) - # arg = op.getarg(1) - # index = op.getarg(2) - # count = op.getarg(3) - # assert isinstance(index, ConstInt) - # assert isinstance(count, ConstInt) - # args = op.getarglist() - # srcloc = self.make_sure_var_in_reg(arg, args) - # resloc = self.xrm.force_result_in_reg(op, op.getarg(0), args) - # residx = index.value # where to put it in result? - # srcidx = 0 - # arglocs = [resloc, srcloc, imm(residx), imm(srcidx), - # imm(count.value), imm(op.bytesize)] - # self.perform(op, arglocs, resloc) + def prepare_vec_pack_i(self, op): + # new_res = vec_pack_i(res, src, index, count) + assert isinstance(op, VectorOp) + arg = op.getarg(1) + index = op.getarg(2) + count = op.getarg(3) + assert isinstance(index, ConstInt) + assert isinstance(count, ConstInt) + srcloc = self.ensure_vector_reg(arg) + resloc = self.force_allocate_vector_reg(op) + residx = index.value # where to put it in result? + srcidx = 0 + return [resloc, srcloc, imm(residx), imm(srcidx), imm(count.value)] - #prepare_vec_pack_f = prepare_vec_pack_i + def prepare_vec_pack_f(self, op): + # new_res = vec_pack_i(res, src, index, count) + assert isinstance(op, VectorOp) + arg = op.getarg(1) + index = op.getarg(2) + count = op.getarg(3) + assert isinstance(index, ConstInt) + assert isinstance(count, ConstInt) + assert not arg.is_vector() + srcloc = self.ensure_reg(arg) + vloc = self.ensure_vector_reg(op.getarg(0)) + resloc = self.force_allocate_vector_reg(op) + residx = index.value # where to put it in result? + srcidx = 0 + return [resloc, vloc, srcloc, imm(residx), imm(srcidx), imm(count.value)] #def prepare_vec_unpack_i(self, op): # assert isinstance(op, VectorOp) diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py --- a/rpython/jit/backend/x86/assembler.py +++ b/rpython/jit/backend/x86/assembler.py @@ -645,7 +645,7 @@ bridge_locs = self.rebuild_faillocs_from_descr(bridge_faildescr, version.inputargs) #import pdb; pdb.set_trace() guard_accum_info = faildescr.rd_vector_info - # O(n^2), but usually you only have at most 1 fail argument + # O(n**2), but usually you only have at most 1 fail argument while guard_accum_info: bridge_accum_info = bridge_faildescr.rd_vector_info while bridge_accum_info: diff --git a/rpython/jit/metainterp/optimizeopt/vector.py b/rpython/jit/metainterp/optimizeopt/vector.py --- a/rpython/jit/metainterp/optimizeopt/vector.py +++ b/rpython/jit/metainterp/optimizeopt/vector.py @@ -793,7 +793,10 @@ if pack.reduce_init() == 0: vecop = OpHelpers.create_vec(datatype, bytesize, signed, count) oplist.append(vecop) - vecop = VecOperation(rop.VEC_INT_XOR, [vecop, vecop], + opnum = rop.VEC_INT_XOR + if datatype == FLOAT: + opnum = rop.VEC_FLOAT_XOR + vecop = VecOperation(opnum, [vecop, vecop], vecop, count) oplist.append(vecop) elif pack.reduce_init() == 1: diff --git a/rpython/jit/metainterp/resoperation.py b/rpython/jit/metainterp/resoperation.py --- a/rpython/jit/metainterp/resoperation.py +++ b/rpython/jit/metainterp/resoperation.py @@ -994,6 +994,7 @@ '_VEC_ARITHMETIC_LAST', 'VEC_FLOAT_EQ/2b/i', 'VEC_FLOAT_NE/2b/i', + 'VEC_FLOAT_XOR/2/f', 'VEC_INT_IS_TRUE/1b/i', 'VEC_INT_NE/2b/i', 'VEC_INT_EQ/2b/i', diff --git a/rpython/jit/metainterp/test/test_vector.py b/rpython/jit/metainterp/test/test_vector.py --- a/rpython/jit/metainterp/test/test_vector.py +++ b/rpython/jit/metainterp/test/test_vector.py @@ -78,6 +78,7 @@ enable_opts = 'intbounds:rewrite:virtualize:string:earlyforce:pure:heap:unroll' def setup_method(self, method): + import pdb; pdb.set_trace() if not self.supports_vector_ext(): py.test.skip("this cpu %s has no implemented vector backend" % CPU) From pypy.commits at gmail.com Fri Jul 1 09:02:53 2016 From: pypy.commits at gmail.com (mattip) Date: Fri, 01 Jul 2016 06:02:53 -0700 (PDT) Subject: [pypy-commit] pypy call-via-pyobj: use a list to make the offset lookup work (test passes) Message-ID: <577669fd.6a25c20a.9418c.1427@mx.google.com> Author: Matti Picus Branch: call-via-pyobj Changeset: r85488:02ab445bf7f9 Date: 2016-07-01 16:01 +0300 http://bitbucket.org/pypy/pypy/changeset/02ab445bf7f9/ Log: use a list to make the offset lookup work (test passes) diff --git a/pypy/module/cpyext/methodobject.py b/pypy/module/cpyext/methodobject.py --- a/pypy/module/cpyext/methodobject.py +++ b/pypy/module/cpyext/methodobject.py @@ -152,7 +152,7 @@ class W_PyCWrapperObject(W_Root): def __init__(self, space, pto, method_name, wrapper_func, - wrapper_func_kwds, doc, func, offset=-1): + wrapper_func_kwds, doc, func, offset=None): self.space = space self.method_name = method_name self.wrapper_func = wrapper_func @@ -174,15 +174,15 @@ raise oefmt(space.w_TypeError, "wrapper %s doesn't take any keyword arguments", self.method_name) - if self.offset >= 0: - pto = rffi.cast(PyTypeObjectPtr, as_pyobj(space, self.w_objclass)) - pto_func_as_int = lltype.cast_ptr_to_int(pto) + self.offset - # XXX make pto_func the equivalent of this line - #lltype.cast_int_to_ptr(pto_func_as_int) - func_to_call = rffi.cast(rffi.VOIDP, pto.c_tp_as_number.c_nb_multiply) - # print '\ncalling', func_to_call, 'not', self.func + if self.offset: + ptr = pto = rffi.cast(PyTypeObjectPtr, as_pyobj(space, self.w_objclass)) + # make ptr the equivalent of this, using the offsets + #func_to_call = rffi.cast(rffi.VOIDP, pto.c_tp_as_number.c_nb_multiply) + for o in self.offset: + ptr_as_int = lltype.cast_ptr_to_int(ptr) + ptr = rffi.cast(rffi.VOIDPP, ptr_as_int + o)[0] + func_to_call = ptr else: - # print 'calling', self.method_name,'with no offset' func_to_call = self.func return self.wrapper_func(space, w_self, w_args, func_to_call) diff --git a/pypy/module/cpyext/typeobject.py b/pypy/module/cpyext/typeobject.py --- a/pypy/module/cpyext/typeobject.py +++ b/pypy/module/cpyext/typeobject.py @@ -296,7 +296,7 @@ for method_name, slot_names, wrapper_func, wrapper_func_kwds, doc in slotdefs_for_wrappers: if method_name in dict_w: continue - offset = rffi.offsetof(pto._T, slot_names[0]) + offset = [rffi.offsetof(pto._T, slot_names[0])] if len(slot_names) == 1: func = getattr(pto, slot_names[0]) else: @@ -304,21 +304,13 @@ struct = getattr(pto, slot_names[0]) if not struct: continue - offset += rffi.offsetof(struct._T, slot_names[1]) + offset.append(rffi.offsetof(struct._T, slot_names[1])) func = getattr(struct, slot_names[1]) func_voidp = rffi.cast(rffi.VOIDP, func) if not func: continue if wrapper_func is None and wrapper_func_kwds is None: continue - name = rffi.charp2str(pto.c_tp_name) - if method_name in ('__mul__', '__rmul__') and 'array' in name: - # print '\nsetting', name, method_name, 'from', slot_names - # print ' pto is', pto - # print ' func_voidp is', func_voidp - pass - else: - offset = -1 w_obj = W_PyCWrapperObject(space, pto, method_name, wrapper_func, wrapper_func_kwds, doc, func_voidp, offset=offset) dict_w[method_name] = space.wrap(w_obj) @@ -744,7 +736,6 @@ w_obj = _type_realize(space, py_obj) finally: name = rffi.charp2str(pto.c_tp_name) - print '_type_realize done', name pto.c_tp_flags &= ~Py_TPFLAGS_READYING pto.c_tp_flags |= Py_TPFLAGS_READY return w_obj From pypy.commits at gmail.com Fri Jul 1 09:38:13 2016 From: pypy.commits at gmail.com (plan_rich) Date: Fri, 01 Jul 2016 06:38:13 -0700 (PDT) Subject: [pypy-commit] pypy ppc-vsx-support: finished accum reduce function for f64/i64 + and * (ppc) Message-ID: <57767245.8f1d1c0a.76f43.01b7@mx.google.com> Author: Richard Plangger Branch: ppc-vsx-support Changeset: r85489:89ec178c8b17 Date: 2016-07-01 15:37 +0200 http://bitbucket.org/pypy/pypy/changeset/89ec178c8b17/ Log: finished accum reduce function for f64/i64 + and * (ppc) diff --git a/rpython/jit/backend/ppc/codebuilder.py b/rpython/jit/backend/ppc/codebuilder.py --- a/rpython/jit/backend/ppc/codebuilder.py +++ b/rpython/jit/backend/ppc/codebuilder.py @@ -612,7 +612,7 @@ # mul xvmuldp = XX3(60, XO9=112) xvmulsp = XX3(60, XO9=80) - xsmuldp = XX3(60, XO9=46) + xsmuldp = XX3(60, XO9=48) # div xvdivdp = XX3(60, XO9=102) xvdivsp = XX3(60, XO9=88) diff --git a/rpython/jit/backend/ppc/vector_ext.py b/rpython/jit/backend/ppc/vector_ext.py --- a/rpython/jit/backend/ppc/vector_ext.py +++ b/rpython/jit/backend/ppc/vector_ext.py @@ -333,38 +333,40 @@ if not scalar_loc.is_reg(): scalar_loc = regalloc.force_allocate_reg(scalar_arg) assert scalar_arg is not None - if accum_info.accum_operation == '+': - self._accum_reduce_sum(scalar_arg, vector_loc, scalar_loc) - elif accum_info.accum_operation == '*': - self._accum_reduce_mul(scalar_arg, vector_loc, scalar_loc) - else: - not_implemented("accum operator %s not implemented" % - (accum_info.accum_operation)) + op = accum_info.accum_operation + self._accum_reduce(op, scalar_arg, vector_loc, scalar_loc) accum_info = accum_info.next() - def _accum_reduce_mul(self, arg, accumloc, targetloc): - notimplemented("[ppc reduce mul]") - #scratchloc = X86_64_XMM_SCRATCH_REG - #self.mov(accumloc, scratchloc) - ## swap the two elements - #self.mc.SHUFPD_xxi(scratchloc.value, scratchloc.value, 0x01) - #self.mc.MULSD(accumloc, scratchloc) - #if accumloc is not targetloc: - # self.mov(accumloc, targetloc) - - def _accum_reduce_sum(self, arg, accumloc, targetloc): + def _accum_reduce(self, op, arg, accumloc, targetloc): # Currently the accumulator can ONLY be the biggest # 64 bit float/int tgt = targetloc.value acc = accumloc.value if arg.type == FLOAT: # r = (r[0]+r[1],r[0]+r[1]) - self.mc.xvmr(tgt, acc, acc) if IS_BIG_ENDIAN: self.mc.xxspltd(tgt, acc, acc, 0b00) else: - self.mc.xxspltd(tgt, acc, acc, 0b01) - self.mc.xsadddp(tgt, tgt, acc) + self.mc.xxspltd(tgt, acc, acc, 0b10) + if op == '+': + self.mc.xsadddp(tgt, tgt, acc) + elif op == '*': + self.mc.xsmuldp(tgt, tgt, acc) + else: + not_implemented("sum not implemented") + return + else: + assert arg.type == INT + self.mc.load_imm(r.SCRATCH2, PARAM_SAVE_AREA_OFFSET) + self.mc.stvx(acc, r.SCRATCH2.value, r.SP.value) + self.mc.load(tgt, r.SP.value, PARAM_SAVE_AREA_OFFSET) + self.mc.load(r.SCRATCH.value, r.SP.value, PARAM_SAVE_AREA_OFFSET+8) + if op == '+': + self.mc.add(tgt, tgt, acc) + elif op == '*': + self.mc.mul(tgt, tgt, acc) + else: + not_implemented("sum not implemented") return not_implemented("reduce sum for %s not impl." % arg) @@ -514,59 +516,37 @@ else: notimplemented("[expand int size not impl]") - #def genop_vec_pack_i(self, op, arglocs, regalloc): - # resultloc, sourceloc, residxloc, srcidxloc, countloc, sizeloc = arglocs - # assert isinstance(resultloc, RegLoc) - # assert isinstance(sourceloc, RegLoc) - # size = sizeloc.value - # srcidx = srcidxloc.value - # residx = residxloc.value - # count = countloc.value - # # for small data type conversion this can be quite costy - # # NOTE there might be some combinations that can be handled - # # more efficiently! e.g. - # # v2 = pack(v0,v1,4,4) - # si = srcidx - # ri = residx - # k = count - # while k > 0: - # if size == 8: - # if resultloc.is_xmm and sourceloc.is_xmm: # both xmm - # self.mc.PEXTRQ_rxi(X86_64_SCRATCH_REG.value, sourceloc.value, si) - # self.mc.PINSRQ_xri(resultloc.value, X86_64_SCRATCH_REG.value, ri) - # elif resultloc.is_xmm: # xmm <- reg - # self.mc.PINSRQ_xri(resultloc.value, sourceloc.value, ri) - # else: # reg <- xmm - # self.mc.PEXTRQ_rxi(resultloc.value, sourceloc.value, si) - # elif size == 4: - # if resultloc.is_xmm and sourceloc.is_xmm: - # self.mc.PEXTRD_rxi(X86_64_SCRATCH_REG.value, sourceloc.value, si) - # self.mc.PINSRD_xri(resultloc.value, X86_64_SCRATCH_REG.value, ri) - # elif resultloc.is_xmm: - # self.mc.PINSRD_xri(resultloc.value, sourceloc.value, ri) - # else: - # self.mc.PEXTRD_rxi(resultloc.value, sourceloc.value, si) - # elif size == 2: - # if resultloc.is_xmm and sourceloc.is_xmm: - # self.mc.PEXTRW_rxi(X86_64_SCRATCH_REG.value, sourceloc.value, si) - # self.mc.PINSRW_xri(resultloc.value, X86_64_SCRATCH_REG.value, ri) - # elif resultloc.is_xmm: - # self.mc.PINSRW_xri(resultloc.value, sourceloc.value, ri) - # else: - # self.mc.PEXTRW_rxi(resultloc.value, sourceloc.value, si) - # elif size == 1: - # if resultloc.is_xmm and sourceloc.is_xmm: - # self.mc.PEXTRB_rxi(X86_64_SCRATCH_REG.value, sourceloc.value, si) - # self.mc.PINSRB_xri(resultloc.value, X86_64_SCRATCH_REG.value, ri) - # elif resultloc.is_xmm: - # self.mc.PINSRB_xri(resultloc.value, sourceloc.value, ri) - # else: - # self.mc.PEXTRB_rxi(resultloc.value, sourceloc.value, si) - # si += 1 - # ri += 1 - # k -= 1 + def emit_vec_pack_i(self, op, arglocs, regalloc): + resultloc, vloc, sourceloc, residxloc, srcidxloc, countloc = arglocs + srcidx = srcidxloc.value + residx = residxloc.value + count = countloc.value + # for small data type conversion this can be quite costy + # NOTE there might be some combinations that can be handled + # more efficiently! e.g. + # v2 = pack(v0,v1,4,4) + res = resultloc.value + vector = vloc.value + src = sourceloc.value + size = op.bytesize + if size == 8: + if resultloc.is_vector_reg() and sourceloc.is_vector_reg(): # both vector + notimplemented("[ppc/vec_pack_i]") + elif resultloc.is_vector_reg(): # vector <- reg + self.mc.load_imm(r.SCRATCH, PARAM_SAVE_AREA_OFFSET) + self.mc.stvx(vector, r.SCRATCH2.value, r.SP.value) + self.mc.store(src, r.SP.value, PARAM_SAVE_AREA_OFFSET+8*residx) + self.mc.lvx(res, r.SCRATCH2.value, r.SP.value) + else: + notimplemented("[ppc/vec_pack_i]") + elif size == 4: + notimplemented("[ppc/vec_pack_i]") + elif size == 2: + notimplemented("[ppc/vec_pack_i]") + elif size == 1: + notimplemented("[ppc/vec_pack_i]") - #genop_vec_unpack_i = genop_vec_pack_i + # TODO emit_vec_unpack_i = emit_vec_pack_i def emit_vec_pack_f(self, op, arglocs, resultloc): resloc, vloc, srcloc, residxloc, srcidxloc, countloc = arglocs @@ -755,11 +735,12 @@ count = op.getarg(3) assert isinstance(index, ConstInt) assert isinstance(count, ConstInt) - srcloc = self.ensure_vector_reg(arg) + vloc = self.ensure_vector_reg(op.getarg(0)) + srcloc = self.ensure_reg(arg) resloc = self.force_allocate_vector_reg(op) residx = index.value # where to put it in result? srcidx = 0 - return [resloc, srcloc, imm(residx), imm(srcidx), imm(count.value)] + return [resloc, vloc, srcloc, imm(residx), imm(srcidx), imm(count.value)] def prepare_vec_pack_f(self, op): # new_res = vec_pack_i(res, src, index, count) @@ -769,7 +750,7 @@ count = op.getarg(3) assert isinstance(index, ConstInt) assert isinstance(count, ConstInt) - assert not arg.is_vector() + assert not arg.is_vector_reg() srcloc = self.ensure_reg(arg) vloc = self.ensure_vector_reg(op.getarg(0)) resloc = self.force_allocate_vector_reg(op) diff --git a/rpython/jit/metainterp/test/test_vector.py b/rpython/jit/metainterp/test/test_vector.py --- a/rpython/jit/metainterp/test/test_vector.py +++ b/rpython/jit/metainterp/test/test_vector.py @@ -379,23 +379,30 @@ res = self.meta_interp(f, [count]) assert res == f(count) == breaks - def test_sum(self): + @py.test.mark.parametrize('type,func,cast', + [(rffi.DOUBLE, lambda a,b: a+b, float), + (rffi.DOUBLE, lambda a,b: a*b, float), + (lltype.Signed, lambda a,b: a+b, int), + (lltype.Signed, lambda a,b: a*b, int), + ]) + def test_reduce(self, type, func, cast): + func = always_inline(func) myjitdriver = JitDriver(greens = [], reds = 'auto', vectorize=True) - T = lltype.Array(rffi.DOUBLE, hints={'nolength': True}) + T = lltype.Array(type, hints={'nolength': True}) def f(d): va = lltype.malloc(T, d, flavor='raw', zero=True) for j in range(d): - va[j] = float(j) + va[j] = cast(j+1) i = 0 accum = 0 while i < d: myjitdriver.jit_merge_point() - accum += va[i] + accum = func(accum,va[i]) i += 1 lltype.free(va, flavor='raw') return accum res = self.meta_interp(f, [60]) - assert res == f(60) == sum(range(60)) + assert isclose(res, f(60)) def test_constant_expand(self): myjitdriver = JitDriver(greens = [], reds = 'auto', vectorize=True) From pypy.commits at gmail.com Fri Jul 1 10:09:11 2016 From: pypy.commits at gmail.com (arigo) Date: Fri, 01 Jul 2016 07:09:11 -0700 (PDT) Subject: [pypy-commit] pypy reverse-debugger: Tweak again the breakpoint detection code and the "next" and "bnext" commands Message-ID: <57767987.cc9d1c0a.c6c63.09c6@mx.google.com> Author: Armin Rigo Branch: reverse-debugger Changeset: r85490:b5ea52f65b84 Date: 2016-07-01 16:10 +0200 http://bitbucket.org/pypy/pypy/changeset/b5ea52f65b84/ Log: Tweak again the breakpoint detection code and the "next" and "bnext" commands diff --git a/pypy/interpreter/reverse_debugging.py b/pypy/interpreter/reverse_debugging.py --- a/pypy/interpreter/reverse_debugging.py +++ b/pypy/interpreter/reverse_debugging.py @@ -59,14 +59,14 @@ name = callee_frame.getcode().co_name if name in dbstate.breakpoint_funcnames: revdb.breakpoint(dbstate.breakpoint_funcnames[name]) - if dbstate.breakpoint_stack_id != 0: + if dbstate.breakpoint_stack_id != 0 and caller_frame is not None: if dbstate.breakpoint_stack_id == revdb.get_unique_id(caller_frame): revdb.breakpoint(-1) def leave_call(caller_frame, callee_frame): - if dbstate.breakpoint_stack_id != 0: + if dbstate.breakpoint_stack_id != 0 and caller_frame is not None: if dbstate.breakpoint_stack_id == revdb.get_unique_id(caller_frame): - revdb.breakpoint(-1) + revdb.breakpoint(-2) def jump_backward(frame, jumpto): diff --git a/rpython/translator/revdb/interact.py b/rpython/translator/revdb/interact.py --- a/rpython/translator/revdb/interact.py +++ b/rpython/translator/revdb/interact.py @@ -170,28 +170,31 @@ self.pgroup.go_forward(steps) return None except Breakpoint as b: - self.hit_breakpoint(b) + self.hit_breakpoints(b) return b - def move_backward(self, steps, rel_stop_at=-1): - ignore_bkpt = steps == 1 and rel_stop_at == -1 + def move_backward(self, steps): try: - self.pgroup.go_backward(steps, ignore_breakpoints=ignore_bkpt, - rel_stop_at=rel_stop_at) + self.pgroup.go_backward(steps) return None except Breakpoint as b: - self.hit_breakpoint(b, backward=True) + self.hit_breakpoints(b, backward=True) return b - def hit_breakpoint(self, b, backward=False): - if b.num != -1: - kind, name = self._bp_kind(b.num) - self.print_extra_pending_info = 'Hit %s %d: %s' % (kind, b.num, - name) - elif backward: - b.time -= 1 - if self.pgroup.get_current_time() != b.time: - self.pgroup.jump_in_time(b.time) + def hit_breakpoints(self, b, backward=False): + printing = [] + for num in b.regular_breakpoint_nums(): + kind, name = self._bp_kind(num) + printing.append('%s %s %d: %s' % ( + 'Reverse-hit' if backward else 'Hit', + kind, num, name)) + self.print_extra_pending_info = '\n'.join(printing) + target_time = b.time + if backward: + target_time -= 1 # when going backwards, we stop just before + # the breakpoint time, as opposed to just after + if self.pgroup.get_current_time() != target_time: + self.pgroup.jump_in_time(target_time) def remove_tainting(self): if self.pgroup.is_tainted(): @@ -226,33 +229,44 @@ stack_id = self.pgroup.get_stack_id(is_parent=False) with self._stack_id_break(stack_id): b = self.move_forward(1) - if b is None: - return # no breakpoint hit, and no frame just entered: done - elif b.num != -1: - return # a regular breakpoint was hit - else: - # we entered a frame. Continue running until we leave that - # frame again + while b is not None: + # if we hit a regular breakpoint, stop + if any(b.regular_breakpoint_nums()): + return + # we hit only calls and returns inside stack_id. If the + # last one of these is a "return", then we're now back inside + # stack_id, so stop + if b.nums[-1] == -2: + return + # else, the last one is a "call", so we entered another frame. + # Continue running until the next call/return event occurs + # inside stack_id with self._stack_id_break(stack_id): - self.command_continue("") + b = self.move_forward(self.pgroup.get_max_time() - + self.pgroup.get_current_time()) + # and then look at that 'b' again (closes the loop) command_n = command_next def command_bnext(self, argument): """Run backward for one step, skipping calls""" stack_id = self.pgroup.get_stack_id(is_parent=False) with self._stack_id_break(stack_id): - b = self.move_backward(1, rel_stop_at=0) - if b is None: - return # no breakpoint hit, and no frame just - # reverse-entered: done - elif b.num != -1: - return # a regular breakpoint was hit - else: - # we reverse-entered a frame. Continue running backward - # until we go past the reverse-leave (i.e. the entering) - # of that frame. + b = self.move_backward(1) + while b is not None: + # if we hit a regular breakpoint, stop + if any(b.regular_breakpoint_nums()): + return + # we hit only calls and returns inside stack_id. If the + # first one of these is a "call", then we're now back inside + # stack_id, so stop + if b.nums[0] == -1: + return + # else, the first one is a "return", so before, we were + # inside a different frame. Continue running until the next + # call/return event occurs inside stack_id with self._stack_id_break(stack_id): - self.command_bcontinue("") + b = self.move_backward(self.pgroup.get_current_time() - 1) + # and then look at that 'b' again (closes the loop) command_bn = command_bnext def command_finish(self, argument): diff --git a/rpython/translator/revdb/process.py b/rpython/translator/revdb/process.py --- a/rpython/translator/revdb/process.py +++ b/rpython/translator/revdb/process.py @@ -5,12 +5,21 @@ class Breakpoint(Exception): - def __init__(self, time, num): - self.time = time - self.num = num + def __init__(self, time): + self.time = time # time of the previous stop_point + self.nums = [] # list of breakpoint numbers that occurred, in order + + def record_num(self, num): + self.nums.append(num) + + def regular_breakpoint_nums(self): + for num in self.nums: + if num != -1 and num != -2: + yield num def __repr__(self): - return 'Breakpoint(%d, %d)' % (self.time, self.num) + return 'Breakpoint(%d, %r)' % (self.time, self.nums) + __str__ = __repr__ class AllBreakpoints(object): @@ -151,15 +160,15 @@ self.send(Message(CMD_FORWARD, steps, ord(breakpoint_mode))) # - # record the first ANSWER_BREAKPOINT, drop the others - # (in corner cases only could we get more than one) + # record all breakpoints that occur together during the *last* step bkpt = None while True: msg = self.recv() if msg.cmd != ANSWER_BREAKPOINT: break - if bkpt is None: - bkpt = Breakpoint(msg.arg1, msg.arg3) + if bkpt is None or bkpt.time != msg.arg1: + bkpt = Breakpoint(msg.arg1) + bkpt.record_num(msg.arg3) assert msg.cmd == ANSWER_READY, msg self.update_times(msg) return bkpt @@ -306,7 +315,7 @@ if bkpt: raise bkpt - def go_backward(self, steps, ignore_breakpoints=False, rel_stop_at=-1): + def go_backward(self, steps, ignore_breakpoints=False): """Go backward, for the given number of 'steps' of time. Closes the active process. Implemented as jump_in_time() @@ -323,7 +332,7 @@ first_steps = 957 self._backward_search_forward( search_start_time = initial_time - first_steps, - search_stop_time = initial_time + rel_stop_at, + search_stop_time = initial_time, search_go_on_until_time = initial_time - steps) def _backward_search_forward(self, search_start_time, search_stop_time, diff --git a/rpython/translator/revdb/src-revdb/revdb.c b/rpython/translator/revdb/src-revdb/revdb.c --- a/rpython/translator/revdb/src-revdb/revdb.c +++ b/rpython/translator/revdb/src-revdb/revdb.c @@ -483,6 +483,8 @@ #define ANSWER_AT_END (-23) #define ANSWER_BREAKPOINT (-24) +#define RECORD_BKPT_NUM 50 + typedef void (*rpy_revdb_command_fn)(rpy_revdb_command_t *, RPyString *); static int rpy_rev_sockfd; @@ -493,7 +495,8 @@ static void (*pending_after_forward)(void); static RPyString *empty_string; static uint64_t last_recorded_breakpoint_loc; -static int last_recorded_breakpoint_num; +static int n_last_recorded_breakpoints; +static int last_recorded_breakpoint_nums[RECORD_BKPT_NUM]; static char breakpoint_mode = 'i'; static uint64_t *future_ids, *future_next_id; static void *finalizer_tree, *destructor_tree; @@ -927,10 +930,11 @@ static void answer_recorded_breakpoint(void) { - if (last_recorded_breakpoint_loc != 0) { + int i; + for (i = 0; i < n_last_recorded_breakpoints; i++) write_answer(ANSWER_BREAKPOINT, last_recorded_breakpoint_loc, - 0, last_recorded_breakpoint_num); - } + 0, last_recorded_breakpoint_nums[i]); + n_last_recorded_breakpoints = 0; } static void command_forward(rpy_revdb_command_t *cmd) @@ -943,7 +947,7 @@ interactive_break = saved_state.stop_point_seen + cmd->arg1; breakpoint_mode = (char)cmd->arg2; if (breakpoint_mode == 'r') { - last_recorded_breakpoint_loc = 0; + n_last_recorded_breakpoints = 0; pending_after_forward = &answer_recorded_breakpoint; } } @@ -1102,8 +1106,14 @@ return; /* ignored breakpoints */ case 'r': /* record the breakpoint but continue */ - last_recorded_breakpoint_loc = rpy_revdb.stop_point_seen + 1; - last_recorded_breakpoint_num = num; + if (last_recorded_breakpoint_loc != rpy_revdb.stop_point_seen + 1) { + last_recorded_breakpoint_loc = rpy_revdb.stop_point_seen + 1; + n_last_recorded_breakpoints = 0; + } + if (n_last_recorded_breakpoints < RECORD_BKPT_NUM) { + last_recorded_breakpoint_nums[n_last_recorded_breakpoints] = num; + n_last_recorded_breakpoints++; + } return; case 'b': /* default handling of breakpoints */ From pypy.commits at gmail.com Fri Jul 1 10:25:47 2016 From: pypy.commits at gmail.com (plan_rich) Date: Fri, 01 Jul 2016 07:25:47 -0700 (PDT) Subject: [pypy-commit] pypy ppc-vsx-support: changes vec_guard to pass more tests (work in progress) Message-ID: <57767d6b.2457c20a.cc61d.3f17@mx.google.com> Author: Richard Plangger Branch: ppc-vsx-support Changeset: r85491:dd3b78c15f4f Date: 2016-07-01 16:25 +0200 http://bitbucket.org/pypy/pypy/changeset/dd3b78c15f4f/ Log: changes vec_guard to pass more tests (work in progress) diff --git a/rpython/jit/backend/ppc/vector_ext.py b/rpython/jit/backend/ppc/vector_ext.py --- a/rpython/jit/backend/ppc/vector_ext.py +++ b/rpython/jit/backend/ppc/vector_ext.py @@ -267,10 +267,10 @@ else: notimplemented("[ppc/assembler] float neg for size %d" % size) - def emit_guard_vec_guard_true(self, guard_op, guard_token, arglocs, regalloc): + def emit_vec_guard_true(self, guard_op, arglocs, regalloc): self._emit_guard(guard_op, arglocs) - def emit_guard_vec_guard_false(self, guard_op, guard_token, arglocs, regalloc): + def emit_vec_guard_false(self, guard_op, arglocs, regalloc): self.guard_success_cc = c.negate(self.guard_success_cc) self._emit_guard(guard_op, arglocs) @@ -374,7 +374,7 @@ def emit_vec_int_is_true(self, op, arglocs, regalloc): resloc, argloc, sizeloc = arglocs size = sizeloc.value - tmp = regalloc.get_scratch_reg().value + tmp = regalloc.ivrm.get_scratch_reg().value self.mc.vxor(tmp, tmp, tmp) # argloc[i] > 0: # For an unsigned integer that is equivalent to argloc[i] != 0 @@ -521,24 +521,18 @@ srcidx = srcidxloc.value residx = residxloc.value count = countloc.value - # for small data type conversion this can be quite costy - # NOTE there might be some combinations that can be handled - # more efficiently! e.g. - # v2 = pack(v0,v1,4,4) res = resultloc.value vector = vloc.value src = sourceloc.value size = op.bytesize if size == 8: - if resultloc.is_vector_reg() and sourceloc.is_vector_reg(): # both vector - notimplemented("[ppc/vec_pack_i]") - elif resultloc.is_vector_reg(): # vector <- reg + if resultloc.is_vector_reg(): # vector <- reg self.mc.load_imm(r.SCRATCH, PARAM_SAVE_AREA_OFFSET) self.mc.stvx(vector, r.SCRATCH2.value, r.SP.value) self.mc.store(src, r.SP.value, PARAM_SAVE_AREA_OFFSET+8*residx) self.mc.lvx(res, r.SCRATCH2.value, r.SP.value) else: - notimplemented("[ppc/vec_pack_i]") + notimplemented("[ppc/vec_pack_i] 64 bit float") elif size == 4: notimplemented("[ppc/vec_pack_i]") elif size == 2: @@ -546,9 +540,22 @@ elif size == 1: notimplemented("[ppc/vec_pack_i]") - # TODO emit_vec_unpack_i = emit_vec_pack_i + def emit_vec_unpack_i(self, op, arglocs, regalloc): + resloc, srcloc, idxloc, countloc = arglocs + idx = idxloc.value + res = resloc.value + src = srcloc.value + size = op.bytesize + if size == 8: + if srcloc.is_vector_reg(): # reg <- vector + assert not resloc.is_vector_reg() + self.mc.load_imm(r.SCRATCH, PARAM_SAVE_AREA_OFFSET) + self.mc.stvx(src, r.SCRATCH2.value, r.SP.value) + self.mc.load(res, r.SP.value, PARAM_SAVE_AREA_OFFSET+8*idx) + else: + notimplemented("[ppc/vec_unpack_i] 64 bit integer") - def emit_vec_pack_f(self, op, arglocs, resultloc): + def emit_vec_pack_f(self, op, arglocs, regalloc): resloc, vloc, srcloc, residxloc, srcidxloc, countloc = arglocs vec = vloc.value res = resloc.value @@ -590,7 +597,9 @@ else: self.mc.xxspltd(res, vec, src, 0b01) - emit_vec_unpack_f = emit_vec_pack_f + def emit_vec_unpack_f(self, op, arglocs, regalloc): + resloc, srcloc, idxloc, countloc = arglocs + self.emit_vec_pack_f(op, [resloc, srcloc, srcloc, imm(0), idxloc, countloc], regalloc) # needed as soon as PPC's support_singlefloat is implemented! #def genop_vec_cast_float_to_int(self, op, arglocs, regalloc): @@ -750,7 +759,7 @@ count = op.getarg(3) assert isinstance(index, ConstInt) assert isinstance(count, ConstInt) - assert not arg.is_vector_reg() + assert not arg.is_vector() srcloc = self.ensure_reg(arg) vloc = self.ensure_vector_reg(op.getarg(0)) resloc = self.force_allocate_vector_reg(op) @@ -758,29 +767,28 @@ srcidx = 0 return [resloc, vloc, srcloc, imm(residx), imm(srcidx), imm(count.value)] - #def prepare_vec_unpack_i(self, op): - # assert isinstance(op, VectorOp) - # index = op.getarg(1) - # count = op.getarg(2) - # assert isinstance(index, ConstInt) - # assert isinstance(count, ConstInt) - # args = op.getarglist() - # srcloc = self.make_sure_var_in_reg(op.getarg(0), args) - # if op.is_vector(): - # resloc = self.xrm.force_result_in_reg(op, op.getarg(0), args) - # size = op.bytesize - # else: - # # unpack into iX box - # resloc = self.force_allocate_reg(op, args) - # arg = op.getarg(0) - # assert isinstance(arg, VectorOp) - # size = arg.bytesize - # residx = 0 - # args = op.getarglist() - # arglocs = [resloc, srcloc, imm(residx), imm(index.value), imm(count.value), imm(size)] - # self.perform(op, arglocs, resloc) + def prepare_vec_unpack_f(self, op): + index = op.getarg(1) + count = op.getarg(2) + assert isinstance(index, ConstInt) + assert isinstance(count, ConstInt) + srcloc = self.ensure_vector_reg(op.getarg(0)) + resloc = self.force_allocate_reg(op) + return [resloc, srcloc, imm(index.value), imm(count.value)] - #prepare_vec_unpack_f = prepare_vec_unpack_i + def prepare_vec_unpack_i(self, op): + assert isinstance(op, VectorOp) + index = op.getarg(1) + count = op.getarg(2) + assert isinstance(index, ConstInt) + assert isinstance(count, ConstInt) + arg = op.getarg(0) + if arg.is_vector(): + srcloc = self.ensure_vector_reg(op.getarg(0)) + else: + srcloc = self.ensure_reg(op.getarg(0)) + resloc = self.force_allocate_reg(op) + return [resloc, srcloc, imm(index.value), imm(count.value)] def expand_float(self, size, box): adr = self.assembler.datablockwrapper.malloc_aligned(16, 16) @@ -835,14 +843,19 @@ prepare_vec_cast_int_to_float = prepare_vec_cast_float_to_int - def _prepare_guard(self, box): + def load_vector_condition_into_cc(self, box): if self.assembler.guard_success_cc == c.cond_none: - notimplemented("[ppc/regalloc] guard") + # compare happended before + #loc = self.ensure_reg(box) + #mc = self.assembler.mc + #mc.cmp_op(0, loc.value, 0, imm=True) self.assembler.guard_success_cc = c.NE def prepare_vec_guard_true(self, op): - self._prepare_guard(op.getarg(0)) + self.load_vector_condition_into_cc(op.getarg(0)) + return self._prepare_guard(op) def prepare_vec_guard_false(self, op): - self._prepare_guard(op.getarg(0)) + self.load_vector_condition_into_cc(op.getarg(0)) + return self._prepare_guard(op) diff --git a/rpython/jit/metainterp/test/test_vector.py b/rpython/jit/metainterp/test/test_vector.py --- a/rpython/jit/metainterp/test/test_vector.py +++ b/rpython/jit/metainterp/test/test_vector.py @@ -332,29 +332,29 @@ @py.test.mark.parametrize('type,func,init,insert,at,count,breaks', # all - [(rffi.DOUBLE, lambda x: not bool(x), 1.0, None, -1,32, False), - (rffi.DOUBLE, lambda x: x == 0.0, 1.0, None, -1,33, False), - (rffi.DOUBLE, lambda x: x == 0.0, 1.0, 0.0, 33,34, True), - (rffi.DOUBLE, lambda x: x == 0.0, 1.0, 0.1, 4,34, False), - (lltype.Signed, lambda x: not bool(x), 1, None, -1,32, False), + [#(rffi.DOUBLE, lambda x: not bool(x), 1.0, None, -1,32, False), + #(rffi.DOUBLE, lambda x: x == 0.0, 1.0, None, -1,33, False), + #(rffi.DOUBLE, lambda x: x == 0.0, 1.0, 0.0, 33,34, True), + #(rffi.DOUBLE, lambda x: x == 0.0, 1.0, 0.1, 4,34, False), + #(lltype.Signed, lambda x: not bool(x), 1, None, -1,32, False), (lltype.Signed, lambda x: not bool(x), 1, 0, 14,32, True), - (lltype.Signed, lambda x: not bool(x), 1, 0, 15,31, True), - (lltype.Signed, lambda x: not bool(x), 1, 0, 4,30, True), - (lltype.Signed, lambda x: x == 0, 1, None, -1,33, False), - (lltype.Signed, lambda x: x == 0, 1, 0, 33,34, True), - # any - (rffi.DOUBLE, lambda x: x != 0.0, 0.0, 1.0, 33,35, True), - (rffi.DOUBLE, lambda x: x != 0.0, 0.0, 1.0, -1,36, False), - (rffi.DOUBLE, lambda x: bool(x), 0.0, 1.0, 33,37, True), - (rffi.DOUBLE, lambda x: bool(x), 0.0, 1.0, -1,38, False), - (lltype.Signed, lambda x: x != 0, 0, 1, 33,35, True), - (lltype.Signed, lambda x: x != 0, 0, 1, -1,36, False), - (lltype.Signed, lambda x: bool(x), 0, 1, 33,37, True), - (lltype.Signed, lambda x: bool(x), 0, 1, -1,38, False), - (rffi.INT, lambda x: intmask(x) != 0, rffi.r_int(0), rffi.r_int(1), 33,35, True), - (rffi.INT, lambda x: intmask(x) != 0, rffi.r_int(0), rffi.r_int(1), -1,36, False), - (rffi.INT, lambda x: bool(intmask(x)), rffi.r_int(0), rffi.r_int(1), 33,37, True), - (rffi.INT, lambda x: bool(intmask(x)), rffi.r_int(0), rffi.r_int(1), -1,38, False), + #(lltype.Signed, lambda x: not bool(x), 1, 0, 15,31, True), + #(lltype.Signed, lambda x: not bool(x), 1, 0, 4,30, True), + #(lltype.Signed, lambda x: x == 0, 1, None, -1,33, False), + #(lltype.Signed, lambda x: x == 0, 1, 0, 33,34, True), + ## any + #(rffi.DOUBLE, lambda x: x != 0.0, 0.0, 1.0, 33,35, True), + #(rffi.DOUBLE, lambda x: x != 0.0, 0.0, 1.0, -1,36, False), + #(rffi.DOUBLE, lambda x: bool(x), 0.0, 1.0, 33,37, True), + #(rffi.DOUBLE, lambda x: bool(x), 0.0, 1.0, -1,38, False), + #(lltype.Signed, lambda x: x != 0, 0, 1, 33,35, True), + #(lltype.Signed, lambda x: x != 0, 0, 1, -1,36, False), + #(lltype.Signed, lambda x: bool(x), 0, 1, 33,37, True), + #(lltype.Signed, lambda x: bool(x), 0, 1, -1,38, False), + #(rffi.INT, lambda x: intmask(x) != 0, rffi.r_int(0), rffi.r_int(1), 33,35, True), + #(rffi.INT, lambda x: intmask(x) != 0, rffi.r_int(0), rffi.r_int(1), -1,36, False), + #(rffi.INT, lambda x: bool(intmask(x)), rffi.r_int(0), rffi.r_int(1), 33,37, True), + #(rffi.INT, lambda x: bool(intmask(x)), rffi.r_int(0), rffi.r_int(1), -1,38, False), ]) def test_bool_reduction(self, type, func, init, insert, at, count, breaks): myjitdriver = JitDriver(greens = [], reds = 'auto', vectorize=True) From pypy.commits at gmail.com Fri Jul 1 10:38:08 2016 From: pypy.commits at gmail.com (arigo) Date: Fri, 01 Jul 2016 07:38:08 -0700 (PDT) Subject: [pypy-commit] pypy reverse-debugger: Clean up, fixes reverse-watchpoints stopping one step too far in the past Message-ID: <57768050.0eb81c0a.453d1.fffff1b3@mx.google.com> Author: Armin Rigo Branch: reverse-debugger Changeset: r85492:bbb6f37c65f4 Date: 2016-07-01 16:39 +0200 http://bitbucket.org/pypy/pypy/changeset/bbb6f37c65f4/ Log: Clean up, fixes reverse-watchpoints stopping one step too far in the past diff --git a/rpython/translator/revdb/interact.py b/rpython/translator/revdb/interact.py --- a/rpython/translator/revdb/interact.py +++ b/rpython/translator/revdb/interact.py @@ -189,12 +189,8 @@ 'Reverse-hit' if backward else 'Hit', kind, num, name)) self.print_extra_pending_info = '\n'.join(printing) - target_time = b.time - if backward: - target_time -= 1 # when going backwards, we stop just before - # the breakpoint time, as opposed to just after - if self.pgroup.get_current_time() != target_time: - self.pgroup.jump_in_time(target_time) + if self.pgroup.get_current_time() != b.time: + self.pgroup.jump_in_time(b.time) def remove_tainting(self): if self.pgroup.is_tainted(): diff --git a/rpython/translator/revdb/src-revdb/revdb.c b/rpython/translator/revdb/src-revdb/revdb.c --- a/rpython/translator/revdb/src-revdb/revdb.c +++ b/rpython/translator/revdb/src-revdb/revdb.c @@ -1106,8 +1106,8 @@ return; /* ignored breakpoints */ case 'r': /* record the breakpoint but continue */ - if (last_recorded_breakpoint_loc != rpy_revdb.stop_point_seen + 1) { - last_recorded_breakpoint_loc = rpy_revdb.stop_point_seen + 1; + if (last_recorded_breakpoint_loc != rpy_revdb.stop_point_seen) { + last_recorded_breakpoint_loc = rpy_revdb.stop_point_seen; n_last_recorded_breakpoints = 0; } if (n_last_recorded_breakpoints < RECORD_BKPT_NUM) { From pypy.commits at gmail.com Fri Jul 1 10:39:42 2016 From: pypy.commits at gmail.com (arigo) Date: Fri, 01 Jul 2016 07:39:42 -0700 (PDT) Subject: [pypy-commit] pypy.org extradoc: update the values Message-ID: <577680ae.89dec20a.9be1a.3b55@mx.google.com> Author: Armin Rigo Branch: extradoc Changeset: r764:9b074671829e Date: 2016-07-01 16:41 +0200 http://bitbucket.org/pypy/pypy.org/changeset/9b074671829e/ Log: update the values diff --git a/don1.html b/don1.html --- a/don1.html +++ b/don1.html @@ -9,13 +9,13 @@ - $64624 of $105000 (61.5%) + $64643 of $105000 (61.6%)
@@ -23,7 +23,7 @@
  • From pypy.commits at gmail.com Fri Jul 1 11:10:28 2016 From: pypy.commits at gmail.com (arigo) Date: Fri, 01 Jul 2016 08:10:28 -0700 (PDT) Subject: [pypy-commit] pypy reverse-debugger: test fix Message-ID: <577687e4.24f9c20a.d1a96.4446@mx.google.com> Author: Armin Rigo Branch: reverse-debugger Changeset: r85493:ae8134681b8c Date: 2016-07-01 17:11 +0200 http://bitbucket.org/pypy/pypy/changeset/ae8134681b8c/ Log: test fix diff --git a/rpython/translator/revdb/test/test_process.py b/rpython/translator/revdb/test/test_process.py --- a/rpython/translator/revdb/test/test_process.py +++ b/rpython/translator/revdb/test/test_process.py @@ -68,7 +68,7 @@ group.active.expect(ANSWER_READY, 1, Ellipsis) e = py.test.raises(Breakpoint, group.go_forward, 10, 'b') assert e.value.time == 7 - assert e.value.num == 99 + assert e.value.nums == [99] group._check_current_time(7) def test_breakpoint_r(self): @@ -77,8 +77,8 @@ group.active.expect(42, 100, -43, -44, 'set-breakpoint') group.active.expect(ANSWER_READY, 1, Ellipsis) e = py.test.raises(Breakpoint, group.go_forward, 10, 'r') - assert e.value.time == 8 - assert e.value.num == 99 + assert e.value.time == 7 + assert e.value.nums == [99] group._check_current_time(10) def test_breakpoint_i(self): From pypy.commits at gmail.com Fri Jul 1 11:22:14 2016 From: pypy.commits at gmail.com (mattip) Date: Fri, 01 Jul 2016 08:22:14 -0700 (PDT) Subject: [pypy-commit] pypy call-via-pyobj: fix for translation Message-ID: <57768aa6.c813c20a.df394.6c07@mx.google.com> Author: Matti Picus Branch: call-via-pyobj Changeset: r85494:1b1bf8ef2fb6 Date: 2016-07-01 18:21 +0300 http://bitbucket.org/pypy/pypy/changeset/1b1bf8ef2fb6/ Log: fix for translation diff --git a/pypy/module/cpyext/typeobject.py b/pypy/module/cpyext/typeobject.py --- a/pypy/module/cpyext/typeobject.py +++ b/pypy/module/cpyext/typeobject.py @@ -304,6 +304,7 @@ struct = getattr(pto, slot_names[0]) if not struct: continue + assert isinstance(struct, lltype._ptr) offset.append(rffi.offsetof(struct._T, slot_names[1])) func = getattr(struct, slot_names[1]) func_voidp = rffi.cast(rffi.VOIDP, func) @@ -818,7 +819,6 @@ base_pyo = rffi.cast(PyObject, pto.c_tp_base) if base and not base.c_tp_flags & Py_TPFLAGS_READY: name = rffi.charp2str(base.c_tp_name) - print 'realizing base while creating child', type_realize(space, base_pyo) if base and not pto.c_ob_type: # will be filled later pto.c_ob_type = base.c_ob_type From pypy.commits at gmail.com Fri Jul 1 12:53:49 2016 From: pypy.commits at gmail.com (arigo) Date: Fri, 01 Jul 2016 09:53:49 -0700 (PDT) Subject: [pypy-commit] pypy reverse-debugger: Signals work with a little tweak Message-ID: <5776a01d.e40bc30a.d83a9.4c9c@mx.google.com> Author: Armin Rigo Branch: reverse-debugger Changeset: r85495:6d53b2a13d23 Date: 2016-07-01 18:47 +0200 http://bitbucket.org/pypy/pypy/changeset/6d53b2a13d23/ Log: Signals work with a little tweak diff --git a/pypy/interpreter/reverse_debugging.py b/pypy/interpreter/reverse_debugging.py --- a/pypy/interpreter/reverse_debugging.py +++ b/pypy/interpreter/reverse_debugging.py @@ -7,6 +7,7 @@ from pypy.interpreter.baseobjspace import W_Root from pypy.interpreter import gateway, typedef, pycode, pytraceback, pyframe from pypy.module.marshal import interp_marshal +from pypy.interpreter.executioncontext import AbstractActionFlag class DBState: @@ -21,6 +22,16 @@ dbstate = DBState() +pycode.PyCode.co_revdb_linestarts = None # or a string: see below + +# invariant: "f_revdb_nextline_instr" is the bytecode offset of +# the start of the line that follows "last_instr". +pyframe.PyFrame.f_revdb_nextline_instr = 0 + + +# ____________________________________________________________ + + def setup_revdb(space): """Called at run-time, before the space is set up. @@ -47,11 +58,7 @@ revdb.register_debug_command(revdb.CMD_WATCHVALUES, lambda_watchvalues) -pycode.PyCode.co_revdb_linestarts = None # or a string: see below - -# invariant: "f_revdb_nextline_instr" is the bytecode offset of -# the start of the line that follows "last_instr". -pyframe.PyFrame.f_revdb_nextline_instr = 0 +# ____________________________________________________________ def enter_call(caller_frame, callee_frame): @@ -221,6 +228,9 @@ dbstate.metavars[index] = w_obj +# ____________________________________________________________ + + def fetch_cur_frame(): ec = dbstate.space.getexecutioncontext() frame = ec.topframeref() @@ -526,3 +536,46 @@ w_dict = space.builtin.w_dict w_res = prog.exec_code(space, w_dict, w_dict) return space.str_w(space.repr(w_res)) + + +# ____________________________________________________________ + + +class RDBSignalActionFlag(AbstractActionFlag): + # Used instead of pypy.module.signal.interp_signal.SignalActionFlag + # when we have reverse-debugging. That other class would work too, + # but inefficiently: it would generate two words of data per bytecode. + # This class is tweaked to generate one byte per _SIG_TICKER_COUNT + # bytecodes, at the expense of not reacting to signals instantly. + + _SIG_TICKER_COUNT = 100 + _ticker_cache = 0 + _ticker_count = _SIG_TICKER_COUNT * 10 + + def get_ticker(self): + return self._ticker_cache + + def reset_ticker(self, value): + self._ticker_cache = value + + def rearm_ticker(self): + self._ticker_cache = -1 + + def decrement_ticker(self, by): + if we_are_translated(): + c = self._ticker_count - 1 + if c < 0: + c = self._update_ticker_from_signals() + self._ticker_count = c + if self.has_bytecode_counter: # this 'if' is constant-folded + print ("RDBSignalActionFlag: has_bytecode_counter: " + "not supported for now") + raise NotImplementedError + return self._ticker_cache + + def _update_ticker_from_signals(self): + from rpython.rlib import rsignal + if rsignal.pypysig_check_and_reset(): + self.rearm_ticker() + return self._SIG_TICKER_COUNT + _update_ticker_from_signals._dont_inline_ = True diff --git a/pypy/module/signal/__init__.py b/pypy/module/signal/__init__.py --- a/pypy/module/signal/__init__.py +++ b/pypy/module/signal/__init__.py @@ -46,7 +46,11 @@ space.check_signal_action = interp_signal.CheckSignalAction(space) space.actionflag.register_periodic_action(space.check_signal_action, use_bytecode_counter=False) - space.actionflag.__class__ = interp_signal.SignalActionFlag + if space.config.translation.reverse_debugger: + from pypy.interpreter.reverse_debugging import RDBSignalActionFlag + space.actionflag.__class__ = RDBSignalActionFlag + else: + space.actionflag.__class__ = interp_signal.SignalActionFlag # xxx yes I know the previous line is a hack def startup(self, space): diff --git a/rpython/rlib/rsignal.py b/rpython/rlib/rsignal.py --- a/rpython/rlib/rsignal.py +++ b/rpython/rlib/rsignal.py @@ -87,6 +87,8 @@ pypysig_getaddr_occurred = external('pypysig_getaddr_occurred', [], lltype.Ptr(LONG_STRUCT), _nowrapper=True, elidable_function=True) +pypysig_check_and_reset = external('pypysig_check_and_reset', [], + lltype.Bool, _nowrapper=True) c_alarm = external('alarm', [rffi.INT], rffi.INT) c_pause = external('pause', [], rffi.INT, releasegil=True) c_siginterrupt = external('siginterrupt', [rffi.INT, rffi.INT], rffi.INT, diff --git a/rpython/translator/c/src/signals.h b/rpython/translator/c/src/signals.h --- a/rpython/translator/c/src/signals.h +++ b/rpython/translator/c/src/signals.h @@ -37,4 +37,11 @@ void *pypysig_getaddr_occurred(void); #define pypysig_getaddr_occurred() ((void *)(&pypysig_counter)) +inline static char pypysig_check_and_reset(void) { + /* used by reverse_debugging */ + char result = pypysig_counter.value < 0; + pypysig_counter.value = 0; + return result; +} + #endif From pypy.commits at gmail.com Fri Jul 1 12:53:51 2016 From: pypy.commits at gmail.com (arigo) Date: Fri, 01 Jul 2016 09:53:51 -0700 (PDT) Subject: [pypy-commit] pypy reverse-debugger: Minor clean-up, and don't trace hidden_applevel bytecodes at all Message-ID: <5776a01f.67c0c20a.c1c4d.6634@mx.google.com> Author: Armin Rigo Branch: reverse-debugger Changeset: r85496:0210f3ef956a Date: 2016-07-01 18:55 +0200 http://bitbucket.org/pypy/pypy/changeset/0210f3ef956a/ Log: Minor clean-up, and don't trace hidden_applevel bytecodes at all (previously it would still trigger at bytecode index 0). diff --git a/pypy/interpreter/pyopcode.py b/pypy/interpreter/pyopcode.py --- a/pypy/interpreter/pyopcode.py +++ b/pypy/interpreter/pyopcode.py @@ -53,14 +53,10 @@ ### opcode dispatch ### def dispatch(self, pycode, next_instr, ec): - if self.space.config.translation.reverse_debugger: - from pypy.interpreter.reverse_debugging import prepare_code - prepare_code(pycode) - # # For the sequel, force 'next_instr' to be unsigned for performance next_instr = r_uint(next_instr) co_code = pycode.co_code - # + try: while True: next_instr = self.handle_bytecode(co_code, next_instr, ec) diff --git a/pypy/interpreter/reverse_debugging.py b/pypy/interpreter/reverse_debugging.py --- a/pypy/interpreter/reverse_debugging.py +++ b/pypy/interpreter/reverse_debugging.py @@ -26,7 +26,7 @@ # invariant: "f_revdb_nextline_instr" is the bytecode offset of # the start of the line that follows "last_instr". -pyframe.PyFrame.f_revdb_nextline_instr = 0 +pyframe.PyFrame.f_revdb_nextline_instr = -1 # ____________________________________________________________ @@ -69,6 +69,10 @@ if dbstate.breakpoint_stack_id != 0 and caller_frame is not None: if dbstate.breakpoint_stack_id == revdb.get_unique_id(caller_frame): revdb.breakpoint(-1) + # + code = callee_frame.pycode + if code.co_revdb_linestarts is None: + build_co_revdb_linestarts(code) def leave_call(caller_frame, callee_frame): if dbstate.breakpoint_stack_id != 0 and caller_frame is not None: @@ -172,10 +176,6 @@ code.co_revdb_linestarts = lstart return lstart -def prepare_code(code): - if code.co_revdb_linestarts is None: - build_co_revdb_linestarts(code) - def get_final_lineno(code): lineno = code.co_firstlineno lnotab = code.co_lnotab From pypy.commits at gmail.com Fri Jul 1 12:58:58 2016 From: pypy.commits at gmail.com (amauryfa) Date: Fri, 01 Jul 2016 09:58:58 -0700 (PDT) Subject: [pypy-commit] pypy default: Like CPython, use type(self) to create new instances in datetime.replace(). Message-ID: <5776a152.4dd11c0a.b295d.4010@mx.google.com> Author: Amaury Forgeot d'Arc Branch: Changeset: r85497:7a86c69fafb6 Date: 2016-07-01 18:56 +0200 http://bitbucket.org/pypy/pypy/changeset/7a86c69fafb6/ Log: Like CPython, use type(self) to create new instances in datetime.replace(). diff --git a/lib_pypy/datetime.py b/lib_pypy/datetime.py --- a/lib_pypy/datetime.py +++ b/lib_pypy/datetime.py @@ -839,7 +839,7 @@ month = self._month if day is None: day = self._day - return date(year, month, day) + return type(self)(year, month, day) # Comparisons of date objects with other. @@ -1356,7 +1356,7 @@ microsecond = self.microsecond if tzinfo is True: tzinfo = self.tzinfo - return time(hour, minute, second, microsecond, tzinfo) + return type(self)(hour, minute, second, microsecond, tzinfo) def __nonzero__(self): if self.second or self.microsecond: @@ -1566,7 +1566,7 @@ microsecond = self.microsecond if tzinfo is True: tzinfo = self.tzinfo - return datetime(year, month, day, hour, minute, second, microsecond, + return type(self)(year, month, day, hour, minute, second, microsecond, tzinfo) def astimezone(self, tz): diff --git a/pypy/module/test_lib_pypy/test_datetime.py b/pypy/module/test_lib_pypy/test_datetime.py --- a/pypy/module/test_lib_pypy/test_datetime.py +++ b/pypy/module/test_lib_pypy/test_datetime.py @@ -315,6 +315,14 @@ class sub(datetime.timedelta): pass assert type(+sub()) is datetime.timedelta + def test_subclass(self): + class MyDate(datetime.date): pass + class MyTime(datetime.time): pass + class MyDateTime(datetime.datetime): pass + assert type(MyDate.today().replace(day=1)) is MyDate + assert type(MyTime().replace(hour=1)) is MyTime + assert type(MyDateTime.now().replace(day=1, hour=1)) is MyDateTime + class TestDatetimeHost(BaseTestDatetime): def setup_class(cls): From pypy.commits at gmail.com Fri Jul 1 13:35:51 2016 From: pypy.commits at gmail.com (amauryfa) Date: Fri, 01 Jul 2016 10:35:51 -0700 (PDT) Subject: [pypy-commit] pypy default: Improve the test, and fix: Message-ID: <5776a9f7.46461c0a.24882.ffff80a0@mx.google.com> Author: Amaury Forgeot d'Arc Branch: Changeset: r85498:f616e095f87e Date: 2016-07-01 19:34 +0200 http://bitbucket.org/pypy/pypy/changeset/f616e095f87e/ Log: Improve the test, and fix: The subclass __init__ and __new__ should not be called. Thanks Armin! diff --git a/lib_pypy/datetime.py b/lib_pypy/datetime.py --- a/lib_pypy/datetime.py +++ b/lib_pypy/datetime.py @@ -839,7 +839,7 @@ month = self._month if day is None: day = self._day - return type(self)(year, month, day) + return date.__new__(type(self), year, month, day) # Comparisons of date objects with other. @@ -1356,7 +1356,8 @@ microsecond = self.microsecond if tzinfo is True: tzinfo = self.tzinfo - return type(self)(hour, minute, second, microsecond, tzinfo) + return time.__new__(type(self), + hour, minute, second, microsecond, tzinfo) def __nonzero__(self): if self.second or self.microsecond: @@ -1566,8 +1567,9 @@ microsecond = self.microsecond if tzinfo is True: tzinfo = self.tzinfo - return type(self)(year, month, day, hour, minute, second, microsecond, - tzinfo) + return datetime.__new__(type(self), + year, month, day, hour, minute, second, + microsecond, tzinfo) def astimezone(self, tz): if not isinstance(tz, tzinfo): diff --git a/pypy/module/test_lib_pypy/test_datetime.py b/pypy/module/test_lib_pypy/test_datetime.py --- a/pypy/module/test_lib_pypy/test_datetime.py +++ b/pypy/module/test_lib_pypy/test_datetime.py @@ -315,13 +315,50 @@ class sub(datetime.timedelta): pass assert type(+sub()) is datetime.timedelta - def test_subclass(self): - class MyDate(datetime.date): pass - class MyTime(datetime.time): pass - class MyDateTime(datetime.datetime): pass - assert type(MyDate.today().replace(day=1)) is MyDate - assert type(MyTime().replace(hour=1)) is MyTime - assert type(MyDateTime.now().replace(day=1, hour=1)) is MyDateTime + def test_subclass_date(self): + # replace() should return a subclass but not call __new__ or __init__. + class MyDate(datetime.date): + forbidden = False + def __new__(cls): + if cls.forbidden: FAIL + return datetime.date.__new__(cls, 2016, 2, 3) + def __init__(self, *args): + if self.forbidden: FAIL + d = MyDate() + d.forbidden = True + d2 = d.replace(day=5) + assert type(d2) is MyDate + assert d2 == datetime.date(2016, 2, 5) + + def test_subclass_time(self): + # replace() should return a subclass but not call __new__ or __init__. + class MyTime(datetime.time): + forbidden = False + def __new__(cls): + if cls.forbidden: FAIL + return datetime.time.__new__(cls, 1, 2, 3) + def __init__(self, *args): + if self.forbidden: FAIL + d = MyTime() + d.forbidden = True + d2 = d.replace(hour=5) + assert type(d2) is MyTime + assert d2 == datetime.time(5, 2, 3) + + def test_subclass_datetime(self): + # replace() should return a subclass but not call __new__ or __init__. + class MyDatetime(datetime.datetime): + forbidden = False + def __new__(cls): + if cls.forbidden: FAIL + return datetime.datetime.__new__(cls, 2016, 4, 5, 1, 2, 3) + def __init__(self, *args): + if self.forbidden: FAIL + d = MyDatetime() + d.forbidden = True + d2 = d.replace(hour=7) + assert type(d2) is MyDatetime + assert d2 == datetime.datetime(2016, 4, 5, 7, 2, 3) class TestDatetimeHost(BaseTestDatetime): From pypy.commits at gmail.com Fri Jul 1 13:36:47 2016 From: pypy.commits at gmail.com (arigo) Date: Fri, 01 Jul 2016 10:36:47 -0700 (PDT) Subject: [pypy-commit] pypy default: Support backslashes inside strings passed to dot Message-ID: <5776aa2f.c813c20a.df394.ffff9e85@mx.google.com> Author: Armin Rigo Branch: Changeset: r85499:850a4e0a56cc Date: 2016-07-01 19:37 +0200 http://bitbucket.org/pypy/pypy/changeset/850a4e0a56cc/ Log: Support backslashes inside strings passed to dot diff --git a/rpython/translator/tool/make_dot.py b/rpython/translator/tool/make_dot.py --- a/rpython/translator/tool/make_dot.py +++ b/rpython/translator/tool/make_dot.py @@ -51,7 +51,7 @@ ports=None, ): d = locals() - attrs = [('%s="%s"' % (x, d[x].replace('"', '\\"').replace('\n', '\\n'))) + attrs = [('%s="%s"' % (x, _quote(d[x]))) for x in ['label', 'style', 'color', 'dir', 'weight']] self.emit('edge [%s];' % ", ".join(attrs)) if ports: @@ -69,7 +69,7 @@ width="0.75", ): d = locals() - attrs = [('%s="%s"' % (x, d[x].replace('"', '\\"').replace('\n', '\\n'))) + attrs = [('%s="%s"' % (x, _quote(d[x]))) for x in ['shape', 'label', 'color', 'fillcolor', 'style', 'width']] self.emit('%s [%s];' % (safename(name), ", ".join(attrs))) @@ -193,7 +193,7 @@ name2 = self.blockname(link.target) label = " ".join(map(repr, link.args)) if link.exitcase is not None: - label = "%s: %s" %(repr(link.exitcase).replace('\\', '\\\\'), label) + label = "%s: %s" %(_quote(repr(link.exitcase)), label) self.emit_edge(name, name2, label, style="dotted", color="red") else: self.emit_edge(name, name2, label, style="solid") @@ -237,3 +237,6 @@ # not a keyword name = ''.join([CHAR_MAP[c] for c in name]) return '_' + name + +def _quote(s): + return s.replace('\\', '\\\\').replace('"', '\\"').replace('\n', '\\n') From pypy.commits at gmail.com Fri Jul 1 13:36:50 2016 From: pypy.commits at gmail.com (arigo) Date: Fri, 01 Jul 2016 10:36:50 -0700 (PDT) Subject: [pypy-commit] pypy reverse-debugger: hg merge default Message-ID: <5776aa32.54df1c0a.d172f.6f70@mx.google.com> Author: Armin Rigo Branch: reverse-debugger Changeset: r85500:600dfa2a7206 Date: 2016-07-01 19:38 +0200 http://bitbucket.org/pypy/pypy/changeset/600dfa2a7206/ Log: hg merge default diff too long, truncating to 2000 out of 4923 lines diff --git a/lib_pypy/datetime.py b/lib_pypy/datetime.py --- a/lib_pypy/datetime.py +++ b/lib_pypy/datetime.py @@ -839,7 +839,7 @@ month = self._month if day is None: day = self._day - return date(year, month, day) + return date.__new__(type(self), year, month, day) # Comparisons of date objects with other. @@ -1356,7 +1356,8 @@ microsecond = self.microsecond if tzinfo is True: tzinfo = self.tzinfo - return time(hour, minute, second, microsecond, tzinfo) + return time.__new__(type(self), + hour, minute, second, microsecond, tzinfo) def __nonzero__(self): if self.second or self.microsecond: @@ -1566,8 +1567,9 @@ microsecond = self.microsecond if tzinfo is True: tzinfo = self.tzinfo - return datetime(year, month, day, hour, minute, second, microsecond, - tzinfo) + return datetime.__new__(type(self), + year, month, day, hour, minute, second, + microsecond, tzinfo) def astimezone(self, tz): if not isinstance(tz, tzinfo): diff --git a/pypy/doc/config/commandline.txt b/pypy/doc/config/commandline.txt --- a/pypy/doc/config/commandline.txt +++ b/pypy/doc/config/commandline.txt @@ -9,7 +9,7 @@ PyPy Python interpreter options ------------------------------- -The following options can be used after ``translate.py +The following options can be used after ``rpython targetpypystandalone`` or as options to ``py.py``. .. GENERATE: objspace @@ -22,7 +22,7 @@ General translation options --------------------------- -The following are options of ``translate.py``. They must be +The following are options of ``bin/rpython``. They must be given before the ``targetxxx`` on the command line. * `--opt -O:`__ set the optimization level `[0, 1, size, mem, 2, 3]` diff --git a/pypy/doc/config/index.rst b/pypy/doc/config/index.rst --- a/pypy/doc/config/index.rst +++ b/pypy/doc/config/index.rst @@ -15,12 +15,12 @@ ./py.py <`objspace options`_> -and the ``translate.py`` translation entry +and the ``rpython/bin/rpython`` translation entry point which takes arguments of this form: .. parsed-literal:: - ./translate.py <`translation options`_> + ./rpython/bin/rpython <`translation options`_> For the common case of ```` being ``targetpypystandalone.py``, you can then pass the `object space options`_ after @@ -28,7 +28,7 @@ .. parsed-literal:: - ./translate.py <`translation options`_> targetpypystandalone.py <`objspace options`_> + ./rpython/bin/rpython <`translation options`_> targetpypystandalone.py <`objspace options`_> There is an `overview`_ of all command line arguments that can be passed in either position. diff --git a/pypy/doc/config/opt.rst b/pypy/doc/config/opt.rst --- a/pypy/doc/config/opt.rst +++ b/pypy/doc/config/opt.rst @@ -4,8 +4,8 @@ This meta-option selects a default set of optimization settings to use during a translation. Usage:: - translate.py --opt=# - translate.py -O# + bin/rpython --opt=# + bin/rpython -O# where ``#`` is the desired optimization level. The valid choices are: diff --git a/pypy/doc/config/translation.dont_write_c_files.txt b/pypy/doc/config/translation.dont_write_c_files.txt --- a/pypy/doc/config/translation.dont_write_c_files.txt +++ b/pypy/doc/config/translation.dont_write_c_files.txt @@ -1,4 +1,4 @@ write the generated C files to ``/dev/null`` instead of to the disk. Useful if -you want to use translate.py as a benchmark and don't want to access the disk. +you want to use translation as a benchmark and don't want to access the disk. .. _`translation documentation`: ../translation.html diff --git a/pypy/doc/config/translation.fork_before.txt b/pypy/doc/config/translation.fork_before.txt --- a/pypy/doc/config/translation.fork_before.txt +++ b/pypy/doc/config/translation.fork_before.txt @@ -1,4 +1,4 @@ This is an option mostly useful when working on the PyPy toolchain. If you use -it, translate.py will fork before the specified phase. If the translation +it, translation will fork before the specified phase. If the translation crashes after that fork, you can fix the bug in the toolchain, and continue translation at the fork-point. diff --git a/pypy/doc/cppyy.rst b/pypy/doc/cppyy.rst --- a/pypy/doc/cppyy.rst +++ b/pypy/doc/cppyy.rst @@ -122,7 +122,7 @@ $ hg up reflex-support # optional # This example shows python, but using pypy-c is faster and uses less memory - $ python rpython/translator/goal/translate.py --opt=jit pypy/goal/targetpypystandalone --withmod-cppyy + $ python rpython/bin/rpython --opt=jit pypy/goal/targetpypystandalone --withmod-cppyy This will build a ``pypy-c`` that includes the cppyy module, and through that, Reflex support. diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -48,3 +48,8 @@ Bug fix: if ``socket.socket()`` failed, the ``socket.error`` did not show the errno of the failing system call, but instead some random previous errno. + +.. branch: PyTuple_Type-subclass + +Refactor PyTupleObject to look like cpython's and allow subclassing +PyTuple_Type diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py --- a/pypy/interpreter/baseobjspace.py +++ b/pypy/interpreter/baseobjspace.py @@ -1038,7 +1038,7 @@ return (None, None) def newlist_bytes(self, list_s): - return self.newlist([self.wrap(s) for s in list_s]) + return self.newlist([self.newbytes(s) for s in list_s]) def newlist_unicode(self, list_u): return self.newlist([self.wrap(u) for u in list_u]) @@ -1537,7 +1537,7 @@ # unclear if there is any use at all for getting the bytes in # the unicode buffer.) try: - return self.str_w(w_obj) + return self.bytes_w(w_obj) except OperationError as e: if not e.match(self, self.w_TypeError): raise diff --git a/pypy/interpreter/pycode.py b/pypy/interpreter/pycode.py --- a/pypy/interpreter/pycode.py +++ b/pypy/interpreter/pycode.py @@ -408,14 +408,14 @@ w(self.co_nlocals), w(self.co_stacksize), w(self.co_flags), - w(self.co_code), + space.newbytes(self.co_code), space.newtuple(self.co_consts_w), space.newtuple(self.co_names_w), space.newtuple([w(v) for v in self.co_varnames]), w(self.co_filename), w(self.co_name), w(self.co_firstlineno), - w(self.co_lnotab), + space.newbytes(self.co_lnotab), space.newtuple([w(v) for v in self.co_freevars]), space.newtuple([w(v) for v in self.co_cellvars]), w(self.magic), diff --git a/pypy/interpreter/pyparser/automata.py b/pypy/interpreter/pyparser/automata.py --- a/pypy/interpreter/pyparser/automata.py +++ b/pypy/interpreter/pyparser/automata.py @@ -13,12 +13,11 @@ # PYPY Modification: removed the EMPTY class as it's not needed here -# PYPY Modification: we don't need a particuliar DEFAULT class here -# a simple None works fine. -# (Having a DefaultClass inheriting from str makes -# the annotator crash) -DEFAULT = "\00default" # XXX hack, the rtyper does not support dict of with str|None keys - # anyway using dicts doesn't seem the best final way to store these char indexed tables +# PYPY Modification: DEFAULT is a singleton, used only in the pre-RPython +# dicts (see pytokenize.py). Then DFA.__init__() turns these dicts into +# more compact strings. +DEFAULT = object() + # PYPY Modification : removed all automata functions (any, maybe, # newArcPair, etc.) diff --git a/pypy/interpreter/pyparser/genpytokenize.py b/pypy/interpreter/pyparser/genpytokenize.py --- a/pypy/interpreter/pyparser/genpytokenize.py +++ b/pypy/interpreter/pyparser/genpytokenize.py @@ -296,7 +296,7 @@ i = 0 for k, v in sorted(state.items()): i += 1 - if k == '\x00default': + if k == DEFAULT: k = "automata.DEFAULT" else: k = repr(k) diff --git a/pypy/interpreter/pyparser/parsestring.py b/pypy/interpreter/pyparser/parsestring.py --- a/pypy/interpreter/pyparser/parsestring.py +++ b/pypy/interpreter/pyparser/parsestring.py @@ -81,7 +81,7 @@ if need_encoding: enc = encoding v = PyString_DecodeEscape(space, substr, 'strict', enc) - return space.wrap(v) + return space.newbytes(v) def decode_unicode_utf8(space, s, ps, q): # ****The Python 2.7 version, producing UTF-32 escapes**** diff --git a/pypy/module/_cffi_backend/ctypeprim.py b/pypy/module/_cffi_backend/ctypeprim.py --- a/pypy/module/_cffi_backend/ctypeprim.py +++ b/pypy/module/_cffi_backend/ctypeprim.py @@ -84,7 +84,7 @@ if self.size == 1: with cdataobj as ptr: s = ptr[0] - return self.space.wrap(s) + return self.space.newbytes(s) return W_CType.string(self, cdataobj, maxlen) def unpack_ptr(self, w_ctypeptr, ptr, length): @@ -126,12 +126,12 @@ return self.space.wrap(ord(cdata[0])) def convert_to_object(self, cdata): - return self.space.wrap(cdata[0]) + return self.space.newbytes(cdata[0]) def _convert_to_char(self, w_ob): space = self.space if space.isinstance_w(w_ob, space.w_str): - s = space.str_w(w_ob) + s = space.bytes_w(w_ob) if len(s) == 1: return s[0] if (isinstance(w_ob, cdataobj.W_CData) and @@ -146,7 +146,7 @@ def unpack_ptr(self, w_ctypeptr, ptr, length): s = rffi.charpsize2str(ptr, length) - return self.space.wrapbytes(s) + return self.space.newbytes(s) # XXX explicitly use an integer type instead of lltype.UniChar here, diff --git a/pypy/module/_cffi_backend/ctypeptr.py b/pypy/module/_cffi_backend/ctypeptr.py --- a/pypy/module/_cffi_backend/ctypeptr.py +++ b/pypy/module/_cffi_backend/ctypeptr.py @@ -120,7 +120,7 @@ s = rffi.charp2str(ptr) else: s = rffi.charp2strn(ptr, length) - return space.wrapbytes(s) + return space.newbytes(s) # # pointer to a wchar_t: builds and returns a unicode if self.is_unichar_ptr_or_array(): @@ -129,7 +129,7 @@ u = rffi.wcharp2unicode(cdata) else: u = rffi.wcharp2unicoden(cdata, length) - return space.wrap(u) + return space.newunicode(u) # return W_CType.string(self, cdataobj, maxlen) diff --git a/pypy/module/_codecs/interp_codecs.py b/pypy/module/_codecs/interp_codecs.py --- a/pypy/module/_codecs/interp_codecs.py +++ b/pypy/module/_codecs/interp_codecs.py @@ -36,12 +36,14 @@ w_errorhandler = lookup_error(space, errors) if decode: w_cls = space.w_UnicodeDecodeError + w_input = space.newbytes(input) else: w_cls = space.w_UnicodeEncodeError + w_input = space.newunicode(input) w_exc = space.call_function( w_cls, space.wrap(encoding), - space.wrap(input), + w_input, space.wrap(startpos), space.wrap(endpos), space.wrap(reason)) @@ -314,7 +316,7 @@ @unwrap_spec(errors='str_or_None') def readbuffer_encode(space, w_data, errors='strict'): s = space.getarg_w('s#', w_data) - return space.newtuple([space.wrap(s), space.wrap(len(s))]) + return space.newtuple([space.newbytes(s), space.wrap(len(s))]) @unwrap_spec(errors='str_or_None') def charbuffer_encode(space, w_data, errors='strict'): @@ -377,7 +379,7 @@ state = space.fromcache(CodecState) func = getattr(runicode, rname) result = func(uni, len(uni), errors, state.encode_error_handler) - return space.newtuple([space.wrap(result), space.wrap(len(uni))]) + return space.newtuple([space.newbytes(result), space.wrap(len(uni))]) wrap_encoder.func_name = rname globals()[name] = wrap_encoder @@ -398,7 +400,7 @@ wrap_decoder.func_name = rname globals()[name] = wrap_decoder -for encoders in [ +for encoder in [ "ascii_encode", "latin_1_encode", "utf_7_encode", @@ -412,9 +414,9 @@ "raw_unicode_escape_encode", "unicode_internal_encode", ]: - make_encoder_wrapper(encoders) + make_encoder_wrapper(encoder) -for decoders in [ +for decoder in [ "ascii_decode", "latin_1_decode", "utf_7_decode", @@ -426,7 +428,7 @@ "utf_32_le_decode", "raw_unicode_escape_decode", ]: - make_decoder_wrapper(decoders) + make_decoder_wrapper(decoder) if hasattr(runicode, 'str_decode_mbcs'): make_encoder_wrapper('mbcs_encode') @@ -560,7 +562,7 @@ if space.isinstance_w(w_ch, space.w_str): # Charmap may return a string - return space.str_w(w_ch) + return space.bytes_w(w_ch) elif space.isinstance_w(w_ch, space.w_int): # Charmap may return a number x = space.int_w(w_ch) @@ -608,7 +610,7 @@ result = runicode.unicode_encode_charmap( uni, len(uni), errors, state.encode_error_handler, mapping) - return space.newtuple([space.wrap(result), space.wrap(len(uni))]) + return space.newtuple([space.newbytes(result), space.wrap(len(uni))]) @unwrap_spec(chars=unicode) @@ -696,4 +698,4 @@ def escape_decode(space, data, errors='strict'): from pypy.interpreter.pyparser.parsestring import PyString_DecodeEscape result = PyString_DecodeEscape(space, data, errors, None) - return space.newtuple([space.wrap(result), space.wrap(len(data))]) + return space.newtuple([space.newbytes(result), space.wrap(len(data))]) diff --git a/pypy/module/_hashlib/interp_hashlib.py b/pypy/module/_hashlib/interp_hashlib.py --- a/pypy/module/_hashlib/interp_hashlib.py +++ b/pypy/module/_hashlib/interp_hashlib.py @@ -111,7 +111,7 @@ def digest(self, space): "Return the digest value as a string of binary data." digest = self._digest(space) - return space.wrap(digest) + return space.newbytes(digest) def hexdigest(self, space): "Return the digest value as a string of hexadecimal digits." diff --git a/pypy/module/_io/interp_bufferedio.py b/pypy/module/_io/interp_bufferedio.py --- a/pypy/module/_io/interp_bufferedio.py +++ b/pypy/module/_io/interp_bufferedio.py @@ -343,7 +343,7 @@ self._writer_reset_buf() def _write(self, space, data): - w_data = space.wrap(data) + w_data = space.newbytes(data) while True: try: w_written = space.call_method(self.w_raw, "write", w_data) @@ -415,7 +415,7 @@ else: raise oefmt(space.w_ValueError, "read length must be positive or -1") - return space.wrap(res) + return space.newbytes(res) @unwrap_spec(size=int) def peek_w(self, space, size=0): @@ -432,7 +432,7 @@ have = self._readahead() if have > 0: data = ''.join(self.buffer[self.pos:self.pos+have]) - return space.wrap(data) + return space.newbytes(data) # Fill the buffer from the raw stream, and copy it to the result self._reader_reset_buf() @@ -442,7 +442,7 @@ size = 0 self.pos = 0 data = ''.join(self.buffer[:size]) - return space.wrap(data) + return space.newbytes(data) @unwrap_spec(size=int) def read1_w(self, space, size): @@ -452,7 +452,7 @@ if size < 0: raise oefmt(space.w_ValueError, "read length must be positive") if size == 0: - return space.wrap("") + return space.newbytes("") with self.lock: # Return up to n bytes. If at least one byte is buffered, we only @@ -480,7 +480,7 @@ endpos = self.pos + size data = ''.join(self.buffer[self.pos:endpos]) self.pos = endpos - return space.wrap(data) + return space.newbytes(data) def _read_all(self, space): "Read all the file, don't update the cache" @@ -505,7 +505,7 @@ if current_size == 0: return w_data break - data = space.str_w(w_data) + data = space.bytes_w(w_data) size = len(data) if size == 0: break @@ -513,7 +513,7 @@ current_size += size if self.abs_pos != -1: self.abs_pos += size - return space.wrap(builder.build()) + return space.newbytes(builder.build()) def _raw_read(self, space, buffer, start, length): length = intmask(length) @@ -644,11 +644,11 @@ else: pos = -1 if pos >= 0: - w_res = space.wrap(''.join(self.buffer[self.pos:pos+1])) + w_res = space.newbytes(''.join(self.buffer[self.pos:pos+1])) self.pos = pos + 1 return w_res if have == limit: - w_res = space.wrap(''.join(self.buffer[self.pos:self.pos+have])) + w_res = space.newbytes(''.join(self.buffer[self.pos:self.pos+have])) self.pos += have return w_res @@ -690,7 +690,7 @@ written += have if limit >= 0: limit -= have - return space.wrap(''.join(chunks)) + return space.newbytes(''.join(chunks)) # ____________________________________________________ # Write methods @@ -1024,7 +1024,6 @@ self._deprecated_max_buffer_size(space) self.state = STATE_ZERO - check_readable_w(space, w_raw) check_writable_w(space, w_raw) check_seekable_w(space, w_raw) diff --git a/pypy/module/_io/interp_bytesio.py b/pypy/module/_io/interp_bytesio.py --- a/pypy/module/_io/interp_bytesio.py +++ b/pypy/module/_io/interp_bytesio.py @@ -32,12 +32,12 @@ def read_w(self, space, w_size=None): self._check_closed(space) size = convert_size(space, w_size) - return space.wrap(self.read(size)) + return space.newbytes(self.read(size)) def readline_w(self, space, w_limit=None): self._check_closed(space) limit = convert_size(space, w_limit) - return space.wrap(self.readline(limit)) + return space.newbytes(self.readline(limit)) def read1_w(self, space, w_size): return self.read_w(space, w_size) @@ -81,7 +81,7 @@ def getvalue_w(self, space): self._check_closed(space) - return space.wrap(self.getvalue()) + return space.newbytes(self.getvalue()) def tell_w(self, space): self._check_closed(space) @@ -128,7 +128,7 @@ def getstate_w(self, space): self._check_closed(space) return space.newtuple([ - space.wrap(self.getvalue()), + space.newbytes(self.getvalue()), space.wrap(self.tell()), self.getdict(space)]) diff --git a/pypy/module/_io/interp_fileio.py b/pypy/module/_io/interp_fileio.py --- a/pypy/module/_io/interp_fileio.py +++ b/pypy/module/_io/interp_fileio.py @@ -361,7 +361,7 @@ raise wrap_oserror(space, e, exception_name='w_IOError') - return space.wrap(s) + return space.newbytes(s) def readinto_w(self, space, w_buffer): self._check_closed(space) @@ -405,7 +405,7 @@ break builder.append(chunk) total += len(chunk) - return space.wrap(builder.build()) + return space.newbytes(builder.build()) if sys.platform == "win32": def _truncate(self, size): diff --git a/pypy/module/_io/interp_iobase.py b/pypy/module/_io/interp_iobase.py --- a/pypy/module/_io/interp_iobase.py +++ b/pypy/module/_io/interp_iobase.py @@ -192,7 +192,7 @@ length = space.len_w(w_readahead) if length > 0: n = 0 - buf = space.str_w(w_readahead) + buf = space.bytes_w(w_readahead) if limit >= 0: while True: if n >= length or n >= limit: @@ -219,7 +219,7 @@ raise oefmt(space.w_IOError, "peek() should have returned a bytes object, not " "'%T'", w_read) - read = space.str_w(w_read) + read = space.bytes_w(w_read) if not read: break @@ -229,7 +229,7 @@ if read[-1] == '\n': break - return space.wrap(builder.build()) + return space.newbytes(builder.build()) def readlines_w(self, space, w_hint=None): hint = convert_size(space, w_hint) @@ -339,11 +339,11 @@ if not space.isinstance_w(w_data, space.w_str): raise oefmt(space.w_TypeError, "read() should return bytes") - data = space.str_w(w_data) + data = space.bytes_w(w_data) if not data: break builder.append(data) - return space.wrap(builder.build()) + return space.newbytes(builder.build()) W_RawIOBase.typedef = TypeDef( '_io._RawIOBase', W_IOBase.typedef, diff --git a/pypy/module/_io/interp_textio.py b/pypy/module/_io/interp_textio.py --- a/pypy/module/_io/interp_textio.py +++ b/pypy/module/_io/interp_textio.py @@ -160,7 +160,7 @@ w_buffer, w_flag = space.unpackiterable(w_state, 2) flag = space.r_longlong_w(w_flag) else: - w_buffer = space.wrap("") + w_buffer = space.newbytes("") flag = 0 flag <<= 1 if self.pendingcr: @@ -556,7 +556,7 @@ # Given this, we know there was a valid snapshot point # len(dec_buffer) bytes ago with decoder state (b'', dec_flags). w_dec_buffer, w_dec_flags = space.unpackiterable(w_state, 2) - dec_buffer = space.str_w(w_dec_buffer) + dec_buffer = space.bytes_w(w_dec_buffer) dec_flags = space.int_w(w_dec_flags) else: dec_buffer = None @@ -582,7 +582,7 @@ if self.telling: # At the snapshot point, len(dec_buffer) bytes before the read, # the next input to be decoded is dec_buffer + input_chunk. - next_input = dec_buffer + space.str_w(w_input) + next_input = dec_buffer + space.bytes_w(w_input) self.snapshot = PositionSnapshot(dec_flags, next_input) return not eof @@ -769,7 +769,7 @@ else: w_bytes = space.call_method(self.w_encoder, "encode", w_text) - b = space.str_w(w_bytes) + b = space.bytes_w(w_bytes) if not self.pending_bytes: self.pending_bytes = [] self.pending_bytes_count = 0 @@ -799,7 +799,8 @@ while True: try: - space.call_method(self.w_buffer, "write", space.wrap(pending_bytes)) + space.call_method(self.w_buffer, "write", + space.newbytes(pending_bytes)) except OperationError as e: if trap_eintr(space, e): continue @@ -828,7 +829,7 @@ space.call_method(self.w_decoder, "reset") else: space.call_method(self.w_decoder, "setstate", - space.newtuple([space.wrap(""), + space.newtuple([space.newbytes(""), space.wrap(cookie.dec_flags)])) def _encoder_setstate(self, space, cookie): @@ -904,7 +905,7 @@ raise oefmt(space.w_TypeError, msg, w_chunk) self.snapshot = PositionSnapshot(cookie.dec_flags, - space.str_w(w_chunk)) + space.bytes_w(w_chunk)) w_decoded = space.call_method(self.w_decoder, "decode", w_chunk, space.wrap(cookie.need_eof)) @@ -975,7 +976,7 @@ i = 0 while i < len(input): w_decoded = space.call_method(self.w_decoder, "decode", - space.wrap(input[i])) + space.newbytes(input[i])) check_decoded(space, w_decoded) chars_decoded += len(space.unicode_w(w_decoded)) diff --git a/pypy/module/_md5/interp_md5.py b/pypy/module/_md5/interp_md5.py --- a/pypy/module/_md5/interp_md5.py +++ b/pypy/module/_md5/interp_md5.py @@ -20,7 +20,7 @@ self.update(string) def digest_w(self): - return self.space.wrap(self.digest()) + return self.space.newbytes(self.digest()) def hexdigest_w(self): return self.space.wrap(self.hexdigest()) diff --git a/pypy/module/_minimal_curses/interp_curses.py b/pypy/module/_minimal_curses/interp_curses.py --- a/pypy/module/_minimal_curses/interp_curses.py +++ b/pypy/module/_minimal_curses/interp_curses.py @@ -83,12 +83,12 @@ return space.w_None except curses_error as e: raise convert_error(space, e) - return space.wrap(result) + return space.newbytes(result) @unwrap_spec(s=str) def tparm(space, s, args_w): args = [space.int_w(a) for a in args_w] try: - return space.wrap(_curses_tparm(s, args)) + return space.newbytes(_curses_tparm(s, args)) except curses_error as e: raise convert_error(space, e) diff --git a/pypy/module/_multibytecodec/interp_incremental.py b/pypy/module/_multibytecodec/interp_incremental.py --- a/pypy/module/_multibytecodec/interp_incremental.py +++ b/pypy/module/_multibytecodec/interp_incremental.py @@ -113,7 +113,7 @@ pos = c_codecs.pypy_cjk_enc_inbuf_consumed(self.encodebuf) assert 0 <= pos <= len(object) self.pending = object[pos:] - return space.wrap(output) + return space.newbytes(output) @unwrap_spec(errors="str_or_None") diff --git a/pypy/module/_multibytecodec/interp_multibytecodec.py b/pypy/module/_multibytecodec/interp_multibytecodec.py --- a/pypy/module/_multibytecodec/interp_multibytecodec.py +++ b/pypy/module/_multibytecodec/interp_multibytecodec.py @@ -40,7 +40,7 @@ raise wrap_unicodeencodeerror(space, e, input, self.name) except RuntimeError: raise wrap_runtimeerror(space) - return space.newtuple([space.wrap(output), + return space.newtuple([space.newbytes(output), space.wrap(len(input))]) @@ -66,7 +66,7 @@ space.w_UnicodeDecodeError, space.newtuple([ space.wrap(name), - space.wrap(input), + space.newbytes(input), space.wrap(e.start), space.wrap(e.end), space.wrap(e.reason)])) diff --git a/pypy/module/_multiprocessing/interp_connection.py b/pypy/module/_multiprocessing/interp_connection.py --- a/pypy/module/_multiprocessing/interp_connection.py +++ b/pypy/module/_multiprocessing/interp_connection.py @@ -122,9 +122,9 @@ space, self.BUFFER_SIZE, maxlength) try: if newbuf: - return space.wrap(rffi.charpsize2str(newbuf, res)) + return space.newbytes(rffi.charpsize2str(newbuf, res)) else: - return space.wrap(rffi.charpsize2str(self.buffer, res)) + return space.newbytes(rffi.charpsize2str(self.buffer, res)) finally: if newbuf: rffi.free_charp(newbuf) @@ -138,7 +138,7 @@ space, length - offset, PY_SSIZE_T_MAX) try: if newbuf: - raise BufferTooShort(space, space.wrap( + raise BufferTooShort(space, space.newbytes( rffi.charpsize2str(newbuf, res))) rwbuffer.setslice(offset, rffi.charpsize2str(self.buffer, res)) finally: @@ -166,9 +166,9 @@ space, self.BUFFER_SIZE, PY_SSIZE_T_MAX) try: if newbuf: - w_received = space.wrap(rffi.charpsize2str(newbuf, res)) + w_received = space.newbytes(rffi.charpsize2str(newbuf, res)) else: - w_received = space.wrap(rffi.charpsize2str(self.buffer, res)) + w_received = space.newbytes(rffi.charpsize2str(self.buffer, res)) finally: if newbuf: rffi.free_charp(newbuf) diff --git a/pypy/module/_rawffi/alt/test/test_type_converter.py b/pypy/module/_rawffi/alt/test/test_type_converter.py --- a/pypy/module/_rawffi/alt/test/test_type_converter.py +++ b/pypy/module/_rawffi/alt/test/test_type_converter.py @@ -12,14 +12,14 @@ handle_signed = handle_all handle_unsigned = handle_all handle_pointer = handle_all - handle_char = handle_all + handle_char = handle_all handle_unichar = handle_all handle_longlong = handle_all handle_char_p = handle_all handle_unichar_p = handle_all handle_float = handle_all handle_singlefloat = handle_all - + def handle_struct(self, w_ffitype, w_structinstance): self.lastval = w_structinstance @@ -119,12 +119,12 @@ def test_strings(self): # first, try automatic conversion from applevel - self.check(app_types.char_p, self.space.wrap('foo'), 'foo') - self.check(app_types.unichar_p, self.space.wrap(u'foo\u1234'), u'foo\u1234') - self.check(app_types.unichar_p, self.space.wrap('foo'), u'foo') + self.check(app_types.char_p, self.space.newbytes('foo'), 'foo') + self.check(app_types.unichar_p, self.space.wrap(u'foo\u1234'), u'foo\u1234') + self.check(app_types.unichar_p, self.space.wrap('foo'), u'foo') # then, try to pass explicit pointers self.check(app_types.char_p, self.space.wrap(42), 42) - self.check(app_types.unichar_p, self.space.wrap(42), 42) + self.check(app_types.unichar_p, self.space.wrap(42), 42) @@ -136,7 +136,7 @@ get_signed = get_all get_unsigned = get_all get_pointer = get_all - get_char = get_all + get_char = get_all get_unichar = get_all get_longlong = get_all get_char_p = get_all @@ -144,7 +144,7 @@ get_float = get_all get_singlefloat = get_all get_unsigned_which_fits_into_a_signed = get_all - + def convert(self, w_ffitype, val): self.val = val return self.do_and_wrap(w_ffitype) diff --git a/pypy/module/_rawffi/array.py b/pypy/module/_rawffi/array.py --- a/pypy/module/_rawffi/array.py +++ b/pypy/module/_rawffi/array.py @@ -181,7 +181,7 @@ start, stop = self.decodeslice(space, w_slice) ll_buffer = self.ll_buffer result = [ll_buffer[i] for i in range(start, stop)] - return space.wrap(''.join(result)) + return space.newbytes(''.join(result)) def setslice(self, space, w_slice, w_value): start, stop = self.decodeslice(space, w_slice) diff --git a/pypy/module/_rawffi/interp_rawffi.py b/pypy/module/_rawffi/interp_rawffi.py --- a/pypy/module/_rawffi/interp_rawffi.py +++ b/pypy/module/_rawffi/interp_rawffi.py @@ -570,7 +570,7 @@ s = rffi.charp2str(charp_addr) else: s = rffi.charp2strn(charp_addr, maxlength) - return space.wrap(s) + return space.newbytes(s) @unwrap_spec(address=r_uint, maxlength=int) def wcharp2unicode(space, address, maxlength=-1): @@ -588,7 +588,7 @@ if maxlength == -1: return charp2string(space, address) s = rffi.charpsize2str(rffi.cast(rffi.CCHARP, address), maxlength) - return space.wrap(s) + return space.newbytes(s) @unwrap_spec(address=r_uint, maxlength=int) def wcharp2rawunicode(space, address, maxlength=-1): diff --git a/pypy/module/_socket/interp_func.py b/pypy/module/_socket/interp_func.py --- a/pypy/module/_socket/interp_func.py +++ b/pypy/module/_socket/interp_func.py @@ -209,7 +209,7 @@ buf = rsocket.inet_aton(ip) except SocketError as e: raise converted_error(space, e) - return space.wrap(buf) + return space.newbytes(buf) @unwrap_spec(packed=str) def inet_ntoa(space, packed): @@ -234,7 +234,7 @@ buf = rsocket.inet_pton(family, ip) except SocketError as e: raise converted_error(space, e) - return space.wrap(buf) + return space.newbytes(buf) @unwrap_spec(family=int, packed=str) def inet_ntop(space, family, packed): @@ -263,10 +263,10 @@ if space.is_w(w_host, space.w_None): host = None elif space.isinstance_w(w_host, space.w_str): - host = space.str_w(w_host) + host = space.bytes_w(w_host) elif space.isinstance_w(w_host, space.w_unicode): w_shost = space.call_method(w_host, "encode", space.wrap("idna")) - host = space.str_w(w_shost) + host = space.bytes_w(w_shost) else: raise oefmt(space.w_TypeError, "getaddrinfo() argument 1 must be string or None") @@ -277,7 +277,7 @@ elif space.isinstance_w(w_port, space.w_int) or space.isinstance_w(w_port, space.w_long): port = str(space.int_w(w_port)) elif space.isinstance_w(w_port, space.w_str): - port = space.str_w(w_port) + port = space.bytes_w(w_port) else: raise oefmt(space.w_TypeError, "getaddrinfo() argument 2 must be integer or string") diff --git a/pypy/module/_socket/interp_socket.py b/pypy/module/_socket/interp_socket.py --- a/pypy/module/_socket/interp_socket.py +++ b/pypy/module/_socket/interp_socket.py @@ -296,7 +296,7 @@ except SocketError as e: raise converted_error(space, e) buflen = space.int_w(w_buflen) - return space.wrap(self.sock.getsockopt(level, optname, buflen)) + return space.newbytes(self.sock.getsockopt(level, optname, buflen)) def gettimeout_w(self, space): """gettimeout() -> timeout @@ -345,7 +345,7 @@ data = self.sock.recv(buffersize, flags) except SocketError as e: raise converted_error(space, e) - return space.wrap(data) + return space.newbytes(data) @unwrap_spec(buffersize='nonnegint', flags=int) def recvfrom_w(self, space, buffersize, flags=0): @@ -359,7 +359,7 @@ w_addr = addr_as_object(addr, self.sock.fd, space) else: w_addr = space.w_None - return space.newtuple([space.wrap(data), w_addr]) + return space.newtuple([space.newbytes(data), w_addr]) except SocketError as e: raise converted_error(space, e) @@ -436,7 +436,7 @@ except OperationError as e: if e.async(space): raise - optval = space.str_w(w_optval) + optval = space.bytes_w(w_optval) try: self.sock.setsockopt(level, optname, optval) except SocketError as e: diff --git a/pypy/module/_sre/interp_sre.py b/pypy/module/_sre/interp_sre.py --- a/pypy/module/_sre/interp_sre.py +++ b/pypy/module/_sre/interp_sre.py @@ -36,11 +36,12 @@ def slice_w(space, ctx, start, end, w_default): if 0 <= start <= end: if isinstance(ctx, rsre_core.BufMatchContext): - return space.wrap(ctx._buffer.getslice(start, end, 1, end-start)) + return space.newbytes(ctx._buffer.getslice(start, end, 1, + end-start)) if isinstance(ctx, rsre_core.StrMatchContext): - return space.wrap(ctx._string[start:end]) + return space.newbytes(ctx._string[start:end]) elif isinstance(ctx, rsre_core.UnicodeMatchContext): - return space.wrap(ctx._unicodestr[start:end]) + return space.newunicode(ctx._unicodestr[start:end]) else: # unreachable raise SystemError @@ -242,7 +243,7 @@ space.isinstance_w(w_string, space.w_unicode) and literal) else: try: - filter_as_string = space.str_w(w_ptemplate) + filter_as_string = space.bytes_w(w_ptemplate) except OperationError as e: if e.async(space): raise @@ -331,15 +332,15 @@ strbuilder, unicodebuilder, last_pos, ctx.end) if use_builder: if strbuilder is not None: - return space.wrap(strbuilder.build()), n + return space.newbytes(strbuilder.build()), n else: assert unicodebuilder is not None - return space.wrap(unicodebuilder.build()), n + return space.newunicode(unicodebuilder.build()), n else: if space.isinstance_w(w_string, space.w_unicode): - w_emptystr = space.wrap(u'') + w_emptystr = space.newunicode(u'') else: - w_emptystr = space.wrap('') + w_emptystr = space.newbytes('') w_item = space.call_method(w_emptystr, 'join', space.newlist(sublist_w)) return w_item, n @@ -565,11 +566,11 @@ def fget_string(self, space): ctx = self.ctx if isinstance(ctx, rsre_core.BufMatchContext): - return space.wrap(ctx._buffer.as_str()) + return space.newbytes(ctx._buffer.as_str()) elif isinstance(ctx, rsre_core.StrMatchContext): - return space.wrap(ctx._string) + return space.newbytes(ctx._string) elif isinstance(ctx, rsre_core.UnicodeMatchContext): - return space.wrap(ctx._unicodestr) + return space.newunicode(ctx._unicodestr) else: raise SystemError diff --git a/pypy/module/_ssl/interp_ssl.py b/pypy/module/_ssl/interp_ssl.py --- a/pypy/module/_ssl/interp_ssl.py +++ b/pypy/module/_ssl/interp_ssl.py @@ -666,7 +666,7 @@ length = libssl_SSL_get_peer_finished(self.ssl, buf, CB_MAXLEN) if length > 0: - return space.wrap(rffi.charpsize2str(buf, intmask(length))) + return space.newbytes(rffi.charpsize2str(buf, intmask(length))) def descr_get_context(self, space): return self.w_ctx @@ -707,7 +707,7 @@ if length < 0: raise _ssl_seterror(space, None, 0) try: - return space.wrap(rffi.charpsize2str(buf_ptr[0], length)) + return space.newbytes(rffi.charpsize2str(buf_ptr[0], length)) finally: libssl_OPENSSL_free(buf_ptr[0]) @@ -926,7 +926,7 @@ if length < 0: raise _ssl_seterror(space, None, 0) try: - w_value = space.wrap(rffi.charpsize2str(buf_ptr[0], length)) + w_value = space.newbytes(rffi.charpsize2str(buf_ptr[0], length)) w_value = space.call_method(w_value, "decode", space.wrap("utf-8")) finally: libssl_OPENSSL_free(buf_ptr[0]) @@ -1232,7 +1232,7 @@ w_ssl_socket, space.w_None, w_ctx) else: - w_servername = space.wrapbytes(rffi.charp2str(servername)) + w_servername = space.newbytes(rffi.charp2str(servername)) try: w_servername_idna = space.call_method( w_servername, 'decode', space.wrap('idna')) @@ -1778,7 +1778,7 @@ if not path: return space.w_None else: - return space.wrapbytes(rffi.charp2str(path)) + return space.newbytes(rffi.charp2str(path)) def get_default_verify_paths(space): return space.newtuple([ diff --git a/pypy/module/_ssl/interp_win32.py b/pypy/module/_ssl/interp_win32.py --- a/pypy/module/_ssl/interp_win32.py +++ b/pypy/module/_ssl/interp_win32.py @@ -74,7 +74,7 @@ def w_parseKeyUsage(space, pCertCtx, flags): with lltype.scoped_alloc(rwin32.LPDWORD.TO, 1) as size_ptr: - if not CertGetEnhancedKeyUsage(pCertCtx, flags, + if not CertGetEnhancedKeyUsage(pCertCtx, flags, lltype.nullptr(CERT_ENHKEY_USAGE), size_ptr): last_error = rwin32.lastSavedWindowsError() if last_error.winerror == CRYPT_E_NOT_FOUND: @@ -120,7 +120,7 @@ pCertCtx = CertEnumCertificatesInStore(hStore, pCertCtx) if not pCertCtx: break - w_cert = space.wrapbytes( + w_cert = space.newbytes( rffi.charpsize2str(pCertCtx.c_pbCertEncoded, intmask(pCertCtx.c_cbCertEncoded))) w_enc = w_certEncodingType(space, pCertCtx.c_dwCertEncodingType) @@ -162,7 +162,7 @@ pCrlCtx = CertEnumCRLsInStore(hStore, pCrlCtx) if not pCrlCtx: break - w_crl = space.wrapbytes( + w_crl = space.newbytes( rffi.charpsize2str(pCrlCtx.c_pbCrlEncoded, intmask(pCrlCtx.c_cbCrlEncoded))) w_enc = w_certEncodingType(space, pCrlCtx.c_dwCertEncodingType) diff --git a/pypy/module/_winreg/interp_winreg.py b/pypy/module/_winreg/interp_winreg.py --- a/pypy/module/_winreg/interp_winreg.py +++ b/pypy/module/_winreg/interp_winreg.py @@ -380,7 +380,7 @@ return space.newlist(l) else: # REG_BINARY and all other types - return space.wrap(rffi.charpsize2str(buf, buflen)) + return space.newbytes(rffi.charpsize2str(buf, buflen)) @unwrap_spec(value_name=str, typ=int) def SetValueEx(space, w_hkey, value_name, w_reserved, typ, w_value): diff --git a/pypy/module/array/interp_array.py b/pypy/module/array/interp_array.py --- a/pypy/module/array/interp_array.py +++ b/pypy/module/array/interp_array.py @@ -225,11 +225,11 @@ """ size = self.len if size == 0: - return space.wrap('') + return space.newbytes('') cbuf = self._charbuf_start() s = rffi.charpsize2str(cbuf, size * self.itemsize) self._charbuf_stop() - return self.space.wrap(s) + return self.space.newbytes(s) def descr_fromstring(self, space, w_s): """ fromstring(string) @@ -263,7 +263,7 @@ except OverflowError: raise MemoryError w_item = space.call_method(w_f, 'read', space.wrap(size)) - item = space.str_w(w_item) + item = space.bytes_w(w_item) if len(item) < size: n = len(item) % self.itemsize elems = max(0, len(item) - (len(item) % self.itemsize)) @@ -338,10 +338,10 @@ else: args = [space.wrap(self.typecode)] try: - dct = space.getattr(self, space.wrap('__dict__')) + w_dict = space.getattr(self, space.wrap('__dict__')) except OperationError: - dct = space.w_None - return space.newtuple([space.type(self), space.newtuple(args), dct]) + w_dict = space.w_None + return space.newtuple([space.type(self), space.newtuple(args), w_dict]) def descr_copy(self, space): """ copy(array) diff --git a/pypy/module/binascii/interp_base64.py b/pypy/module/binascii/interp_base64.py --- a/pypy/module/binascii/interp_base64.py +++ b/pypy/module/binascii/interp_base64.py @@ -71,7 +71,7 @@ if leftbits != 0: raise_Error(space, "Incorrect padding") - return space.wrap(res.build()) + return space.newbytes(res.build()) # ____________________________________________________________ @@ -110,4 +110,4 @@ res.append(table_b2a_base64[(leftchar & 0xf) << 2]) res.append(PAD) res.append('\n') - return space.wrap(res.build()) + return space.newbytes(res.build()) diff --git a/pypy/module/binascii/interp_hexlify.py b/pypy/module/binascii/interp_hexlify.py --- a/pypy/module/binascii/interp_hexlify.py +++ b/pypy/module/binascii/interp_hexlify.py @@ -24,7 +24,7 @@ for c in data: res.append(_value2char(ord(c) >> 4)) res.append(_value2char(ord(c) & 0xf)) - return space.wrap(res.build()) + return space.newbytes(res.build()) # ____________________________________________________________ @@ -53,4 +53,4 @@ a = _char2value(space, hexstr[i]) b = _char2value(space, hexstr[i+1]) res.append(chr((a << 4) | b)) - return space.wrap(res.build()) + return space.newbytes(res.build()) diff --git a/pypy/module/binascii/interp_hqx.py b/pypy/module/binascii/interp_hqx.py --- a/pypy/module/binascii/interp_hqx.py +++ b/pypy/module/binascii/interp_hqx.py @@ -11,37 +11,37 @@ FAIL = 0x7d table_a2b_hqx = [ - #^@ ^A ^B ^C ^D ^E ^F ^G + #^@ ^A ^B ^C ^D ^E ^F ^G FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, - #\b \t \n ^K ^L \r ^N ^O + #\b \t \n ^K ^L \r ^N ^O FAIL, FAIL, SKIP, FAIL, FAIL, SKIP, FAIL, FAIL, - #^P ^Q ^R ^S ^T ^U ^V ^W + #^P ^Q ^R ^S ^T ^U ^V ^W FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, - #^X ^Y ^Z ^[ ^\ ^] ^^ ^_ + #^X ^Y ^Z ^[ ^\ ^] ^^ ^_ FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, - # ! " # $ % & ' + # ! " # $ % & ' FAIL, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, - #( ) * + , - . / + #( ) * + , - . / 0x07, 0x08, 0x09, 0x0A, 0x0B, 0x0C, FAIL, FAIL, - #0 1 2 3 4 5 6 7 + #0 1 2 3 4 5 6 7 0x0D, 0x0E, 0x0F, 0x10, 0x11, 0x12, 0x13, FAIL, - #8 9 : ; < = > ? + #8 9 : ; < = > ? 0x14, 0x15, DONE, FAIL, FAIL, FAIL, FAIL, FAIL, - #@ A B C D E F G + #@ A B C D E F G 0x16, 0x17, 0x18, 0x19, 0x1A, 0x1B, 0x1C, 0x1D, - #H I J K L M N O + #H I J K L M N O 0x1E, 0x1F, 0x20, 0x21, 0x22, 0x23, 0x24, FAIL, - #P Q R S T U V W + #P Q R S T U V W 0x25, 0x26, 0x27, 0x28, 0x29, 0x2A, 0x2B, FAIL, - #X Y Z [ \ ] ^ _ + #X Y Z [ \ ] ^ _ 0x2C, 0x2D, 0x2E, 0x2F, FAIL, FAIL, FAIL, FAIL, - #` a b c d e f g + #` a b c d e f g 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, FAIL, - #h i j k l m n o + #h i j k l m n o 0x37, 0x38, 0x39, 0x3A, 0x3B, 0x3C, FAIL, FAIL, - #p q r s t u v w + #p q r s t u v w 0x3D, 0x3E, 0x3F, FAIL, FAIL, FAIL, FAIL, FAIL, - #x y z { | } ~ ^? + #x y z { | } ~ ^? FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, @@ -97,7 +97,7 @@ else: if pending_bits > 0: raise_Incomplete(space, 'String has incomplete number of bytes') - return space.newtuple([space.wrap(res.build()), space.wrap(done)]) + return space.newtuple([space.newbytes(res.build()), space.wrap(done)]) # ____________________________________________________________ @@ -128,7 +128,7 @@ if leftbits > 0: leftchar <<= (6 - leftbits) res.append(hqx_encoding[leftchar & 0x3f]) - return space.wrap(res.build()) + return space.newbytes(res.build()) # ____________________________________________________________ @@ -150,7 +150,7 @@ lastpushed = ord(c) else: if i == end: - raise_Incomplete(space, 'String ends with the RLE code \x90') + raise_Incomplete(space, 'String ends with the RLE code \\x90') count = ord(hexbin[i]) - 1 i += 1 if count < 0: @@ -158,9 +158,9 @@ lastpushed = 0x90 else: if lastpushed < 0: - raise_Error(space, 'String starts with the RLE code \x90') + raise_Error(space, 'String starts with the RLE code \\x90') res.append_multiple_char(chr(lastpushed), count) - return space.wrap(res.build()) + return space.newbytes(res.build()) # ____________________________________________________________ @@ -197,7 +197,7 @@ # string that rledecode_hqx() would expand back to 'data', there are # some programs somewhere that would start failing obscurely in rare # cases. - return space.wrap(res.build()) + return space.newbytes(res.build()) # ____________________________________________________________ diff --git a/pypy/module/binascii/interp_qp.py b/pypy/module/binascii/interp_qp.py --- a/pypy/module/binascii/interp_qp.py +++ b/pypy/module/binascii/interp_qp.py @@ -56,7 +56,7 @@ if header and c == '_': c = ' ' odata.append(c) - return space.wrap(odata.build()) + return space.newbytes(odata.build()) # ____________________________________________________________ @@ -159,4 +159,4 @@ odata.append(c) inp += 1 - return space.wrap(odata.build()) + return space.newbytes(odata.build()) diff --git a/pypy/module/binascii/interp_uu.py b/pypy/module/binascii/interp_uu.py --- a/pypy/module/binascii/interp_uu.py +++ b/pypy/module/binascii/interp_uu.py @@ -54,7 +54,7 @@ remaining = length - res.getlength() if remaining > 0: res.append_multiple_char('\x00', remaining) - return space.wrap(res.build()) + return space.newbytes(res.build()) # ____________________________________________________________ @@ -86,4 +86,4 @@ res.append(chr(0x20 + (C & 0x3F))) res.append('\n') - return space.wrap(res.build()) + return space.newbytes(res.build()) diff --git a/pypy/module/bz2/interp_bz2.py b/pypy/module/bz2/interp_bz2.py --- a/pypy/module/bz2/interp_bz2.py +++ b/pypy/module/bz2/interp_bz2.py @@ -557,7 +557,7 @@ datasize = len(data) if datasize == 0: - return self.space.wrap("") + return self.space.newbytes("") if not self.running: raise oefmt(self.space.w_ValueError, @@ -582,7 +582,7 @@ out.prepare_next_chunk() res = out.make_result_string() - return self.space.wrap(res) + return self.space.newbytes(res) def flush(self): if not self.running: @@ -602,7 +602,7 @@ out.prepare_next_chunk() res = out.make_result_string() - return self.space.wrap(res) + return self.space.newbytes(res) W_BZ2Compressor.typedef = TypeDef("BZ2Compressor", __doc__ = W_BZ2Compressor.__doc__, @@ -669,7 +669,7 @@ raise oefmt(self.space.w_EOFError, "end of stream was already found") if data == '': - return self.space.wrap('') + return self.space.newbytes('') in_bufsize = len(data) @@ -698,7 +698,7 @@ out.prepare_next_chunk() res = out.make_result_string() - return self.space.wrap(res) + return self.space.newbytes(res) W_BZ2Decompressor.typedef = TypeDef("BZ2Decompressor", diff --git a/pypy/module/cpyext/bufferobject.py b/pypy/module/cpyext/bufferobject.py --- a/pypy/module/cpyext/bufferobject.py +++ b/pypy/module/cpyext/bufferobject.py @@ -79,5 +79,5 @@ Py_DecRef(space, py_buf.c_b_base) else: rffi.free_charp(rffi.cast(rffi.CCHARP, py_buf.c_b_ptr)) - from pypy.module.cpyext.object import PyObject_dealloc - PyObject_dealloc(space, py_obj) + from pypy.module.cpyext.object import _dealloc + _dealloc(space, py_obj) diff --git a/pypy/module/cpyext/bytearrayobject.py b/pypy/module/cpyext/bytearrayobject.py --- a/pypy/module/cpyext/bytearrayobject.py +++ b/pypy/module/cpyext/bytearrayobject.py @@ -25,55 +25,11 @@ PyByteArrayObjectStruct = lltype.ForwardReference() PyByteArrayObject = lltype.Ptr(PyByteArrayObjectStruct) -PyByteArrayObjectFields = PyVarObjectFields -# (("ob_exports", rffi.INT), ("ob_alloc", rffi.LONG), ("ob_bytes", rffi.CCHARP)) +PyByteArrayObjectFields = PyVarObjectFields cpython_struct("PyByteArrayObject", PyByteArrayObjectFields, PyByteArrayObjectStruct) - at bootstrap_function -def init_bytearrayobject(space): - "Type description of PyByteArrayObject" - #make_typedescr(space.w_bytearray.layout.typedef, - # basestruct=PyByteArrayObject.TO, - # attach=bytearray_attach, - # dealloc=bytearray_dealloc, - # realize=bytearray_realize) - PyByteArray_Check, PyByteArray_CheckExact = build_type_checkers("ByteArray", "w_bytearray") -# XXX dead code to be removed -#def bytearray_attach(space, py_obj, w_obj): -# """ -# Fills a newly allocated PyByteArrayObject with the given bytearray object -# """ -# py_ba = rffi.cast(PyByteArrayObject, py_obj) -# py_ba.c_ob_size = len(space.str_w(w_obj)) -# py_ba.c_ob_bytes = lltype.nullptr(rffi.CCHARP.TO) -# py_ba.c_ob_exports = rffi.cast(rffi.INT, 0) - -#def bytearray_realize(space, py_obj): -# """ -# Creates the bytearray in the interpreter. -# """ -# py_ba = rffi.cast(PyByteArrayObject, py_obj) -# if not py_ba.c_ob_bytes: -# py_ba.c_buffer = lltype.malloc(rffi.CCHARP.TO, py_ba.c_ob_size + 1, -# flavor='raw', zero=True) -# s = rffi.charpsize2str(py_ba.c_ob_bytes, py_ba.c_ob_size) -# w_obj = space.wrap(s) -# py_ba.c_ob_exports = rffi.cast(rffi.INT, 0) -# track_reference(space, py_obj, w_obj) -# return w_obj - -#@cpython_api([PyObject], lltype.Void, header=None) -#def bytearray_dealloc(space, py_obj): -# """Frees allocated PyByteArrayObject resources. -# """ -# py_ba = rffi.cast(PyByteArrayObject, py_obj) -# if py_ba.c_ob_bytes: -# lltype.free(py_ba.c_ob_bytes, flavor="raw") -# from pypy.module.cpyext.object import PyObject_dealloc -# PyObject_dealloc(space, py_obj) - #_______________________________________________________________________ @cpython_api([PyObject], PyObject, result_is_ll=True) @@ -90,9 +46,9 @@ """Create a new bytearray object from string and its length, len. On failure, NULL is returned.""" if char_p: - w_s = space.wrap(rffi.charpsize2str(char_p, length)) + w_s = space.newbytes(rffi.charpsize2str(char_p, length)) else: - w_s = space.wrap(length) + w_s = space.newint(length) w_buffer = space.call_function(space.w_bytearray, w_s) return make_ref(space, w_buffer) @@ -124,7 +80,7 @@ if space.isinstance_w(w_obj, space.w_bytearray): oldlen = space.len_w(w_obj) if newlen > oldlen: - space.call_method(w_obj, 'extend', space.wrap('\x00' * (newlen - oldlen))) + space.call_method(w_obj, 'extend', space.newbytes('\x00' * (newlen - oldlen))) elif oldlen > newlen: assert newlen >= 0 space.delslice(w_obj, space.wrap(newlen), space.wrap(oldlen)) diff --git a/pypy/module/cpyext/bytesobject.py b/pypy/module/cpyext/bytesobject.py --- a/pypy/module/cpyext/bytesobject.py +++ b/pypy/module/cpyext/bytesobject.py @@ -11,7 +11,7 @@ from pypy.objspace.std.bytesobject import W_BytesObject ## -## Implementation of PyStringObject +## Implementation of PyBytesObject ## ================================ ## ## The problem @@ -29,15 +29,15 @@ ## Solution ## -------- ## -## PyStringObject contains two additional members: the ob_size and a pointer to a +## PyBytesObject contains two additional members: the ob_size and a pointer to a ## char ob_sval; it may be NULL. ## -## - A string allocated by pypy will be converted into a PyStringObject with a +## - A string allocated by pypy will be converted into a PyBytesObject with a ## NULL buffer. The first time PyString_AsString() is called, memory is ## allocated (with flavor='raw') and content is copied. ## ## - A string allocated with PyString_FromStringAndSize(NULL, size) will -## allocate a PyStringObject structure, and a buffer with the specified +## allocate a PyBytesObject structure, and a buffer with the specified ## size+1, but the reference won't be stored in the global map; there is no ## corresponding object in pypy. When from_ref() or Py_INCREF() is called, ## the pypy string is created, and added to the global map of tracked @@ -55,58 +55,58 @@ ## corresponds to the pypy gc-managed string. ## -PyStringObjectStruct = lltype.ForwardReference() -PyStringObject = lltype.Ptr(PyStringObjectStruct) -PyStringObjectFields = PyVarObjectFields + \ +PyBytesObjectStruct = lltype.ForwardReference() +PyBytesObject = lltype.Ptr(PyBytesObjectStruct) +PyBytesObjectFields = PyVarObjectFields + \ (("ob_shash", rffi.LONG), ("ob_sstate", rffi.INT), ("ob_sval", rffi.CArray(lltype.Char))) -cpython_struct("PyStringObject", PyStringObjectFields, PyStringObjectStruct) +cpython_struct("PyStringObject", PyBytesObjectFields, PyBytesObjectStruct) @bootstrap_function -def init_stringobject(space): - "Type description of PyStringObject" +def init_bytesobject(space): + "Type description of PyBytesObject" make_typedescr(space.w_str.layout.typedef, - basestruct=PyStringObject.TO, - attach=string_attach, - dealloc=string_dealloc, - realize=string_realize) + basestruct=PyBytesObject.TO, + attach=bytes_attach, + dealloc=bytes_dealloc, + realize=bytes_realize) PyString_Check, PyString_CheckExact = build_type_checkers("String", "w_str") def new_empty_str(space, length): """ - Allocate a PyStringObject and its ob_sval, but without a corresponding - interpreter object. The ob_sval may be mutated, until string_realize() is + Allocate a PyBytesObject and its ob_sval, but without a corresponding + interpreter object. The ob_sval may be mutated, until bytes_realize() is called. Refcount of the result is 1. """ typedescr = get_typedescr(space.w_str.layout.typedef) py_obj = typedescr.allocate(space, space.w_str, length) - py_str = rffi.cast(PyStringObject, py_obj) + py_str = rffi.cast(PyBytesObject, py_obj) py_str.c_ob_shash = -1 py_str.c_ob_sstate = rffi.cast(rffi.INT, 0) # SSTATE_NOT_INTERNED return py_str -def string_attach(space, py_obj, w_obj): +def bytes_attach(space, py_obj, w_obj): """ - Copy RPython string object contents to a PyStringObject. The + Copy RPython string object contents to a PyBytesObject. The c_ob_sval must not be modified. """ - py_str = rffi.cast(PyStringObject, py_obj) + py_str = rffi.cast(PyBytesObject, py_obj) s = space.str_w(w_obj) if py_str.c_ob_size < len(s): raise oefmt(space.w_ValueError, - "string_attach called on object with ob_size %d but trying to store %d", - py_str.c_ob_size, len(s)) + "bytes_attach called on object with ob_size %d but trying to store %d", + py_str.c_ob_size, len(s)) rffi.c_memcpy(py_str.c_ob_sval, rffi.str2charp(s), len(s)) py_str.c_ob_sval[len(s)] = '\0' py_str.c_ob_shash = space.hash_w(w_obj) py_str.c_ob_sstate = rffi.cast(rffi.INT, 1) # SSTATE_INTERNED_MORTAL -def string_realize(space, py_obj): +def bytes_realize(space, py_obj): """ - Creates the string in the interpreter. The PyStringObject ob_sval must not + Creates the string in the interpreter. The PyBytesObject ob_sval must not be modified after this call. """ - py_str = rffi.cast(PyStringObject, py_obj) + py_str = rffi.cast(PyBytesObject, py_obj) s = rffi.charpsize2str(py_str.c_ob_sval, py_str.c_ob_size) w_type = from_ref(space, rffi.cast(PyObject, py_obj.c_ob_type)) w_obj = space.allocate_instance(W_BytesObject, w_type) @@ -117,11 +117,11 @@ return w_obj @cpython_api([PyObject], lltype.Void, header=None) -def string_dealloc(space, py_obj): - """Frees allocated PyStringObject resources. +def bytes_dealloc(space, py_obj): + """Frees allocated PyBytesObject resources. """ - from pypy.module.cpyext.object import PyObject_dealloc - PyObject_dealloc(space, py_obj) + from pypy.module.cpyext.object import _dealloc + _dealloc(space, py_obj) #_______________________________________________________________________ @@ -154,10 +154,10 @@ raise oefmt(space.w_TypeError, "expected string or Unicode object, %T found", from_ref(space, ref)) - ref_str = rffi.cast(PyStringObject, ref) + ref_str = rffi.cast(PyBytesObject, ref) if not pyobj_has_w_obj(ref): # XXX Force the ref? - string_realize(space, ref) + bytes_realize(space, ref) return ref_str.c_ob_sval @cpython_api([rffi.VOIDP], rffi.CCHARP, error=0) @@ -166,7 +166,7 @@ # if no w_str is associated with this ref, # return the c-level ptr as RW if not pyobj_has_w_obj(ref): - py_str = rffi.cast(PyStringObject, ref) + py_str = rffi.cast(PyBytesObject, ref) return py_str.c_ob_sval return _PyString_AsString(space, ref) @@ -183,8 +183,8 @@ from_ref(space, ref)) if not pyobj_has_w_obj(ref): # force the ref - string_realize(space, ref) - ref_str = rffi.cast(PyStringObject, ref) + bytes_realize(space, ref) + ref_str = rffi.cast(PyBytesObject, ref) data[0] = ref_str.c_ob_sval if length: length[0] = ref_str.c_ob_size @@ -200,7 +200,7 @@ @cpython_api([PyObject], Py_ssize_t, error=-1) def PyString_Size(space, ref): if from_ref(space, rffi.cast(PyObject, ref.c_ob_type)) is space.w_str: - ref = rffi.cast(PyStringObject, ref) + ref = rffi.cast(PyBytesObject, ref) return ref.c_ob_size else: w_obj = from_ref(space, ref) @@ -222,7 +222,7 @@ if pyobj_has_w_obj(ref[0]): raise oefmt(space.w_SystemError, "_PyString_Resize called on already created string") - py_str = rffi.cast(PyStringObject, ref[0]) + py_str = rffi.cast(PyBytesObject, ref[0]) try: py_newstr = new_empty_str(space, newsize) except MemoryError: diff --git a/pypy/module/cpyext/eval.py b/pypy/module/cpyext/eval.py --- a/pypy/module/cpyext/eval.py +++ b/pypy/module/cpyext/eval.py @@ -13,7 +13,7 @@ "PyCompilerFlags", (("cf_flags", rffi.INT),)) PyCompilerFlagsPtr = lltype.Ptr(PyCompilerFlags) -PyCF_MASK = (consts.CO_FUTURE_DIVISION | +PyCF_MASK = (consts.CO_FUTURE_DIVISION | consts.CO_FUTURE_ABSOLUTE_IMPORT | consts.CO_FUTURE_WITH_STATEMENT | consts.CO_FUTURE_PRINT_FUNCTION | @@ -94,7 +94,7 @@ Py_eval_input = 258 def compile_string(space, source, filename, start, flags=0): - w_source = space.wrap(source) + w_source = space.newbytes(source) start = rffi.cast(lltype.Signed, start) if start == Py_file_input: mode = 'exec' @@ -227,4 +227,4 @@ cf.c_cf_flags = rffi.cast(rffi.INT, flags) return result - + diff --git a/pypy/module/cpyext/frameobject.py b/pypy/module/cpyext/frameobject.py --- a/pypy/module/cpyext/frameobject.py +++ b/pypy/module/cpyext/frameobject.py @@ -46,8 +46,8 @@ Py_DecRef(space, py_code) Py_DecRef(space, py_frame.c_f_globals) Py_DecRef(space, py_frame.c_f_locals) - from pypy.module.cpyext.object import PyObject_dealloc - PyObject_dealloc(space, py_obj) + from pypy.module.cpyext.object import _dealloc + _dealloc(space, py_obj) def frame_realize(space, py_obj): """ diff --git a/pypy/module/cpyext/funcobject.py b/pypy/module/cpyext/funcobject.py --- a/pypy/module/cpyext/funcobject.py +++ b/pypy/module/cpyext/funcobject.py @@ -60,8 +60,8 @@ def function_dealloc(space, py_obj): py_func = rffi.cast(PyFunctionObject, py_obj) Py_DecRef(space, py_func.c_func_name) - from pypy.module.cpyext.object import PyObject_dealloc - PyObject_dealloc(space, py_obj) + from pypy.module.cpyext.object import _dealloc + _dealloc(space, py_obj) def code_attach(space, py_obj, w_obj): py_code = rffi.cast(PyCodeObject, py_obj) @@ -80,8 +80,8 @@ py_code = rffi.cast(PyCodeObject, py_obj) Py_DecRef(space, py_code.c_co_name) Py_DecRef(space, py_code.c_co_filename) - from pypy.module.cpyext.object import PyObject_dealloc - PyObject_dealloc(space, py_obj) + from pypy.module.cpyext.object import _dealloc + _dealloc(space, py_obj) @cpython_api([PyObject], PyObject, result_borrowed=True) def PyFunction_GetCode(space, w_func): diff --git a/pypy/module/cpyext/include/tupleobject.h b/pypy/module/cpyext/include/tupleobject.h --- a/pypy/module/cpyext/include/tupleobject.h +++ b/pypy/module/cpyext/include/tupleobject.h @@ -8,9 +8,12 @@ #endif typedef struct { - PyObject_HEAD - Py_ssize_t ob_size; - PyObject **ob_item; /* XXX optimize to ob_item[] */ + PyObject_VAR_HEAD + PyObject *ob_item[1]; + /* ob_item contains space for 'ob_size' elements. + * Items must normally not be NULL, except during construction when + * the tuple is not yet visible outside the function that builds it. + */ } PyTupleObject; /* defined in varargswrapper.c */ diff --git a/pypy/module/cpyext/methodobject.py b/pypy/module/cpyext/methodobject.py --- a/pypy/module/cpyext/methodobject.py +++ b/pypy/module/cpyext/methodobject.py @@ -55,8 +55,8 @@ py_func = rffi.cast(PyCFunctionObject, py_obj) Py_DecRef(space, py_func.c_m_self) Py_DecRef(space, py_func.c_m_module) - from pypy.module.cpyext.object import PyObject_dealloc - PyObject_dealloc(space, py_obj) + from pypy.module.cpyext.object import _dealloc + _dealloc(space, py_obj) class W_PyCFunctionObject(W_Root): diff --git a/pypy/module/cpyext/object.py b/pypy/module/cpyext/object.py --- a/pypy/module/cpyext/object.py +++ b/pypy/module/cpyext/object.py @@ -54,6 +54,9 @@ @cpython_api([PyObject], lltype.Void) def PyObject_dealloc(space, obj): + return _dealloc(space, obj) + +def _dealloc(space, obj): # This frees an object after its refcount dropped to zero, so we # assert that it is really zero here. assert obj.c_ob_refcnt == 0 diff --git a/pypy/module/cpyext/pyfile.py b/pypy/module/cpyext/pyfile.py --- a/pypy/module/cpyext/pyfile.py +++ b/pypy/module/cpyext/pyfile.py @@ -23,7 +23,7 @@ try: w_readline = space.getattr(w_obj, space.wrap('readline')) except OperationError: - raise oefmt(space.w_TypeError, + raise oefmt(space.w_TypeError, "argument must be a file, or have a readline() method.") n = rffi.cast(lltype.Signed, n) @@ -41,7 +41,7 @@ On success, return a new file object that is opened on the file given by filename, with a file mode given by mode, where mode has the same semantics as the standard C routine fopen(). On failure, return NULL.""" - w_filename = space.wrap(rffi.charp2str(filename)) + w_filename = space.newbytes(rffi.charp2str(filename)) w_mode = space.wrap(rffi.charp2str(mode)) return space.call_method(space.builtin, 'file', w_filename, w_mode) diff --git a/pypy/module/cpyext/pyobject.py b/pypy/module/cpyext/pyobject.py --- a/pypy/module/cpyext/pyobject.py +++ b/pypy/module/cpyext/pyobject.py @@ -164,7 +164,7 @@ pytype = rffi.cast(PyTypeObjectPtr, as_pyobj(space, w_type)) typedescr = get_typedescr(w_obj.typedef) if pytype.c_tp_itemsize != 0: - itemcount = space.len_w(w_obj) # PyStringObject and subclasses + itemcount = space.len_w(w_obj) # PyBytesObject and subclasses else: itemcount = 0 py_obj = typedescr.allocate(space, w_type, itemcount=itemcount) diff --git a/pypy/module/cpyext/pytraceback.py b/pypy/module/cpyext/pytraceback.py --- a/pypy/module/cpyext/pytraceback.py +++ b/pypy/module/cpyext/pytraceback.py @@ -46,5 +46,5 @@ py_traceback = rffi.cast(PyTracebackObject, py_obj) Py_DecRef(space, rffi.cast(PyObject, py_traceback.c_tb_next)) Py_DecRef(space, rffi.cast(PyObject, py_traceback.c_tb_frame)) - from pypy.module.cpyext.object import PyObject_dealloc - PyObject_dealloc(space, py_obj) + from pypy.module.cpyext.object import _dealloc + _dealloc(space, py_obj) diff --git a/pypy/module/cpyext/sliceobject.py b/pypy/module/cpyext/sliceobject.py --- a/pypy/module/cpyext/sliceobject.py +++ b/pypy/module/cpyext/sliceobject.py @@ -38,14 +38,14 @@ @cpython_api([PyObject], lltype.Void, header=None) def slice_dealloc(space, py_obj): - """Frees allocated PyStringObject resources. + """Frees allocated PyBytesObject resources. """ py_slice = rffi.cast(PySliceObject, py_obj) Py_DecRef(space, py_slice.c_start) Py_DecRef(space, py_slice.c_stop) Py_DecRef(space, py_slice.c_step) - from pypy.module.cpyext.object import PyObject_dealloc - PyObject_dealloc(space, py_obj) + from pypy.module.cpyext.object import _dealloc + _dealloc(space, py_obj) PySlice_Check, PySlice_CheckExact = build_type_checkers("Slice") @@ -73,7 +73,7 @@ length length, and store the length of the slice in slicelength. Out of bounds indices are clipped in a manner consistent with the handling of normal slices. - + Returns 0 on success and -1 on error with exception set.""" if not PySlice_Check(space, w_slice): PyErr_BadInternalCall(space) @@ -88,11 +88,11 @@ """Retrieve the start, stop and step indices from the slice object slice, assuming a sequence of length length. Treats indices greater than length as errors. - + Returns 0 on success and -1 on error with no exception set (unless one of the indices was not None and failed to be converted to an integer, in which case -1 is returned with an exception set). - + You probably do not want to use this function. If you want to use slice objects in versions of Python prior to 2.3, you would probably do well to incorporate the source of PySlice_GetIndicesEx(), suitably renamed, diff --git a/pypy/module/cpyext/test/array.c b/pypy/module/cpyext/test/array.c --- a/pypy/module/cpyext/test/array.c +++ b/pypy/module/cpyext/test/array.c @@ -1858,6 +1858,110 @@ } } +static PyObject* +array_multiply(PyObject* obj1, PyObject* obj2) +{ + if (PyList_Check(obj1) && ((arrayobject*)obj2)->ob_descr->typecode == 'i' && Py_SIZE(obj2) == 1) + { + int ii, nn; + int n = PyList_Size(obj1); + PyObject *v = getarrayitem(obj2, 0); + int i = ((PyIntObject*)v)->ob_ival; + PyObject * ret = PyList_New(n*i); + for (ii = 0; ii < i; ii++) + for (nn = 0; nn < n; nn++) + { + v = PyList_GetItem(obj1, nn); + PyList_SetItem(ret, nn+ii*n, v); + } + return ret; + } + else if (PyList_Check(obj2) && ((arrayobject*)obj1)->ob_descr->typecode == 'i' && Py_SIZE(obj1) == 1) + { + int ii, nn; + int n = PyList_Size(obj2); + PyObject *v = getarrayitem(obj1, 0); + int i = ((PyIntObject*)v)->ob_ival; + PyObject * ret = PyList_New(n*i); + for (ii = 0; ii < i; ii++) + for (nn = 0; nn < n; nn++) + { + v = PyList_GetItem(obj2, nn); + PyList_SetItem(ret, nn+ii*n, v); + } + return ret; + } + else if(obj1->ob_type == &Arraytype) + fprintf(stderr, "\nCannot multiply array of type %c and %s\n", + ((arrayobject*)obj1)->ob_descr->typecode, obj2->ob_type->tp_name); + else if(obj2->ob_type == &Arraytype) + fprintf(stderr, "\nCannot multiply array of type %c and %s\n", + ((arrayobject*)obj2)->ob_descr->typecode, obj1->ob_type->tp_name); + Py_INCREF(Py_NotImplemented); + return Py_NotImplemented; +} + +static PyNumberMethods array_as_number = { + (binaryfunc)NULL, /* nb_add*/ + (binaryfunc)NULL, /* nb_subtract */ + (binaryfunc)array_multiply, /* nb_multiply */ + (binaryfunc)NULL, /* nb_divide */ +}; + +static PyObject* +array_base_multiply(PyObject* obj1, PyObject* obj2) +{ + if (PyList_Check(obj1) && ((arrayobject*)obj2)->ob_descr->typecode == 'i' && Py_SIZE(obj2) == 1) + { + int nn; + int n = PyList_Size(obj1); + PyObject *v = getarrayitem(obj2, 0); + int i = ((PyIntObject*)v)->ob_ival; + PyObject * ret = PyList_New(n); + for (nn = 0; nn < n; nn++) + { + v = PyList_GetItem(obj1, nn); + if (PyInt_Check(v)) + PyList_SetItem(ret, nn, PyLong_FromLong(i * ((PyIntObject*)v)->ob_ival)); + else + PyList_SetItem(ret, nn, v); + } + return ret; + } + else if (PyList_Check(obj2) && ((arrayobject*)obj1)->ob_descr->typecode == 'i' && Py_SIZE(obj1) == 1) + { + int nn; + int n = PyList_Size(obj2); + PyObject *v = getarrayitem(obj1, 0); + int i = ((PyIntObject*)v)->ob_ival; + PyObject * ret = PyList_New(n); + for (nn = 0; nn < n; nn++) + { + v = PyList_GetItem(obj2, nn); + if (PyInt_Check(v)) + PyList_SetItem(ret, nn, PyLong_FromLong(i * ((PyIntObject*)v)->ob_ival)); + else + PyList_SetItem(ret, nn, v); + } + return ret; + } + else if(obj1->ob_type == &Arraytype) + fprintf(stderr, "\nCannot multiply array of type %c and %s\n", + ((arrayobject*)obj1)->ob_descr->typecode, obj2->ob_type->tp_name); + else if(obj2->ob_type == &Arraytype) + fprintf(stderr, "\nCannot multiply array of type %c and %s\n", + ((arrayobject*)obj2)->ob_descr->typecode, obj1->ob_type->tp_name); + Py_INCREF(Py_NotImplemented); + return Py_NotImplemented; +} + +static PyNumberMethods array_base_as_number = { + (binaryfunc)NULL, /* nb_add*/ + (binaryfunc)NULL, /* nb_subtract */ + (binaryfunc)array_base_multiply, /* nb_multiply */ + (binaryfunc)NULL, /* nb_divide */ +}; + static PyMappingMethods array_as_mapping = { (lenfunc)array_length, (binaryfunc)array_subscr, @@ -2106,6 +2210,49 @@ static PyObject *array_iter(arrayobject *ao); +static PyTypeObject ArrayBasetype = { + PyVarObject_HEAD_INIT(NULL, 0) + "array.basearray", + sizeof(arrayobject), + 0, + (destructor)array_dealloc, /* tp_dealloc */ + 0, /* tp_print */ + 0, /* tp_getattr */ + 0, /* tp_setattr */ + 0, /* tp_compare */ + (reprfunc)array_repr, /* tp_repr */ + &array_base_as_number, /* tp_as_number*/ + &array_as_sequence, /* tp_as_sequence*/ + &array_as_mapping, /* tp_as_mapping*/ + 0, /* tp_hash */ + 0, /* tp_call */ + 0, /* tp_str */ + PyObject_GenericGetAttr, /* tp_getattro */ + 0, /* tp_setattro */ + &array_as_buffer, /* tp_as_buffer*/ + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | + Py_TPFLAGS_HAVE_WEAKREFS | Py_TPFLAGS_CHECKTYPES, /* tp_flags */ + arraytype_doc, /* tp_doc */ + 0, /* tp_traverse */ + 0, /* tp_clear */ + array_richcompare, /* tp_richcompare */ + offsetof(arrayobject, weakreflist), /* tp_weaklistoffset */ + (getiterfunc)array_iter, /* tp_iter */ + 0, /* tp_iternext */ + array_methods, /* tp_methods */ + 0, /* tp_members */ + array_getsets, /* tp_getset */ + 0, /* tp_base */ + 0, /* tp_dict */ + 0, /* tp_descr_get */ + 0, /* tp_descr_set */ + 0, /* tp_dictoffset */ + 0, /* tp_init */ + PyType_GenericAlloc, /* tp_alloc */ + array_new, /* tp_new */ + PyObject_Del, /* tp_free */ +}; + static PyTypeObject Arraytype = { PyVarObject_HEAD_INIT(NULL, 0) "array.array", @@ -2117,7 +2264,7 @@ 0, /* tp_setattr */ 0, /* tp_compare */ (reprfunc)array_repr, /* tp_repr */ - 0, /* tp_as_number*/ + &array_as_number, /* tp_as_number*/ &array_as_sequence, /* tp_as_sequence*/ &array_as_mapping, /* tp_as_mapping*/ 0, /* tp_hash */ @@ -2126,7 +2273,8 @@ PyObject_GenericGetAttr, /* tp_getattro */ 0, /* tp_setattro */ From pypy.commits at gmail.com Fri Jul 1 13:46:38 2016 From: pypy.commits at gmail.com (raffael_t) Date: Fri, 01 Jul 2016 10:46:38 -0700 (PDT) Subject: [pypy-commit] pypy py3.5: Check multiple values for keyword in map_unpack_with_call Message-ID: <5776ac7e.83261c0a.85f21.ffff960e@mx.google.com> Author: Raffael Tfirst Branch: py3.5 Changeset: r85501:17c11ede8071 Date: 2016-07-01 19:46 +0200 http://bitbucket.org/pypy/pypy/changeset/17c11ede8071/ Log: Check multiple values for keyword in map_unpack_with_call diff --git a/pypy/interpreter/pyopcode.py b/pypy/interpreter/pyopcode.py --- a/pypy/interpreter/pyopcode.py +++ b/pypy/interpreter/pyopcode.py @@ -1356,8 +1356,19 @@ def BUILD_MAP_UNPACK_WITH_CALL(self, itemcount, next_instr): num_maps = itemcount & 0xff w_dict = self.space.newdict() - import pdb; pdb.set_trace() - self.BUILD_MAP_UNPACK(num_maps, next_instr) + for i in range(num_maps, 0, -1): + w_item = self.peekvalue(i-1) + num_items = w_item.length() + for j in range(num_items): + (w_key, w_value) = w_item.popitem() + if self.space.is_true(self.space.contains(w_dict,w_key)): + raise oefmt(self.space.w_TypeError, + "got multiple values for keyword argument %s", w_key) + self.space.setitem(w_dict, w_key, w_value) + while num_maps != 0: + self.popvalue() + num_maps -= 1 + self.pushvalue(w_dict) def BUILD_MAP_UNPACK(self, itemcount, next_instr): w_dict = self.space.newdict() From pypy.commits at gmail.com Fri Jul 1 14:00:32 2016 From: pypy.commits at gmail.com (raffael_t) Date: Fri, 01 Jul 2016 11:00:32 -0700 (PDT) Subject: [pypy-commit] pypy py3.5: Show only value of unicodeobject in case of non-empty intersection Message-ID: <5776afc0.a90bc30a.4d501.475c@mx.google.com> Author: Raffael Tfirst Branch: py3.5 Changeset: r85502:53b6c2e3609c Date: 2016-07-01 19:59 +0200 http://bitbucket.org/pypy/pypy/changeset/53b6c2e3609c/ Log: Show only value of unicodeobject in case of non-empty intersection diff --git a/pypy/interpreter/pyopcode.py b/pypy/interpreter/pyopcode.py --- a/pypy/interpreter/pyopcode.py +++ b/pypy/interpreter/pyopcode.py @@ -1363,7 +1363,7 @@ (w_key, w_value) = w_item.popitem() if self.space.is_true(self.space.contains(w_dict,w_key)): raise oefmt(self.space.w_TypeError, - "got multiple values for keyword argument %s", w_key) + "got multiple values for keyword argument %s", self.space.unicode_w(w_key)) self.space.setitem(w_dict, w_key, w_value) while num_maps != 0: self.popvalue() From pypy.commits at gmail.com Fri Jul 1 14:21:00 2016 From: pypy.commits at gmail.com (arigo) Date: Fri, 01 Jul 2016 11:21:00 -0700 (PDT) Subject: [pypy-commit] pypy reverse-debugger: Write down both implementations for cffi callbacks: the old one which Message-ID: <5776b48c.c626c20a.1a063.7345@mx.google.com> Author: Armin Rigo Branch: reverse-debugger Changeset: r85503:59aff804be9c Date: 2016-07-01 20:22 +0200 http://bitbucket.org/pypy/pypy/changeset/59aff804be9c/ Log: Write down both implementations for cffi callbacks: the old one which uses RWeakList, and the new one which uses direct casts of gc pointers to addresses (but doesn't work e.g. with the reverse debugger) diff --git a/pypy/module/_cffi_backend/ccallback.py b/pypy/module/_cffi_backend/ccallback.py --- a/pypy/module/_cffi_backend/ccallback.py +++ b/pypy/module/_cffi_backend/ccallback.py @@ -3,9 +3,9 @@ """ import sys, os, py -from rpython.rlib import clibffi, jit, rgc, objectmodel +from rpython.rlib import clibffi, jit, objectmodel from rpython.rlib.objectmodel import keepalive_until_here -from rpython.rtyper.lltypesystem import lltype, llmemory, rffi +from rpython.rtyper.lltypesystem import lltype, rffi from pypy.interpreter.error import OperationError, oefmt from pypy.module._cffi_backend import cerrno, misc, parse_c_type @@ -13,6 +13,7 @@ from pypy.module._cffi_backend.ctypefunc import SIZE_OF_FFI_ARG, W_CTypeFunc from pypy.module._cffi_backend.ctypeprim import W_CTypePrimitiveSigned from pypy.module._cffi_backend.ctypevoid import W_CTypeVoid +from pypy.module._cffi_backend.hide_reveal import hide_reveal BIG_ENDIAN = sys.byteorder == 'big' @@ -30,9 +31,7 @@ return cdata def reveal_callback(raw_ptr): - addr = rffi.cast(llmemory.Address, raw_ptr) - gcref = rgc.reveal_gcref(addr) - return rgc.try_cast_gcref_to_instance(W_ExternPython, gcref) + return hide_reveal().reveal_object(W_ExternPython, raw_ptr) class Closure(object): @@ -92,9 +91,7 @@ return ctype def hide_object(self): - gcref = rgc.cast_instance_to_gcref(self) - raw = rgc.hide_nonmovable_gcref(gcref) - return rffi.cast(rffi.VOIDP, raw) + return hide_reveal().hide_object(self) def _repr_extra(self): space = self.space diff --git a/pypy/module/_cffi_backend/hide_reveal.py b/pypy/module/_cffi_backend/hide_reveal.py new file mode 100644 --- /dev/null +++ b/pypy/module/_cffi_backend/hide_reveal.py @@ -0,0 +1,54 @@ +from rpython.rlib import rgc +from rpython.rlib.rweaklist import RWeakListMixin +from rpython.rlib.objectmodel import fetch_translated_config +from rpython.rtyper.lltypesystem import lltype, llmemory, rffi + + +class HideRevealRWeakList: + """Slow implementation of HideReveal: uses a RWeakListMixin.""" + + def __init__(self): + class GlobGcrefs(RWeakListMixin): + pass + self.glob_gcrefs = GlobGcrefs() + self.glob_gcrefs.initialize() + + def _freeze_(self): + return True + + def hide_object(self, obj): + # XXX leaks if we call this function often on the same object + index = self.glob_gcrefs.add_handle(obj) + return rffi.cast(llmemory.Address, index) + + def reveal_object(self, Class, addr): + index = rffi.cast(lltype.Signed, addr) + return self.glob_gcrefs.fetch_handle(index) + + +class HideRevealCast: + """Fast implementation of HideReveal: just a cast.""" + + def _freeze_(self): + return True + + def hide_object(self, obj): + gcref = rgc.cast_instance_to_gcref(obj) + raw = rgc.hide_nonmovable_gcref(gcref) + return rffi.cast(rffi.VOIDP, raw) + + def reveal_object(self, Class, raw_ptr): + addr = rffi.cast(llmemory.Address, raw_ptr) + gcref = rgc.reveal_gcref(addr) + return rgc.try_cast_gcref_to_instance(Class, gcref) + + +hide_reveal_slow = HideRevealRWeakList() +hide_reveal_fast = HideRevealCast() + +def hide_reveal(): + config = fetch_translated_config() + if config is not None and config.translation.split_gc_address_space: + return hide_reveal_slow + else: + return hide_reveal_fast From pypy.commits at gmail.com Fri Jul 1 14:43:20 2016 From: pypy.commits at gmail.com (raffael_t) Date: Fri, 01 Jul 2016 11:43:20 -0700 (PDT) Subject: [pypy-commit] pypy py3.5: Only allow dicts in map_unpack_with_call Message-ID: <5776b9c8.aa29c20a.dcdee.53ab@mx.google.com> Author: Raffael Tfirst Branch: py3.5 Changeset: r85504:a8f99affbbfd Date: 2016-07-01 20:42 +0200 http://bitbucket.org/pypy/pypy/changeset/a8f99affbbfd/ Log: Only allow dicts in map_unpack_with_call diff --git a/pypy/interpreter/pyopcode.py b/pypy/interpreter/pyopcode.py --- a/pypy/interpreter/pyopcode.py +++ b/pypy/interpreter/pyopcode.py @@ -1358,6 +1358,9 @@ w_dict = self.space.newdict() for i in range(num_maps, 0, -1): w_item = self.peekvalue(i-1) + if not issubclass(w_item.__class__, self.space.newdict().__class__): + raise oefmt(self.space.w_TypeError, + "%s is not a mapping", w_item.__class__.__name__) num_items = w_item.length() for j in range(num_items): (w_key, w_value) = w_item.popitem() From pypy.commits at gmail.com Fri Jul 1 17:27:43 2016 From: pypy.commits at gmail.com (arigo) Date: Fri, 01 Jul 2016 14:27:43 -0700 (PDT) Subject: [pypy-commit] pypy default: Backed out changeset 850a4e0a56cc Message-ID: <5776e04f.0410c20a.2eebe.ffff88d2@mx.google.com> Author: Armin Rigo Branch: Changeset: r85505:3a1612568705 Date: 2016-07-01 23:29 +0200 http://bitbucket.org/pypy/pypy/changeset/3a1612568705/ Log: Backed out changeset 850a4e0a56cc Seems that we end up with big one-line blocks with '\n' written in the middle diff --git a/rpython/translator/tool/make_dot.py b/rpython/translator/tool/make_dot.py --- a/rpython/translator/tool/make_dot.py +++ b/rpython/translator/tool/make_dot.py @@ -51,7 +51,7 @@ ports=None, ): d = locals() - attrs = [('%s="%s"' % (x, _quote(d[x]))) + attrs = [('%s="%s"' % (x, d[x].replace('"', '\\"').replace('\n', '\\n'))) for x in ['label', 'style', 'color', 'dir', 'weight']] self.emit('edge [%s];' % ", ".join(attrs)) if ports: @@ -69,7 +69,7 @@ width="0.75", ): d = locals() - attrs = [('%s="%s"' % (x, _quote(d[x]))) + attrs = [('%s="%s"' % (x, d[x].replace('"', '\\"').replace('\n', '\\n'))) for x in ['shape', 'label', 'color', 'fillcolor', 'style', 'width']] self.emit('%s [%s];' % (safename(name), ", ".join(attrs))) @@ -193,7 +193,7 @@ name2 = self.blockname(link.target) label = " ".join(map(repr, link.args)) if link.exitcase is not None: - label = "%s: %s" %(_quote(repr(link.exitcase)), label) + label = "%s: %s" %(repr(link.exitcase).replace('\\', '\\\\'), label) self.emit_edge(name, name2, label, style="dotted", color="red") else: self.emit_edge(name, name2, label, style="solid") @@ -237,6 +237,3 @@ # not a keyword name = ''.join([CHAR_MAP[c] for c in name]) return '_' + name - -def _quote(s): - return s.replace('\\', '\\\\').replace('"', '\\"').replace('\n', '\\n') From pypy.commits at gmail.com Fri Jul 1 18:17:57 2016 From: pypy.commits at gmail.com (arigo) Date: Fri, 01 Jul 2016 15:17:57 -0700 (PDT) Subject: [pypy-commit] pypy reverse-debugger: fix (port from stmgc-c8) Message-ID: <5776ec15.2606c20a.439fe.7223@mx.google.com> Author: Armin Rigo Branch: reverse-debugger Changeset: r85507:2c199ada1854 Date: 2016-07-01 23:29 +0200 http://bitbucket.org/pypy/pypy/changeset/2c199ada1854/ Log: fix (port from stmgc-c8) diff --git a/rpython/rlib/rrawarray.py b/rpython/rlib/rrawarray.py --- a/rpython/rlib/rrawarray.py +++ b/rpython/rlib/rrawarray.py @@ -1,6 +1,6 @@ from rpython.rtyper.llannotation import lltype_to_annotation from rpython.rlib.objectmodel import specialize -from rpython.rlib import jit +from rpython.rlib import jit, rgc from rpython.rtyper.lltypesystem import lltype, llmemory from rpython.rtyper.extregistry import ExtRegistryEntry from rpython.tool.pairtype import pair @@ -50,6 +50,10 @@ @jit.dont_look_inside def ll_copy_list_to_raw_array(ll_list, dst_ptr): + if rgc.must_split_gc_address_space(): + for i in range(ll_list.ll_length()): + dst_ptr[i] = ll_list.ll_getitem_fast(i) + return # this code is delicate: we must ensure that there are no GC operations # around the call to raw_memcopy # @@ -64,9 +68,13 @@ @jit.dont_look_inside def ll_populate_list_from_raw_array(ll_list, src_ptr, length): + ll_list._ll_resize(length) + if rgc.must_split_gc_address_space(): + for i in range(length): + ll_list.ll_setitem_fast(i, src_ptr[i]) + return ITEM = lltype.typeOf(src_ptr).TO.OF size = llmemory.sizeof(ITEM) * length - ll_list._ll_resize(length) # start of no-GC section src_adr = get_raw_buf(src_ptr) dst_adr = get_raw_buf(ll_list.ll_items()) From pypy.commits at gmail.com Fri Jul 1 18:17:59 2016 From: pypy.commits at gmail.com (arigo) Date: Fri, 01 Jul 2016 15:17:59 -0700 (PDT) Subject: [pypy-commit] pypy reverse-debugger: hg merge default Message-ID: <5776ec17.0eb81c0a.7b088.0848@mx.google.com> Author: Armin Rigo Branch: reverse-debugger Changeset: r85508:496ddf1997fe Date: 2016-07-01 23:33 +0200 http://bitbucket.org/pypy/pypy/changeset/496ddf1997fe/ Log: hg merge default diff --git a/rpython/translator/tool/make_dot.py b/rpython/translator/tool/make_dot.py --- a/rpython/translator/tool/make_dot.py +++ b/rpython/translator/tool/make_dot.py @@ -51,7 +51,7 @@ ports=None, ): d = locals() - attrs = [('%s="%s"' % (x, _quote(d[x]))) + attrs = [('%s="%s"' % (x, d[x].replace('"', '\\"').replace('\n', '\\n'))) for x in ['label', 'style', 'color', 'dir', 'weight']] self.emit('edge [%s];' % ", ".join(attrs)) if ports: @@ -69,7 +69,7 @@ width="0.75", ): d = locals() - attrs = [('%s="%s"' % (x, _quote(d[x]))) + attrs = [('%s="%s"' % (x, d[x].replace('"', '\\"').replace('\n', '\\n'))) for x in ['shape', 'label', 'color', 'fillcolor', 'style', 'width']] self.emit('%s [%s];' % (safename(name), ", ".join(attrs))) @@ -193,7 +193,7 @@ name2 = self.blockname(link.target) label = " ".join(map(repr, link.args)) if link.exitcase is not None: - label = "%s: %s" %(_quote(repr(link.exitcase)), label) + label = "%s: %s" %(repr(link.exitcase).replace('\\', '\\\\'), label) self.emit_edge(name, name2, label, style="dotted", color="red") else: self.emit_edge(name, name2, label, style="solid") @@ -237,6 +237,3 @@ # not a keyword name = ''.join([CHAR_MAP[c] for c in name]) return '_' + name - -def _quote(s): - return s.replace('\\', '\\\\').replace('"', '\\"').replace('\n', '\\n') From pypy.commits at gmail.com Fri Jul 1 18:18:01 2016 From: pypy.commits at gmail.com (arigo) Date: Fri, 01 Jul 2016 15:18:01 -0700 (PDT) Subject: [pypy-commit] pypy reverse-debugger: fix for tests Message-ID: <5776ec19.081ac20a.b39c.ffffb6bb@mx.google.com> Author: Armin Rigo Branch: reverse-debugger Changeset: r85509:5bcd3e5d3eda Date: 2016-07-01 23:34 +0200 http://bitbucket.org/pypy/pypy/changeset/5bcd3e5d3eda/ Log: fix for tests diff --git a/rpython/memory/gctransform/boehm.py b/rpython/memory/gctransform/boehm.py --- a/rpython/memory/gctransform/boehm.py +++ b/rpython/memory/gctransform/boehm.py @@ -48,7 +48,7 @@ self.WEAKLINK, self.convert_weakref_to ) = build_weakref(self.translator.config) self.weakref_create_ptr = self.inittime_helper( - ll_weakref_create, [llmemory.Address], llmemory.WeakRefPtr, + ll_weakref_create, [llmemory.GCREF], llmemory.WeakRefPtr, inline=False) self.weakref_deref_ptr = self.inittime_helper( ll_weakref_deref, [llmemory.WeakRefPtr], llmemory.GCREF) From pypy.commits at gmail.com Fri Jul 1 18:17:55 2016 From: pypy.commits at gmail.com (arigo) Date: Fri, 01 Jul 2016 15:17:55 -0700 (PDT) Subject: [pypy-commit] pypy reverse-debugger: fixes Message-ID: <5776ec13.c5461c0a.cee39.2539@mx.google.com> Author: Armin Rigo Branch: reverse-debugger Changeset: r85506:e5e2c4c6d62f Date: 2016-07-01 23:29 +0200 http://bitbucket.org/pypy/pypy/changeset/e5e2c4c6d62f/ Log: fixes diff --git a/pypy/module/_cffi_backend/hide_reveal.py b/pypy/module/_cffi_backend/hide_reveal.py --- a/pypy/module/_cffi_backend/hide_reveal.py +++ b/pypy/module/_cffi_backend/hide_reveal.py @@ -1,6 +1,5 @@ from rpython.rlib import rgc from rpython.rlib.rweaklist import RWeakListMixin -from rpython.rlib.objectmodel import fetch_translated_config from rpython.rtyper.lltypesystem import lltype, llmemory, rffi @@ -19,7 +18,7 @@ def hide_object(self, obj): # XXX leaks if we call this function often on the same object index = self.glob_gcrefs.add_handle(obj) - return rffi.cast(llmemory.Address, index) + return rffi.cast(rffi.VOIDP, index) def reveal_object(self, Class, addr): index = rffi.cast(lltype.Signed, addr) @@ -47,8 +46,7 @@ hide_reveal_fast = HideRevealCast() def hide_reveal(): - config = fetch_translated_config() - if config is not None and config.translation.split_gc_address_space: + if rgc.must_split_gc_address_space(): return hide_reveal_slow else: return hide_reveal_fast From pypy.commits at gmail.com Fri Jul 1 18:18:03 2016 From: pypy.commits at gmail.com (arigo) Date: Fri, 01 Jul 2016 15:18:03 -0700 (PDT) Subject: [pypy-commit] pypy reverse-debugger: A second place with hide/reveal Message-ID: <5776ec1b.48371c0a.8871a.7944@mx.google.com> Author: Armin Rigo Branch: reverse-debugger Changeset: r85510:0b7b98628f9f Date: 2016-07-02 00:19 +0200 http://bitbucket.org/pypy/pypy/changeset/0b7b98628f9f/ Log: A second place with hide/reveal diff --git a/pypy/module/_cffi_backend/ccallback.py b/pypy/module/_cffi_backend/ccallback.py --- a/pypy/module/_cffi_backend/ccallback.py +++ b/pypy/module/_cffi_backend/ccallback.py @@ -13,7 +13,7 @@ from pypy.module._cffi_backend.ctypefunc import SIZE_OF_FFI_ARG, W_CTypeFunc from pypy.module._cffi_backend.ctypeprim import W_CTypePrimitiveSigned from pypy.module._cffi_backend.ctypevoid import W_CTypeVoid -from pypy.module._cffi_backend.hide_reveal import hide_reveal +from pypy.module._cffi_backend.hide_reveal import hide_reveal1 BIG_ENDIAN = sys.byteorder == 'big' @@ -31,7 +31,7 @@ return cdata def reveal_callback(raw_ptr): - return hide_reveal().reveal_object(W_ExternPython, raw_ptr) + return hide_reveal1().reveal_object(W_ExternPython, raw_ptr) class Closure(object): @@ -91,7 +91,7 @@ return ctype def hide_object(self): - return hide_reveal().hide_object(self) + return hide_reveal1().hide_object(rffi.VOIDP, self) def _repr_extra(self): space = self.space diff --git a/pypy/module/_cffi_backend/handle.py b/pypy/module/_cffi_backend/handle.py --- a/pypy/module/_cffi_backend/handle.py +++ b/pypy/module/_cffi_backend/handle.py @@ -3,8 +3,9 @@ from pypy.interpreter.gateway import unwrap_spec from pypy.interpreter.baseobjspace import W_Root from pypy.module._cffi_backend import ctypeobj, ctypeptr, cdataobj +from pypy.module._cffi_backend.hide_reveal import hide_reveal2 from rpython.rtyper.lltypesystem import lltype, llmemory, rffi -from rpython.rlib import rgc, objectmodel, jit +from rpython.rlib import objectmodel, jit # ____________________________________________________________ @@ -15,9 +16,7 @@ # we can cast the CCHARP back to a W_CDataHandle with reveal_gcref(). new_cdataobj = objectmodel.instantiate(cdataobj.W_CDataHandle, nonmovable=True) - gcref = rgc.cast_instance_to_gcref(new_cdataobj) - _cdata = rgc.hide_nonmovable_gcref(gcref) - _cdata = rffi.cast(rffi.CCHARP, _cdata) + _cdata = hide_reveal2().hide_object(rffi.CCHARP, new_cdataobj) cdataobj.W_CDataHandle.__init__(new_cdataobj, space, _cdata, w_ctype, w_x) return new_cdataobj @@ -43,11 +42,10 @@ @jit.dont_look_inside def _reveal(space, ptr): addr = rffi.cast(llmemory.Address, ptr) - gcref = rgc.reveal_gcref(addr) - if not gcref: + if not addr: raise oefmt(space.w_RuntimeError, "cannot use from_handle() on NULL pointer") - cd = rgc.try_cast_gcref_to_instance(cdataobj.W_CDataHandle, gcref) + cd = hide_reveal2().reveal_object(cdataobj.W_CDataHandle, addr) if cd is None: raise oefmt(space.w_SystemError, "ffi.from_handle(): dead or bogus object handle") diff --git a/pypy/module/_cffi_backend/hide_reveal.py b/pypy/module/_cffi_backend/hide_reveal.py --- a/pypy/module/_cffi_backend/hide_reveal.py +++ b/pypy/module/_cffi_backend/hide_reveal.py @@ -9,44 +9,58 @@ def __init__(self): class GlobGcrefs(RWeakListMixin): pass - self.glob_gcrefs = GlobGcrefs() - self.glob_gcrefs.initialize() + glob_gcrefs = GlobGcrefs() + glob_gcrefs.initialize() + + def hide_object(PTR, obj): + # XXX leaks if we call this function often on the same object + index = glob_gcrefs.add_handle(obj) + return rffi.cast(PTR, index + 1) + + def reveal_object(Class, addr): + index = rffi.cast(lltype.Signed, addr) - 1 + return glob_gcrefs.fetch_handle(index) + + self.hide_object = hide_object + self.reveal_object = reveal_object def _freeze_(self): return True - def hide_object(self, obj): - # XXX leaks if we call this function often on the same object - index = self.glob_gcrefs.add_handle(obj) - return rffi.cast(rffi.VOIDP, index) - - def reveal_object(self, Class, addr): - index = rffi.cast(lltype.Signed, addr) - return self.glob_gcrefs.fetch_handle(index) - class HideRevealCast: """Fast implementation of HideReveal: just a cast.""" + def __init__(self): + + def hide_object(PTR, obj): + gcref = rgc.cast_instance_to_gcref(obj) + raw = rgc.hide_nonmovable_gcref(gcref) + return rffi.cast(PTR, raw) + + def reveal_object(Class, raw_ptr): + addr = rffi.cast(llmemory.Address, raw_ptr) + gcref = rgc.reveal_gcref(addr) + return rgc.try_cast_gcref_to_instance(Class, gcref) + + self.hide_object = hide_object + self.reveal_object = reveal_object + def _freeze_(self): return True - def hide_object(self, obj): - gcref = rgc.cast_instance_to_gcref(obj) - raw = rgc.hide_nonmovable_gcref(gcref) - return rffi.cast(rffi.VOIDP, raw) - def reveal_object(self, Class, raw_ptr): - addr = rffi.cast(llmemory.Address, raw_ptr) - gcref = rgc.reveal_gcref(addr) - return rgc.try_cast_gcref_to_instance(Class, gcref) +def make_hide_reveal(): + hide_reveal_slow = HideRevealRWeakList() + hide_reveal_fast = HideRevealCast() + def hide_reveal(): + if rgc.must_split_gc_address_space(): + return hide_reveal_slow + else: + return hide_reveal_fast -hide_reveal_slow = HideRevealRWeakList() -hide_reveal_fast = HideRevealCast() + return hide_reveal -def hide_reveal(): - if rgc.must_split_gc_address_space(): - return hide_reveal_slow - else: - return hide_reveal_fast +hide_reveal1 = make_hide_reveal() # for ccallback.py +hide_reveal2 = make_hide_reveal() # for handles.py From pypy.commits at gmail.com Sat Jul 2 03:07:55 2016 From: pypy.commits at gmail.com (arigo) Date: Sat, 02 Jul 2016 00:07:55 -0700 (PDT) Subject: [pypy-commit] pypy reverse-debugger: Disable some modules automatically if we translate pypy with --revdb Message-ID: <5777684b.a758c20a.6664d.72cf@mx.google.com> Author: Armin Rigo Branch: reverse-debugger Changeset: r85511:ec9b40e3781d Date: 2016-07-02 09:09 +0200 http://bitbucket.org/pypy/pypy/changeset/ec9b40e3781d/ Log: Disable some modules automatically if we translate pypy with --revdb diff --git a/pypy/config/pypyoption.py b/pypy/config/pypyoption.py --- a/pypy/config/pypyoption.py +++ b/pypy/config/pypyoption.py @@ -55,6 +55,10 @@ "termios", "_minimal_curses", ]) +reverse_debugger_disable_modules = set([ + "thread", "_continuation", "_vmprof", "_multiprocessing", + ]) + # XXX this should move somewhere else, maybe to platform ("is this posixish" # check or something) if sys.platform == "win32": @@ -279,6 +283,8 @@ modules = working_modules.copy() if config.translation.sandbox: modules = default_modules + if config.translation.reverse_debugger: + modules -= reverse_debugger_disable_modules # ignore names from 'essential_modules', notably 'exceptions', which # may not be present in config.objspace.usemodules at all modules = [name for name in modules if name not in essential_modules] From pypy.commits at gmail.com Sat Jul 2 03:25:42 2016 From: pypy.commits at gmail.com (arigo) Date: Sat, 02 Jul 2016 00:25:42 -0700 (PDT) Subject: [pypy-commit] pypy reverse-debugger: Test and fix: fork() creates children which also continue to write to Message-ID: <57776c76.56311c0a.94406.79cb@mx.google.com> Author: Armin Rigo Branch: reverse-debugger Changeset: r85512:d23597283e75 Date: 2016-07-02 09:27 +0200 http://bitbucket.org/pypy/pypy/changeset/d23597283e75/ Log: Test and fix: fork() creates children which also continue to write to the same log file, corrupting it diff --git a/rpython/translator/revdb/src-revdb/revdb.c b/rpython/translator/revdb/src-revdb/revdb.c --- a/rpython/translator/revdb/src-revdb/revdb.c +++ b/rpython/translator/revdb/src-revdb/revdb.c @@ -127,6 +127,14 @@ } } +static void close_revdb_fileno_in_fork_child(void) +{ + if (rpy_rev_fileno >= 0) { + close(rpy_rev_fileno); + rpy_rev_fileno = -1; + } +} + static void setup_record_mode(int argc, char *argv[]) { char *filename = getenv("PYPYRDB"); @@ -169,6 +177,8 @@ rpy_revdb.buf_p = rpy_rev_buffer + sizeof(int16_t); rpy_revdb.buf_limit = rpy_rev_buffer + sizeof(rpy_rev_buffer) - 32; rpy_revdb.unique_id_seen = 1; + + pthread_atfork(NULL, NULL, close_revdb_fileno_in_fork_child); } static void flush_buffer(void) @@ -221,10 +231,11 @@ { /* ===== FINALIZERS ===== - When the GC wants to invoke some finalizers, it causes this - to be called at the stop point. The new-style finalizers - are only enqueued at this point. The old-style finalizers - run immediately, conceptually just *after* the stop point. + When the GC wants to invoke some finalizers, it causes this to + be called at the stop point. (This is not called at *every* + stop point.) The new-style finalizers are only enqueued at + this point. The old-style finalizers run immediately, + conceptually just *after* the stop point. */ int i; char *p = rpy_rev_buffer; diff --git a/rpython/translator/revdb/test/test_basic.py b/rpython/translator/revdb/test/test_basic.py --- a/rpython/translator/revdb/test/test_basic.py +++ b/rpython/translator/revdb/test/test_basic.py @@ -369,6 +369,10 @@ dbstate.stuff.x = i + 1000 revdb.stop_point() print op + if i == 1: + if os.fork() == 0: # child + os.write(2, "this line is from the fork child.\n") + return 0 return 9 compile(cls, main, backendopt=False) assert run(cls, 'abc d ef') == 'abc\nd\nef\n' From pypy.commits at gmail.com Sat Jul 2 04:26:12 2016 From: pypy.commits at gmail.com (arigo) Date: Sat, 02 Jul 2016 01:26:12 -0700 (PDT) Subject: [pypy-commit] pypy reverse-debugger: Debugging helper Message-ID: <57777aa4.89acc20a.6baf0.ffff97c2@mx.google.com> Author: Armin Rigo Branch: reverse-debugger Changeset: r85513:6b5aaea0d7f0 Date: 2016-07-02 10:27 +0200 http://bitbucket.org/pypy/pypy/changeset/6b5aaea0d7f0/ Log: Debugging helper diff --git a/rpython/translator/revdb/pplog.py b/rpython/translator/revdb/pplog.py new file mode 100755 --- /dev/null +++ b/rpython/translator/revdb/pplog.py @@ -0,0 +1,36 @@ +#!/usr/bin/env python2 + +# Post-process log files to make them diff-able. +# +# When you get errors caused by revdb.py getting out of sync with the +# original log file, recompile the program (e.g. pypy-c) by editing +# revdb_include.h, enabling the "#if 0" (at least the first one, +# possibly the second one too). This prints locations to stderr of +# all the EMITs. Then create the log file by redirecting stderr to +# "log.err1", and then run revdb.py by redirecting stderr to +# "log.err2" (typically, entering the "c" command to continue to the +# end). Then diff them both after applying this filter: +# +# diff -u <(cat log.err1 | .../rpython/translator/revdb/pplog.py) \ +# <(cat log.err2 | .../rpython/translator/revdb/pplog.py) | less + + +import sys, re + +r_hide_tail = re.compile(r"revdb[.]c:\d+: ([0-9a-f]+)") + +r_remove = re.compile("\w+[.]c:\d+: obj 92233720368") + + +def post_process(fin, fout): + for line in fin: + match = r_hide_tail.match(line) + if match: + line = 'revdb.c:after pplog.py: %s\n' % ('#' * len(match.group(1)),) + elif r_remove.match(line): + continue + fout.write(line) + + +if __name__ == '__main__': + post_process(sys.stdin, sys.stdout) diff --git a/rpython/translator/revdb/revdb.py b/rpython/translator/revdb/revdb.py --- a/rpython/translator/revdb/revdb.py +++ b/rpython/translator/revdb/revdb.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python2 import sys, os diff --git a/rpython/translator/revdb/src-revdb/revdb.c b/rpython/translator/revdb/src-revdb/revdb.c --- a/rpython/translator/revdb/src-revdb/revdb.c +++ b/rpython/translator/revdb/src-revdb/revdb.c @@ -1326,6 +1326,4 @@ RPY_EXTERN void seeing_uid(uint64_t uid) { - if (uid == 1895569) - attach_gdb(); } diff --git a/rpython/translator/revdb/src-revdb/revdb_include.h b/rpython/translator/revdb/src-revdb/revdb_include.h --- a/rpython/translator/revdb/src-revdb/revdb_include.h +++ b/rpython/translator/revdb/src-revdb/revdb_include.h @@ -29,7 +29,7 @@ RPY_EXTERN void rpy_reverse_db_setup(int *argc_p, char **argv_p[]); RPY_EXTERN void rpy_reverse_db_teardown(void); -#if 0 /* enable to print locations to stderr of all the EMITs */ +#if 1 /* enable to print locations to stderr of all the EMITs */ # define _RPY_REVDB_PRINT(mode) \ fprintf(stderr, \ "%s:%d: %0*llx\n", \ @@ -37,7 +37,8 @@ ((unsigned long long)_e) & ((2ULL << (8*sizeof(_e)-1)) - 1)) #endif -#if 0 /* enable to print all allocs to stderr */ +#if 1 /* enable to print all allocs to stderr */ +RPY_EXTERN void seeing_uid(uint64_t uid); # define _RPY_REVDB_PRUID() \ seeing_uid(uid); \ fprintf(stderr, \ From pypy.commits at gmail.com Sat Jul 2 05:24:27 2016 From: pypy.commits at gmail.com (raffael_t) Date: Sat, 02 Jul 2016 02:24:27 -0700 (PDT) Subject: [pypy-commit] pypy py3.5: Type check also in map_unpack Message-ID: <5777884b.06a81c0a.5c9df.7c46@mx.google.com> Author: Raffael Tfirst Branch: py3.5 Changeset: r85514:07a2694317a1 Date: 2016-07-02 11:23 +0200 http://bitbucket.org/pypy/pypy/changeset/07a2694317a1/ Log: Type check also in map_unpack diff --git a/pypy/interpreter/pyopcode.py b/pypy/interpreter/pyopcode.py --- a/pypy/interpreter/pyopcode.py +++ b/pypy/interpreter/pyopcode.py @@ -1377,6 +1377,9 @@ w_dict = self.space.newdict() for i in range(itemcount, 0, -1): w_item = self.peekvalue(i-1) + if not issubclass(w_item.__class__, self.space.newdict().__class__): + raise oefmt(self.space.w_TypeError, + "%s is not a mapping", w_item.__class__.__name__) num_items = w_item.length() for j in range(num_items): (w_key, w_value) = w_item.popitem() From pypy.commits at gmail.com Sat Jul 2 10:30:47 2016 From: pypy.commits at gmail.com (arigo) Date: Sat, 02 Jul 2016 07:30:47 -0700 (PDT) Subject: [pypy-commit] pypy default: Remove these lines, they appear twice in the file Message-ID: <5777d017.69fac20a.cbe44.7858@mx.google.com> Author: Armin Rigo Branch: Changeset: r85515:13d2148e1008 Date: 2016-07-02 15:45 +0200 http://bitbucket.org/pypy/pypy/changeset/13d2148e1008/ Log: Remove these lines, they appear twice in the file diff --git a/rpython/rlib/ropenssl.py b/rpython/rlib/ropenssl.py --- a/rpython/rlib/ropenssl.py +++ b/rpython/rlib/ropenssl.py @@ -586,11 +586,6 @@ HASH_MALLOC_SIZE = EVP_MD_SIZE + EVP_MD_CTX_SIZE \ + rffi.sizeof(EVP_MD) * 2 + 208 -OBJ_NAME_CALLBACK = lltype.Ptr(lltype.FuncType( - [OBJ_NAME, rffi.VOIDP], lltype.Void)) -OBJ_NAME_do_all = external( - 'OBJ_NAME_do_all', [rffi.INT, OBJ_NAME_CALLBACK, rffi.VOIDP], lltype.Void) - def init_ssl(): libssl_SSL_load_error_strings() libssl_SSL_library_init() From pypy.commits at gmail.com Sat Jul 2 14:17:09 2016 From: pypy.commits at gmail.com (mattip) Date: Sat, 02 Jul 2016 11:17:09 -0700 (PDT) Subject: [pypy-commit] pypy call-via-pyobj: fixes for translation, but not done yet Message-ID: <57780525.a2d1c20a.65503.fffffd1d@mx.google.com> Author: Matti Picus Branch: call-via-pyobj Changeset: r85516:349eb5ca6183 Date: 2016-07-02 21:15 +0300 http://bitbucket.org/pypy/pypy/changeset/349eb5ca6183/ Log: fixes for translation, but not done yet diff --git a/pypy/module/cpyext/methodobject.py b/pypy/module/cpyext/methodobject.py --- a/pypy/module/cpyext/methodobject.py +++ b/pypy/module/cpyext/methodobject.py @@ -174,16 +174,25 @@ raise oefmt(space.w_TypeError, "wrapper %s doesn't take any keyword arguments", self.method_name) + func_to_call = self.func if self.offset: - ptr = pto = rffi.cast(PyTypeObjectPtr, as_pyobj(space, self.w_objclass)) + ptr = as_pyobj(space, self.w_objclass) # make ptr the equivalent of this, using the offsets - #func_to_call = rffi.cast(rffi.VOIDP, pto.c_tp_as_number.c_nb_multiply) - for o in self.offset: - ptr_as_int = lltype.cast_ptr_to_int(ptr) - ptr = rffi.cast(rffi.VOIDPP, ptr_as_int + o)[0] - func_to_call = ptr - else: - func_to_call = self.func + #func_to_call = rffi.cast(rffi.VOIDP, ptr.c_tp_as_number.c_nb_multiply) + if not ptr is rffi.VOIDP: + ptr = rffi.cast(rffi.VOIDP, ptr) + for o in self.offset: + if not ptr: + break + ptr_as_int = lltype.cast_ptr_to_int(ptr) + pptr = rffi.cast(rffi.VOIDPP, ptr_as_int + o) + if not pptr: + break + ptr = pptr[0] + ptr = rffi.cast(rffi.VOIDP, ptr) + if ptr: + func_to_call = ptr + assert func_to_call return self.wrapper_func(space, w_self, w_args, func_to_call) def descr_method_repr(self): diff --git a/pypy/module/cpyext/typeobject.py b/pypy/module/cpyext/typeobject.py --- a/pypy/module/cpyext/typeobject.py +++ b/pypy/module/cpyext/typeobject.py @@ -296,7 +296,7 @@ for method_name, slot_names, wrapper_func, wrapper_func_kwds, doc in slotdefs_for_wrappers: if method_name in dict_w: continue - offset = [rffi.offsetof(pto._T, slot_names[0])] + offset = [rffi.offsetof(lltype.typeOf(pto).TO, slot_names[0])] if len(slot_names) == 1: func = getattr(pto, slot_names[0]) else: @@ -304,8 +304,7 @@ struct = getattr(pto, slot_names[0]) if not struct: continue - assert isinstance(struct, lltype._ptr) - offset.append(rffi.offsetof(struct._T, slot_names[1])) + offset.append(rffi.offsetof(lltype.typeOf(struct).TO, slot_names[1])) func = getattr(struct, slot_names[1]) func_voidp = rffi.cast(rffi.VOIDP, func) if not func: From pypy.commits at gmail.com Sat Jul 2 14:37:08 2016 From: pypy.commits at gmail.com (arigo) Date: Sat, 02 Jul 2016 11:37:08 -0700 (PDT) Subject: [pypy-commit] pypy reverse-debugger: Callbacks. Message-ID: <577809d4.a90bc30a.68325.734d@mx.google.com> Author: Armin Rigo Branch: reverse-debugger Changeset: r85517:00fce40d3026 Date: 2016-07-02 20:38 +0200 http://bitbucket.org/pypy/pypy/changeset/00fce40d3026/ Log: Callbacks. diff --git a/rpython/translator/c/funcgen.py b/rpython/translator/c/funcgen.py --- a/rpython/translator/c/funcgen.py +++ b/rpython/translator/c/funcgen.py @@ -172,6 +172,13 @@ # ____________________________________________________________ def cfunction_body(self): + extra_return_text = None + if self.db.reverse_debugger: + from rpython.translator.revdb import gencsupp + (extra_enter_text, extra_return_text) = ( + gencsupp.prepare_function(self)) + if extra_enter_text: + yield extra_enter_text graph = self.graph yield 'goto block0;' # to avoid a warning "this label is not used" @@ -193,6 +200,8 @@ retval = self.expr(block.inputargs[0]) if self.exception_policy != "exc_helper": yield 'RPY_DEBUG_RETURN();' + if extra_return_text: + yield extra_return_text yield 'return %s;' % retval continue elif block.exitswitch is None: @@ -405,13 +414,8 @@ line += '\nPYPY_INHIBIT_TAIL_CALL();' break elif self.db.reverse_debugger: - if getattr(getattr(self.graph, 'func', None), - '_revdb_do_all_calls_', False): - pass # a hack for ll_call_destructor() to mean - # that the calls should really be done - else: - from rpython.translator.revdb import gencsupp - line = gencsupp.emit(line, self.lltypename(v_result), r) + from rpython.translator.revdb import gencsupp + line = gencsupp.emit_residual_call(self, line, v_result, r) return line def OP_DIRECT_CALL(self, op): diff --git a/rpython/translator/c/genc.py b/rpython/translator/c/genc.py --- a/rpython/translator/c/genc.py +++ b/rpython/translator/c/genc.py @@ -735,6 +735,8 @@ print >> fc, '#include "preimpl.h"' print >> fc, '#define PYPY_FILE_NAME "%s"' % name print >> fc, '#include "src/g_include.h"' + if self.database.reverse_debugger: + print >> fc, '#include "revdb_def.h"' print >> fc print >> fc, MARKER for node, impl in nodeiter: @@ -904,6 +906,9 @@ n = database.instrument_ncounter print >>fi, "#define PYPY_INSTRUMENT_NCOUNTER %d" % n fi.close() + if database.reverse_debugger: + from rpython.translator.revdb import gencsupp + gencsupp.write_revdb_def_file(database, targetdir.join('revdb_def.h')) eci = add_extra_files(database, eci) eci = eci.convert_sources_to_files() diff --git a/rpython/translator/revdb/gencsupp.py b/rpython/translator/revdb/gencsupp.py --- a/rpython/translator/revdb/gencsupp.py +++ b/rpython/translator/revdb/gencsupp.py @@ -11,6 +11,24 @@ srcdir / 'revdb.c', ] +def prepare_function(funcgen): + stack_bottom = False + for block in funcgen.graph.iterblocks(): + for op in block.operations: + if op.opname == 'gc_stack_bottom': + stack_bottom = True + if stack_bottom: + name = funcgen.functionname + funcgen.db.stack_bottom_funcnames.append(name) + extra_enter_text = '\n'.join( + ['RPY_REVDB_CALLBACKLOC(RPY_CALLBACKLOC_%s);' % name] + + ['\t' + emit('/*arg*/', funcgen.lltypename(v), funcgen.expr(v)) + for v in funcgen.graph.getargs()]) + extra_return_text = '/* RPY_CALLBACK_LEAVE(); */' + return extra_enter_text, extra_return_text + else: + return None, None + def emit_void(normal_code): return 'RPY_REVDB_EMIT_VOID(%s);' % (normal_code,) @@ -19,6 +37,21 @@ return emit_void(normal_code) return 'RPY_REVDB_EMIT(%s, %s, %s);' % (normal_code, cdecl(tp, '_e'), value) +def emit_residual_call(funcgen, call_code, v_result, expr_result): + if getattr(getattr(funcgen.graph, 'func', None), + '_revdb_do_all_calls_', False): + return call_code # a hack for ll_call_destructor() to mean + # that the calls should really be done + # haaaaack + if call_code in ('RPyGilRelease();', 'RPyGilAcquire();'): + return '/* ' + call_code + ' */' + # + tp = funcgen.lltypename(v_result) + if tp == 'void @': + return 'RPY_REVDB_CALL_VOID(%s);' % (call_code,) + return 'RPY_REVDB_CALL(%s, %s, %s);' % (call_code, cdecl(tp, '_e'), + expr_result) + def record_malloc_uid(expr): return ' RPY_REVDB_REC_UID(%s);' % (expr,) @@ -67,3 +100,21 @@ exports.EXPORTS_obj2name[s._as_obj()] = 'rpy_revdb_commands' db.get(s) + + db.stack_bottom_funcnames = [] + +def write_revdb_def_file(db, target_path): + funcnames = sorted(db.stack_bottom_funcnames) + f = target_path.open('w') + for i, fn in enumerate(funcnames): + print >> f, '#define RPY_CALLBACKLOC_%s %d' % (fn, i) + print >> f + print >> f, '#define RPY_CALLBACKLOCS \\' + funcnames = funcnames or ['NULL'] + for i, fn in enumerate(funcnames): + if i == len(funcnames) - 1: + tail = '' + else: + tail = ', \\' + print >> f, '\t(void *)%s%s' % (fn, tail) + f.close() diff --git a/rpython/translator/revdb/src-revdb/revdb.c b/rpython/translator/revdb/src-revdb/revdb.c --- a/rpython/translator/revdb/src-revdb/revdb.c +++ b/rpython/translator/revdb/src-revdb/revdb.c @@ -16,6 +16,7 @@ #include "structdef.h" #include "forwarddecl.h" #include "preimpl.h" +#include "revdb_def.h" #include "src/rtyper.h" #include "src/mem.h" #include "src-revdb/revdb_include.h" @@ -460,6 +461,20 @@ return result; } +RPY_EXTERN +void rpy_reverse_db_callback_loc(int locnum) +{ + union { + unsigned char n[2]; + uint16_t u; + } r; + assert(locnum < 0xFC00); + if (!RPY_RDB_REPLAY) { + RPY_REVDB_EMIT(r.n[0] = locnum >> 8; r.n[1] = locnum & 0xFF;, + uint16_t _e, r.u); + } +} + /* ------------------------------------------------------------ */ /* Replaying mode */ @@ -862,8 +877,8 @@ char dummy[1]; if (rpy_revdb.buf_p != rpy_revdb.buf_limit - 1 || read(rpy_rev_fileno, dummy, 1) > 0) { - fprintf(stderr, "RevDB file error: corrupted file (too much data or, " - "more likely, non-deterministic run, e.g. a " + fprintf(stderr, "RevDB file error: too much data: corrupted file, " + "revdb bug, or non-deterministic run, e.g. a " "watchpoint with side effects)\n"); exit(1); } @@ -1320,6 +1335,35 @@ fq_trigger(); } +static void *callbacklocs[] = { + RPY_CALLBACKLOCS /* macro from revdb_def.h */ +}; + +RPY_EXTERN +void rpy_reverse_db_invoke_callback(unsigned char e) +{ + /* Replaying: we have read the byte which follows calls, expecting + to see 0xFC, but we saw something else. It's part of a two-bytes + callback identifier. */ + + do { + unsigned long index; + unsigned char e2; + void (*pfn)(void); + _RPY_REVDB_EMIT_REPLAY(unsigned char _e, e2) + index = (e << 8) | e2; + if (index >= (sizeof(callbacklocs) / sizeof(callbacklocs[0]))) { + fprintf(stderr, "bad callback index\n"); + exit(1); + } + pfn = callbacklocs[index]; + pfn(); + + _RPY_REVDB_EMIT_REPLAY(unsigned char _e, e) + } while (e != 0xFC); +} + + /* ------------------------------------------------------------ */ diff --git a/rpython/translator/revdb/src-revdb/revdb_include.h b/rpython/translator/revdb/src-revdb/revdb_include.h --- a/rpython/translator/revdb/src-revdb/revdb_include.h +++ b/rpython/translator/revdb/src-revdb/revdb_include.h @@ -29,7 +29,7 @@ RPY_EXTERN void rpy_reverse_db_setup(int *argc_p, char **argv_p[]); RPY_EXTERN void rpy_reverse_db_teardown(void); -#if 1 /* enable to print locations to stderr of all the EMITs */ +#if 0 /* enable to print locations to stderr of all the EMITs */ # define _RPY_REVDB_PRINT(mode) \ fprintf(stderr, \ "%s:%d: %0*llx\n", \ @@ -37,7 +37,7 @@ ((unsigned long long)_e) & ((2ULL << (8*sizeof(_e)-1)) - 1)) #endif -#if 1 /* enable to print all allocs to stderr */ +#if 0 /* enable to print all mallocs to stderr */ RPY_EXTERN void seeing_uid(uint64_t uid); # define _RPY_REVDB_PRUID() \ seeing_uid(uid); \ @@ -54,17 +54,27 @@ #endif -#define RPY_REVDB_EMIT(normal_code, decl_e, variable) \ - if (!RPY_RDB_REPLAY) { \ - normal_code \ +#define _RPY_REVDB_EMIT_RECORD(decl_e, variable) \ { \ decl_e = variable; \ _RPY_REVDB_PRINT("write"); \ memcpy(rpy_revdb.buf_p, &_e, sizeof(_e)); \ if ((rpy_revdb.buf_p += sizeof(_e)) > rpy_revdb.buf_limit) \ rpy_reverse_db_flush(); \ - } \ - } else { \ + } + +#define _RPY_REVDB_EMIT_RECORD_EXTRA(extra, decl_e, variable) \ + { \ + decl_e = variable; \ + _RPY_REVDB_PRINT("write"); \ + rpy_revdb.buf_p[0] = extra; \ + memcpy(rpy_revdb.buf_p + 1, &_e, sizeof(_e)); \ + if ((rpy_revdb.buf_p += 1 + sizeof(_e)) > rpy_revdb.buf_limit) \ + rpy_reverse_db_flush(); \ + } + +#define _RPY_REVDB_EMIT_REPLAY(decl_e, variable) \ + { \ decl_e; \ char *_src = rpy_revdb.buf_p; \ char *_end1 = _src + sizeof(_e); \ @@ -74,11 +84,45 @@ if (_end1 >= rpy_revdb.buf_limit) \ rpy_reverse_db_fetch(__FILE__, __LINE__); \ variable = _e; \ - } + } + +#define RPY_REVDB_EMIT(normal_code, decl_e, variable) \ + if (!RPY_RDB_REPLAY) { \ + normal_code \ + _RPY_REVDB_EMIT_RECORD(decl_e, variable) \ + } else \ + _RPY_REVDB_EMIT_REPLAY(decl_e, variable) #define RPY_REVDB_EMIT_VOID(normal_code) \ if (!RPY_RDB_REPLAY) { normal_code } else { } +#define RPY_REVDB_CALL(call_code, decl_e, variable) \ + if (!RPY_RDB_REPLAY) { \ + call_code \ + _RPY_REVDB_EMIT_RECORD_EXTRA(0xFC, decl_e, variable) \ + } else { \ + unsigned char _re; \ + _RPY_REVDB_EMIT_REPLAY(unsigned char _e, _re) \ + if (_re != 0xFC) \ + rpy_reverse_db_invoke_callback(_re); \ + _RPY_REVDB_EMIT_REPLAY(decl_e, variable) \ + } + +#define RPY_REVDB_CALL_VOID(call_code) \ + if (!RPY_RDB_REPLAY) { \ + call_code \ + _RPY_REVDB_EMIT_RECORD(unsigned char _e, 0xFC) \ + } \ + else { \ + unsigned char _re; \ + _RPY_REVDB_EMIT_REPLAY(unsigned char _e, _re) \ + if (_re != 0xFC) \ + rpy_reverse_db_invoke_callback(_re); \ + } + +#define RPY_REVDB_CALLBACKLOC(locnum) \ + rpy_reverse_db_callback_loc(locnum) + #define RPY_REVDB_REC_UID(expr) \ do { \ uint64_t uid = rpy_revdb.unique_id_seen; \ @@ -151,5 +195,7 @@ RPY_EXTERN void *rpy_reverse_db_next_dead(void *result); RPY_EXTERN void rpy_reverse_db_register_destructor(void *obj, void(*)(void *)); RPY_EXTERN void rpy_reverse_db_call_destructor(void *obj); +RPY_EXTERN void rpy_reverse_db_invoke_callback(unsigned char); +RPY_EXTERN void rpy_reverse_db_callback_loc(int); /* ------------------------------------------------------------ */ diff --git a/rpython/translator/revdb/test/test_basic.py b/rpython/translator/revdb/test/test_basic.py --- a/rpython/translator/revdb/test/test_basic.py +++ b/rpython/translator/revdb/test/test_basic.py @@ -85,9 +85,14 @@ def write_call(self, expected_string): x = self.next() # raw_malloc: the pointer we got + self.same_thread() x = self.next(); assert x == len(expected_string) + self.same_thread() x = self.next('i'); assert x == 0 # errno + def same_thread(self): + x = self.next('c'); assert x == '\xFC' + def compile(self, entry_point, backendopt=True, withsmallfuncsets=None): diff --git a/rpython/translator/revdb/test/test_callback.py b/rpython/translator/revdb/test/test_callback.py new file mode 100644 --- /dev/null +++ b/rpython/translator/revdb/test/test_callback.py @@ -0,0 +1,109 @@ +from rpython.translator.tool.cbuild import ExternalCompilationInfo +from rpython.rtyper.lltypesystem import lltype, rffi +from rpython.rlib.rarithmetic import intmask +from rpython.rlib import revdb +from rpython.translator.revdb.test.test_basic import BaseRecordingTests +from rpython.translator.revdb.test.test_basic import InteractiveTests + +from rpython.translator.revdb.message import * + + +def get_callback_demo(): + eci = ExternalCompilationInfo(separate_module_sources=[''' + int callme(int(*cb)(int)) { + return cb(40) * cb(3); + } + '''], post_include_bits=[''' + int callme(int(*)(int)); + ''']) + FUNCPTR = lltype.Ptr(lltype.FuncType([rffi.INT], rffi.INT)) + callme = rffi.llexternal('callme', [FUNCPTR], rffi.INT, + compilation_info=eci) + + def callback(n): + print intmask(n) + return n + + def main(argv): + revdb.stop_point() + print intmask(callme(callback)) + revdb.stop_point() + return 9 + + return main + + +class TestRecording(BaseRecordingTests): + + def test_callback_simple(self): + eci = ExternalCompilationInfo(separate_module_sources=[''' + int callme(int(*cb)(int)) { + return cb(40) * cb(3); + } + int callmesimple(void) { + return 55555; + } + '''], post_include_bits=[''' + int callme(int(*)(int)); + int callmesimple(void); + ''']) + FUNCPTR = lltype.Ptr(lltype.FuncType([rffi.INT], rffi.INT)) + callme = rffi.llexternal('callme', [FUNCPTR], rffi.INT, + compilation_info=eci) + callmesimple = rffi.llexternal('callmesimple', [], rffi.INT, + compilation_info=eci) + + def callback(n): + return intmask(n) * 100 + + def main(argv): + print intmask(callmesimple()) + print intmask(callme(callback)) + return 9 + self.compile(main, backendopt=False) + out = self.run('Xx') + rdb = self.fetch_rdb([self.exename, 'Xx']) + rdb.same_thread() # callmesimple() + x = rdb.next('i'); assert x == 55555 + rdb.write_call('55555\n') + b = rdb.next('!h'); assert 0 <= b < 10 # -> callback + x = rdb.next('i'); assert x == 40 # arg n + x = rdb.next('!h'); assert x == b # -> callback + x = rdb.next('i'); assert x == 3 # arg n + rdb.same_thread() # <- return in main thread + x = rdb.next('i'); assert x == 4000 * 300 # return from callme() + rdb.write_call('%s\n' % (4000 * 300,)) + x = rdb.next('q'); assert x == 0 # number of stop points + assert rdb.done() + + def test_callback_with_effects(self): + main = get_callback_demo() + self.compile(main, backendopt=False) + out = self.run('Xx') + rdb = self.fetch_rdb([self.exename, 'Xx']) + b = rdb.next('!h'); assert 0 <= b < 10 # -> callback + x = rdb.next('i'); assert x == 40 # arg n + rdb.write_call('40\n') + x = rdb.next('!h'); assert x == b # -> callback again + x = rdb.next('i'); assert x == 3 # arg n + rdb.write_call('3\n') + rdb.same_thread() # -> return in main thread + x = rdb.next('i'); assert x == 120 # <- return from callme() + rdb.write_call('120\n') + x = rdb.next('q'); assert x == 2 # number of stop points + assert rdb.done() + + +class TestReplayingCallback(InteractiveTests): + expected_stop_points = 2 + + def setup_class(cls): + from rpython.translator.revdb.test.test_basic import compile, run + main = get_callback_demo() + compile(cls, main, backendopt=False) + run(cls, '') + + def test_replaying_callback(self): + child = self.replay() + child.send(Message(CMD_FORWARD, 3)) + child.expect(ANSWER_AT_END) diff --git a/rpython/translator/revdb/test/test_weak.py b/rpython/translator/revdb/test/test_weak.py --- a/rpython/translator/revdb/test/test_weak.py +++ b/rpython/translator/revdb/test/test_weak.py @@ -203,6 +203,7 @@ assert time == i + 1 y = intmask(rdb.next('q')); assert y == -1 triggered = True + rdb.same_thread() j = rdb.next() assert j == i + 1000000 * triggered if triggered: @@ -214,6 +215,7 @@ assert uid > 0 and uid not in uid_seen uid_seen.add(uid) lst.append(uid) + rdb.same_thread() totals.append((lst, intmask(rdb.next()))) x = rdb.next('q'); assert x == 3000 # number of stop points # @@ -243,11 +245,13 @@ assert x != -1 assert x not in seen_uids seen_uids.add(x) + rdb.same_thread() y = intmask(rdb.next()) assert y == -7 # from the __del__ x = intmask(rdb.next()) if x == -1: break + rdb.same_thread() x = rdb.next() assert x == len(seen_uids) assert len(seen_uids) == int(out) From pypy.commits at gmail.com Sat Jul 2 15:35:23 2016 From: pypy.commits at gmail.com (raffael_t) Date: Sat, 02 Jul 2016 12:35:23 -0700 (PDT) Subject: [pypy-commit] pypy py3.5-async: Add async to grammar Message-ID: <5778177b.972e1c0a.33748.ffff9cdf@mx.google.com> Author: Raffael Tfirst Branch: py3.5-async Changeset: r85518:618d170d0fa1 Date: 2016-07-02 21:34 +0200 http://bitbucket.org/pypy/pypy/changeset/618d170d0fa1/ Log: Add async to grammar diff --git a/pypy/interpreter/pyparser/data/Grammar3.5 b/pypy/interpreter/pyparser/data/Grammar3.5 --- a/pypy/interpreter/pyparser/data/Grammar3.5 +++ b/pypy/interpreter/pyparser/data/Grammar3.5 @@ -21,10 +21,9 @@ decorator: '@' dotted_name [ '(' [arglist] ')' ] NEWLINE decorators: decorator+ -decorated: decorators (classdef | funcdef) -# | async_funcdef) +decorated: decorators (classdef | funcdef | async_funcdef) -# async_funcdef: ASYNC funcdef +async_funcdef: ASYNC funcdef funcdef: 'def' NAME parameters ['->' test] ':' suite parameters: '(' [typedargslist] ')' @@ -69,9 +68,8 @@ nonlocal_stmt: 'nonlocal' NAME (',' NAME)* assert_stmt: 'assert' test [',' test] -compound_stmt: if_stmt | while_stmt | for_stmt | try_stmt | with_stmt | funcdef | classdef | decorated -# | async_stmt -# async_stmt: ASYNC (funcdef | with_stmt | for_stmt) +compound_stmt: if_stmt | while_stmt | for_stmt | try_stmt | with_stmt | funcdef | classdef | decorated | async_stmt +async_stmt: ASYNC (funcdef | with_stmt | for_stmt) if_stmt: 'if' test ':' suite ('elif' test ':' suite)* ['else' ':' suite] while_stmt: 'while' test ':' suite ['else' ':' suite] for_stmt: 'for' exprlist 'in' testlist ':' suite ['else' ':' suite] @@ -105,9 +103,8 @@ arith_expr: term (('+'|'-') term)* term: factor (('*'|'@'|'/'|'%'|'//') factor)* factor: ('+'|'-'|'~') factor | power -# power: atom_expr ['**' factor] -power: atom trailer* ['**' factor] -# atom_expr: [AWAIT] atom trailer* +power: atom_expr ['**' factor] +atom_expr: [AWAIT] atom trailer* atom: ('(' [yield_expr|testlist_comp] ')' | '[' [testlist_comp] ']' | '{' [dictorsetmaker] '}' | @@ -119,8 +116,6 @@ sliceop: ':' [test] exprlist: (expr|star_expr) (',' (expr|star_expr))* [','] testlist: test (',' test)* [','] -#dictorsetmaker: ( (test ':' test (comp_for | (',' test ':' test)* [','])) | -# (test (comp_for | (',' test)* [','])) ) dictorsetmaker: ( ((test ':' test | '**' expr) (comp_for | (',' (test ':' test | '**' expr))* [','])) | ((test | star_expr) @@ -128,9 +123,6 @@ classdef: 'class' NAME ['(' [arglist] ')'] ':' suite -#arglist: (argument ',')* (argument [','] -# |'*' test (',' argument)* [',' '**' test] -# |'**' test) arglist: argument (',' argument)* [','] # The reason that keywords are test nodes instead of NAME is that using NAME @@ -142,7 +134,6 @@ # Illegal combinations and orderings are blocked in ast.c: # multiple (test comp_for) arguements are blocked; keyword unpackings # that precede iterable unpackings are blocked; etc. -#argument: test [comp_for] | test '=' test # Really [keyword '='] test argument: ( test [comp_for] | test '=' test | '**' test | From pypy.commits at gmail.com Sat Jul 2 15:58:52 2016 From: pypy.commits at gmail.com (raffael_t) Date: Sat, 02 Jul 2016 12:58:52 -0700 (PDT) Subject: [pypy-commit] pypy py3.5-async: Change asdl, add tokens for async and await Message-ID: <57781cfc.c255c20a.578c7.ffffc56d@mx.google.com> Author: Raffael Tfirst Branch: py3.5-async Changeset: r85519:aca645000678 Date: 2016-07-02 21:58 +0200 http://bitbucket.org/pypy/pypy/changeset/aca645000678/ Log: Change asdl, add tokens for async and await diff --git a/pypy/interpreter/astcompiler/tools/Python.asdl b/pypy/interpreter/astcompiler/tools/Python.asdl --- a/pypy/interpreter/astcompiler/tools/Python.asdl +++ b/pypy/interpreter/astcompiler/tools/Python.asdl @@ -11,6 +11,8 @@ stmt = FunctionDef(identifier name, arguments args, stmt* body, expr* decorator_list, expr? returns) + | AsyncFunctionDef(identifier name, arguments args, + stmt* body, expr* decorator_list, expr? returns) | ClassDef(identifier name, expr* bases, keyword* keywords, @@ -24,9 +26,11 @@ -- use 'orelse' because else is a keyword in target languages | For(expr target, expr iter, stmt* body, stmt* orelse) + | AsyncFor(expr target, expr iter, stmt* body, stmt* orelse) | While(expr test, stmt* body, stmt* orelse) | If(expr test, stmt* body, stmt* orelse) | With(withitem* items, stmt* body) + | AsyncWith(withitem* items, stmt* body) | Raise(expr? exc, expr? cause) | Try(stmt* body, excepthandler* handlers, stmt* orelse, stmt* finalbody) @@ -57,6 +61,7 @@ | DictComp(expr key, expr value, comprehension* generators) | GeneratorExp(expr elt, comprehension* generators) -- the grammar constrains where yield expressions can occur + | Await(expr value) | Yield(expr? value) | YieldFrom(expr value) -- need sequences for compare to distinguish between diff --git a/pypy/interpreter/pyparser/pytoken.py b/pypy/interpreter/pyparser/pytoken.py --- a/pypy/interpreter/pyparser/pytoken.py +++ b/pypy/interpreter/pyparser/pytoken.py @@ -65,6 +65,8 @@ _add_tok('RARROW', "->") _add_tok('ELLIPSIS', "...") _add_tok('OP') +_add_tok('ASYNC') +_add_tok('AWAIT') _add_tok('ERRORTOKEN') # extra PyPy-specific tokens From pypy.commits at gmail.com Sat Jul 2 16:27:52 2016 From: pypy.commits at gmail.com (mattip) Date: Sat, 02 Jul 2016 13:27:52 -0700 (PDT) Subject: [pypy-commit] pypy call-via-pyobj: fix translation (arigato) Message-ID: <577823c8.06a61c0a.d7864.ffff80cd@mx.google.com> Author: Matti Picus Branch: call-via-pyobj Changeset: r85520:192a002624b0 Date: 2016-07-02 22:34 +0300 http://bitbucket.org/pypy/pypy/changeset/192a002624b0/ Log: fix translation (arigato) diff --git a/pypy/module/cpyext/methodobject.py b/pypy/module/cpyext/methodobject.py --- a/pypy/module/cpyext/methodobject.py +++ b/pypy/module/cpyext/methodobject.py @@ -1,4 +1,4 @@ -from rpython.rtyper.lltypesystem import lltype, rffi +from rpython.rtyper.lltypesystem import lltype, rffi, llmemory from pypy.interpreter.baseobjspace import W_Root from pypy.interpreter.error import OperationError, oefmt @@ -176,22 +176,14 @@ self.method_name) func_to_call = self.func if self.offset: - ptr = as_pyobj(space, self.w_objclass) + pto = as_pyobj(space, self.w_objclass) # make ptr the equivalent of this, using the offsets #func_to_call = rffi.cast(rffi.VOIDP, ptr.c_tp_as_number.c_nb_multiply) - if not ptr is rffi.VOIDP: - ptr = rffi.cast(rffi.VOIDP, ptr) + if pto: + ptr = llmemory.cast_ptr_to_adr(pto) for o in self.offset: - if not ptr: - break - ptr_as_int = lltype.cast_ptr_to_int(ptr) - pptr = rffi.cast(rffi.VOIDPP, ptr_as_int + o) - if not pptr: - break - ptr = pptr[0] - ptr = rffi.cast(rffi.VOIDP, ptr) - if ptr: - func_to_call = ptr + ptr = (ptr + o).address[0] + func_to_call = llmemory.cast_adr_to_ptr(ptr, rffi.VOIDP) assert func_to_call return self.wrapper_func(space, w_self, w_args, func_to_call) From pypy.commits at gmail.com Sat Jul 2 16:27:54 2016 From: pypy.commits at gmail.com (mattip) Date: Sat, 02 Jul 2016 13:27:54 -0700 (PDT) Subject: [pypy-commit] pypy call-via-pyobj: try another approach (arigato), also refactor Message-ID: <577823ca.c8a61c0a.2ede3.69e6@mx.google.com> Author: Matti Picus Branch: call-via-pyobj Changeset: r85521:c9f4f6cebf76 Date: 2016-07-02 23:26 +0300 http://bitbucket.org/pypy/pypy/changeset/c9f4f6cebf76/ Log: try another approach (arigato), also refactor diff --git a/pypy/module/cpyext/methodobject.py b/pypy/module/cpyext/methodobject.py --- a/pypy/module/cpyext/methodobject.py +++ b/pypy/module/cpyext/methodobject.py @@ -166,25 +166,30 @@ self.w_objclass = w_type def call(self, space, w_self, w_args, w_kw): - if self.wrapper_func is None: - assert self.wrapper_func_kwds is not None - return self.wrapper_func_kwds(space, w_self, w_args, self.func, - w_kw) - if space.is_true(w_kw): - raise oefmt(space.w_TypeError, - "wrapper %s doesn't take any keyword arguments", - self.method_name) func_to_call = self.func if self.offset: pto = as_pyobj(space, self.w_objclass) # make ptr the equivalent of this, using the offsets #func_to_call = rffi.cast(rffi.VOIDP, ptr.c_tp_as_number.c_nb_multiply) if pto: - ptr = llmemory.cast_ptr_to_adr(pto) + cptr = rffi.cast(rffi.CCHARP, pto) for o in self.offset: - ptr = (ptr + o).address[0] - func_to_call = llmemory.cast_adr_to_ptr(ptr, rffi.VOIDP) + ptr = rffi.cast(rffi.VOIDPP, rffi.ptradd(cptr, o))[0] + cptr = rffi.cast(rffi.CCHARP, ptr) + func_to_call = rffi.cast(rffi.VOIDP, cptr) + else: + # Should never happen, assert to get a traceback + assert False, "failed to convert w_type %s to PyObject" % str( + self.w_objclass) assert func_to_call + if self.wrapper_func is None: + assert self.wrapper_func_kwds is not None + return self.wrapper_func_kwds(space, w_self, w_args, func_to_call, + w_kw) + if space.is_true(w_kw): + raise oefmt(space.w_TypeError, + "wrapper %s doesn't take any keyword arguments", + self.method_name) return self.wrapper_func(space, w_self, w_args, func_to_call) def descr_method_repr(self): From pypy.commits at gmail.com Sat Jul 2 16:48:17 2016 From: pypy.commits at gmail.com (arigo) Date: Sat, 02 Jul 2016 13:48:17 -0700 (PDT) Subject: [pypy-commit] pypy reverse-debugger: Next bug Message-ID: <57782891.50991c0a.b744b.9b36@mx.google.com> Author: Armin Rigo Branch: reverse-debugger Changeset: r85522:7a8820692e0a Date: 2016-07-02 22:49 +0200 http://bitbucket.org/pypy/pypy/changeset/7a8820692e0a/ Log: Next bug diff --git a/rpython/translator/revdb/test/test_bug.py b/rpython/translator/revdb/test/test_bug.py new file mode 100644 --- /dev/null +++ b/rpython/translator/revdb/test/test_bug.py @@ -0,0 +1,41 @@ +import os, subprocess +from rpython.rlib import revdb +from rpython.rtyper.lltypesystem import lltype +from rpython.translator.revdb.test.test_basic import InteractiveTests + +from rpython.translator.revdb.message import * + + +class TestReplayingBug(InteractiveTests): + expected_stop_points = 1 + + def setup_class(cls): + from rpython.translator.revdb.test.test_basic import compile, run + + FOO = lltype.Struct('FOO') + foo = lltype.malloc(FOO, flavor='raw', immortal=True) + + BAR = lltype.Struct('BAR', ('p', lltype.Ptr(FOO))) + bar = lltype.malloc(BAR, flavor='raw', immortal=True) + bar.p = foo + + def main(argv): + assert bar.p == foo + revdb.stop_point() + return 9 + + compile(cls, main, backendopt=False) + run(cls, '') + + def test_replaying_bug(self): + # This tiny test seems to always have foo at the same address + # in multiple runs. Here we recompile with different options + # just to change that address. + subprocess.check_call(["make", "clean"], + cwd=os.path.dirname(str(self.exename))) + subprocess.check_call(["make", "lldebug"], + cwd=os.path.dirname(str(self.exename))) + # + child = self.replay() + child.send(Message(CMD_FORWARD, 2)) + child.expect(ANSWER_AT_END) From pypy.commits at gmail.com Sun Jul 3 11:50:48 2016 From: pypy.commits at gmail.com (arigo) Date: Sun, 03 Jul 2016 08:50:48 -0700 (PDT) Subject: [pypy-commit] pypy reverse-debugger: in-progress: fixes the inconsistency in static pointers to raw Message-ID: <57793458.c4cb1c0a.58bbf.ffff8510@mx.google.com> Author: Armin Rigo Branch: reverse-debugger Changeset: r85523:2c7ee80382b2 Date: 2016-07-03 16:34 +0200 http://bitbucket.org/pypy/pypy/changeset/2c7ee80382b2/ Log: in-progress: fixes the inconsistency in static pointers to raw structures: the code containing directly a pointer would use the address of the real structure in the current process; but if loading this address from memory, we would instead see the old recorded address. diff --git a/rpython/memory/gctransform/boehm.py b/rpython/memory/gctransform/boehm.py --- a/rpython/memory/gctransform/boehm.py +++ b/rpython/memory/gctransform/boehm.py @@ -31,7 +31,8 @@ fields = [("hash", lltype.Signed)] if translator and translator.config.translation.reverse_debugger: fields.append(("uid", lltype.SignedLongLong)) - self.HDR = lltype.Struct("header", *fields) + hints = {'hints': {'gcheader': True}} + self.HDR = lltype.Struct("header", *fields, **hints) HDRPTR = lltype.Ptr(self.HDR) if self.translator: diff --git a/rpython/translator/c/genc.py b/rpython/translator/c/genc.py --- a/rpython/translator/c/genc.py +++ b/rpython/translator/c/genc.py @@ -695,6 +695,8 @@ print >> f, '#define PYPY_FILE_NAME "%s"' % os.path.basename(f.name) print >> f, '#include "src/g_include.h"' + if self.database.reverse_debugger: + print >> f, '#include "revdb_def.h"' print >> f nextralines = 11 + 1 diff --git a/rpython/translator/c/node.py b/rpython/translator/c/node.py --- a/rpython/translator/c/node.py +++ b/rpython/translator/c/node.py @@ -459,19 +459,23 @@ self.implementationtypename = db.gettype( T, varlength=self.getvarlength()) parent, parentindex = parentlink(obj) + mangled = False if obj in exports.EXPORTS_obj2name: self.name = exports.EXPORTS_obj2name[obj] self.globalcontainer = 2 # meh elif parent is None: self.name = db.namespace.uniquename('g_' + self.basename()) self.globalcontainer = True + if db.reverse_debugger and T._gckind != 'gc': + from rpython.translator.revdb import gencsupp + mangled = gencsupp.mangle_name_prebuilt_raw(db, self, T) else: self.globalcontainer = False parentnode = db.getcontainernode(parent) defnode = db.gettypedefnode(parentnode.getTYPE()) self.name = defnode.access_expr(parentnode.name, parentindex) if self.typename != self.implementationtypename: - if db.gettypedefnode(T).extra_union_for_varlength: + if db.gettypedefnode(T).extra_union_for_varlength and not mangled: self.name += '.b' self._funccodegen_owner = None @@ -492,19 +496,23 @@ return getattr(self.obj, self.eci_name, None) def get_declaration(self): - if self.name[-2:] == '.b': + name = self.name + if name.startswith('RPY_RDB_A('): + assert name.endswith(')') + name = name[len('RPY_RDB_A('):-1] + if name[-2:] == '.b': # xxx fish fish assert self.implementationtypename.startswith('struct ') assert self.implementationtypename.endswith(' @') uniontypename = 'union %su @' % self.implementationtypename[7:-2] - return uniontypename, self.name[:-2] + return uniontypename, name[:-2], True else: - return self.implementationtypename, self.name + return self.implementationtypename, name, False def forward_declaration(self): if llgroup.member_of_group(self.obj): return - type, name = self.get_declaration() + type, name, is_union = self.get_declaration() yield '%s;' % ( forward_cdecl(type, name, self.db.standalone, is_thread_local=self.is_thread_local(), @@ -514,12 +522,12 @@ if llgroup.member_of_group(self.obj): return [] lines = list(self.initializationexpr()) - type, name = self.get_declaration() - if name != self.name and len(lines) < 2: + type, name, is_union = self.get_declaration() + if is_union and len(lines) < 2: # a union with length 0 lines[0] = cdecl(type, name, self.is_thread_local()) else: - if name != self.name: + if is_union: lines[0] = '{ ' + lines[0] # extra braces around the 'a' part lines[-1] += ' }' # of the union lines[0] = '%s = %s' % ( @@ -597,8 +605,8 @@ padding_drop = T._hints['get_padding_drop'](d) else: padding_drop = [] - type, name = self.get_declaration() - if name != self.name and self.getvarlength() < 1 and len(data) < 2: + type, name, is_union = self.get_declaration() + if is_union and self.getvarlength() < 1 and len(data) < 2: # an empty union yield '' return @@ -795,6 +803,11 @@ expr = db.get(value) if typeOf(value) is Void: comma = '' + elif expr.startswith('(&RPY_RDB_A('): + # can't use this in static initialization code + assert db.reverse_debugger + db.late_initializations.append(('%s' % access_expr, expr)) + expr = 'NULL /* patched later with %s */' % (expr,) expr += comma i = expr.find('\n') if i < 0: diff --git a/rpython/translator/revdb/gencsupp.py b/rpython/translator/revdb/gencsupp.py --- a/rpython/translator/revdb/gencsupp.py +++ b/rpython/translator/revdb/gencsupp.py @@ -1,4 +1,4 @@ -import py +import py, random, sys from rpython.rtyper.lltypesystem import lltype, llmemory, rffi, rstr from rpython.rtyper.lltypesystem.lloperation import LL_OPERATIONS from rpython.translator.c.support import cdecl @@ -11,6 +11,17 @@ srcdir / 'revdb.c', ] +def mangle_name_prebuilt_raw(database, node, S): + if (S._gckind != 'gc' and not S._hints.get('is_excdata') + and not S._hints.get('static_immutable') + and not S._hints.get('ignore_revdb') + and not S._hints.get('gcheader')): + database.all_raw_structures.append(node) + node.name = 'RPY_RDB_A(%s)' % (node.name,) + return True + else: + return False + def prepare_function(funcgen): stack_bottom = False for block in funcgen.graph.iterblocks(): @@ -102,10 +113,13 @@ db.get(s) db.stack_bottom_funcnames = [] + db.all_raw_structures = [] def write_revdb_def_file(db, target_path): + f = target_path.open('w') funcnames = sorted(db.stack_bottom_funcnames) - f = target_path.open('w') + print >> f, "#define RDB_VERSION 0x%x" % random.randrange(0, sys.maxint) + print >> f for i, fn in enumerate(funcnames): print >> f, '#define RPY_CALLBACKLOC_%s %d' % (fn, i) print >> f @@ -117,4 +131,36 @@ else: tail = ', \\' print >> f, '\t(void *)%s%s' % (fn, tail) + print >> f + + def _base(name): + assert name.startswith('RPY_RDB_A(') + if name.endswith('.b'): + name = name[:-2] + name = name[len('RPY_RDB_A('):-1] + return name + + rawstructs = sorted(db.all_raw_structures, key=lambda node: node.name) + print >> f, '#define RPY_RDB_A(name) (*rpy_rdb_struct.name)' + print >> f, 'struct rpy_rdb_a_s {' + for i, node in enumerate(rawstructs): + print >> f, '\t%s;' % (cdecl(node.typename, '*'+_base(node.name)),) + if not rawstructs: + print >> f, '\tchar dummy;' + print >> f, '};' + print >> f, 'RPY_EXTERN struct rpy_rdb_a_s rpy_rdb_struct;' + print >> f + print >> f, '#define RPY_RDB_STRUCT_CONTENT \\' + if not rawstructs: + print >> f, '\t0' + else: + for i, node in enumerate(rawstructs): + if i == len(rawstructs) - 1: + tail = '' + else: + tail = ', \\' + name = '&' + _base(node.name) + if node.typename != node.implementationtypename: + name = '(%s)%s' % (cdecl(node.typename, '*'), name) + print >> f, '\t%s%s' % (name, tail) f.close() diff --git a/rpython/translator/revdb/src-revdb/revdb.c b/rpython/translator/revdb/src-revdb/revdb.c --- a/rpython/translator/revdb/src-revdb/revdb.c +++ b/rpython/translator/revdb/src-revdb/revdb.c @@ -22,7 +22,6 @@ #include "src-revdb/revdb_include.h" #define RDB_SIGNATURE "RevDB:" -#define RDB_VERSION 0x00FF0002 #define WEAKREF_AFTERWARDS_DEAD ((char)0xf2) #define WEAKREF_AFTERWARDS_ALIVE ((char)0xeb) @@ -37,6 +36,7 @@ typedef struct { Signed version; uint64_t reserved1, reserved2; + unsigned int size_rdb_struct; int argc; char **argv; } rdb_header_t; @@ -164,9 +164,14 @@ memset(&h, 0, sizeof(h)); h.version = RDB_VERSION; + h.size_rdb_struct = sizeof(rpy_rdb_struct); h.argc = argc; h.argv = argv; write_all((const char *)&h, sizeof(h)); + + /* write the whole content of rpy_rdb_struct */ + write_all((const char *)&rpy_rdb_struct, sizeof(rpy_rdb_struct)); + fprintf(stderr, "PID %d: recording revdb log to '%s'\n", (int)getpid(), filename); } @@ -464,14 +469,11 @@ RPY_EXTERN void rpy_reverse_db_callback_loc(int locnum) { - union { - unsigned char n[2]; - uint16_t u; - } r; + locnum += 300; assert(locnum < 0xFC00); if (!RPY_RDB_REPLAY) { - RPY_REVDB_EMIT(r.n[0] = locnum >> 8; r.n[1] = locnum & 0xFF;, - uint16_t _e, r.u); + _RPY_REVDB_EMIT_RECORD(unsigned char _e, (locnum >> 8)); + _RPY_REVDB_EMIT_RECORD(unsigned char _e, (locnum & 0xFF)); } } @@ -663,6 +665,10 @@ (long)h.version, (long)RDB_VERSION); exit(1); } + if (h.size_rdb_struct != sizeof(rpy_rdb_struct)) { + fprintf(stderr, "bad size_rdb_struct\n"); + exit(1); + } *argc_p = h.argc; *argv_p = h.argv; @@ -676,6 +682,9 @@ exit(1); } + /* read the whole content of rpy_rdb_struct */ + read_all((char *)&rpy_rdb_struct, sizeof(rpy_rdb_struct)); + rpy_revdb.buf_p = rpy_rev_buffer; rpy_revdb.buf_limit = rpy_rev_buffer; rpy_revdb.buf_readend = rpy_rev_buffer; @@ -1335,6 +1344,10 @@ fq_trigger(); } +struct rpy_rdb_a_s rpy_rdb_struct = { + RPY_RDB_STRUCT_CONTENT /* macro from revdb_def.h */ +}; + static void *callbacklocs[] = { RPY_CALLBACKLOCS /* macro from revdb_def.h */ }; @@ -1352,6 +1365,7 @@ void (*pfn)(void); _RPY_REVDB_EMIT_REPLAY(unsigned char _e, e2) index = (e << 8) | e2; + index -= 300; if (index >= (sizeof(callbacklocs) / sizeof(callbacklocs[0]))) { fprintf(stderr, "bad callback index\n"); exit(1); diff --git a/rpython/translator/revdb/src-revdb/revdb_include.h b/rpython/translator/revdb/src-revdb/revdb_include.h --- a/rpython/translator/revdb/src-revdb/revdb_include.h +++ b/rpython/translator/revdb/src-revdb/revdb_include.h @@ -29,15 +29,15 @@ RPY_EXTERN void rpy_reverse_db_setup(int *argc_p, char **argv_p[]); RPY_EXTERN void rpy_reverse_db_teardown(void); -#if 0 /* enable to print locations to stderr of all the EMITs */ -# define _RPY_REVDB_PRINT(mode) \ +#if 1 /* enable to print locations to stderr of all the EMITs */ +# define _RPY_REVDB_PRINT(mode, _e) \ fprintf(stderr, \ "%s:%d: %0*llx\n", \ __FILE__, __LINE__, 2 * sizeof(_e), \ ((unsigned long long)_e) & ((2ULL << (8*sizeof(_e)-1)) - 1)) #endif -#if 0 /* enable to print all mallocs to stderr */ +#if 1 /* enable to print all mallocs to stderr */ RPY_EXTERN void seeing_uid(uint64_t uid); # define _RPY_REVDB_PRUID() \ seeing_uid(uid); \ @@ -47,7 +47,7 @@ #endif #ifndef _RPY_REVDB_PRINT -# define _RPY_REVDB_PRINT(mode) /* nothing */ +# define _RPY_REVDB_PRINT(mode, _e) /* nothing */ #endif #ifndef _RPY_REVDB_PRUID # define _RPY_REVDB_PRUID() /* nothing */ @@ -57,22 +57,12 @@ #define _RPY_REVDB_EMIT_RECORD(decl_e, variable) \ { \ decl_e = variable; \ - _RPY_REVDB_PRINT("write"); \ + _RPY_REVDB_PRINT("write", _e); \ memcpy(rpy_revdb.buf_p, &_e, sizeof(_e)); \ if ((rpy_revdb.buf_p += sizeof(_e)) > rpy_revdb.buf_limit) \ rpy_reverse_db_flush(); \ } -#define _RPY_REVDB_EMIT_RECORD_EXTRA(extra, decl_e, variable) \ - { \ - decl_e = variable; \ - _RPY_REVDB_PRINT("write"); \ - rpy_revdb.buf_p[0] = extra; \ - memcpy(rpy_revdb.buf_p + 1, &_e, sizeof(_e)); \ - if ((rpy_revdb.buf_p += 1 + sizeof(_e)) > rpy_revdb.buf_limit) \ - rpy_reverse_db_flush(); \ - } - #define _RPY_REVDB_EMIT_REPLAY(decl_e, variable) \ { \ decl_e; \ @@ -80,7 +70,7 @@ char *_end1 = _src + sizeof(_e); \ memcpy(&_e, _src, sizeof(_e)); \ rpy_revdb.buf_p = _end1; \ - _RPY_REVDB_PRINT("read"); \ + _RPY_REVDB_PRINT("read", _e); \ if (_end1 >= rpy_revdb.buf_limit) \ rpy_reverse_db_fetch(__FILE__, __LINE__); \ variable = _e; \ @@ -99,7 +89,8 @@ #define RPY_REVDB_CALL(call_code, decl_e, variable) \ if (!RPY_RDB_REPLAY) { \ call_code \ - _RPY_REVDB_EMIT_RECORD_EXTRA(0xFC, decl_e, variable) \ + _RPY_REVDB_EMIT_RECORD(unsigned char _e, 0xFC) \ + _RPY_REVDB_EMIT_RECORD(decl_e, variable) \ } else { \ unsigned char _re; \ _RPY_REVDB_EMIT_REPLAY(unsigned char _e, _re) \ diff --git a/rpython/translator/revdb/test/test_basic.py b/rpython/translator/revdb/test/test_basic.py --- a/rpython/translator/revdb/test/test_basic.py +++ b/rpython/translator/revdb/test/test_basic.py @@ -22,11 +22,14 @@ # self.cur = 0 x = self.read1('c'); assert x == '\x00' - x = self.read1('P'); assert x == 0x00FF0002 + x = self.read1('P'); #assert x == ...random version number... x = self.read1('P'); assert x == 0 x = self.read1('P'); assert x == 0 - self.argc = self.read1('P') + size_rdb_struct = self.read1('I') + self.argc = self.read1('i') self.argv = self.read1('P') + self.rdb_struct = self.buffer[self.cur : self.cur + size_rdb_struct] + self.cur += size_rdb_struct self.current_packet_end = self.cur self.read_check_argv(expected_argv) diff --git a/rpython/translator/revdb/test/test_callback.py b/rpython/translator/revdb/test/test_callback.py --- a/rpython/translator/revdb/test/test_callback.py +++ b/rpython/translator/revdb/test/test_callback.py @@ -66,7 +66,7 @@ rdb.same_thread() # callmesimple() x = rdb.next('i'); assert x == 55555 rdb.write_call('55555\n') - b = rdb.next('!h'); assert 0 <= b < 10 # -> callback + b = rdb.next('!h'); assert 300 <= b < 310 # -> callback x = rdb.next('i'); assert x == 40 # arg n x = rdb.next('!h'); assert x == b # -> callback x = rdb.next('i'); assert x == 3 # arg n @@ -81,7 +81,7 @@ self.compile(main, backendopt=False) out = self.run('Xx') rdb = self.fetch_rdb([self.exename, 'Xx']) - b = rdb.next('!h'); assert 0 <= b < 10 # -> callback + b = rdb.next('!h'); assert 300 <= b < 310 # -> callback x = rdb.next('i'); assert x == 40 # arg n rdb.write_call('40\n') x = rdb.next('!h'); assert x == b # -> callback again diff --git a/rpython/translator/revdb/test/test_bug.py b/rpython/translator/revdb/test/test_raw.py rename from rpython/translator/revdb/test/test_bug.py rename to rpython/translator/revdb/test/test_raw.py --- a/rpython/translator/revdb/test/test_bug.py +++ b/rpython/translator/revdb/test/test_raw.py @@ -6,11 +6,12 @@ from rpython.translator.revdb.message import * -class TestReplayingBug(InteractiveTests): +class TestReplayingRaw(InteractiveTests): expected_stop_points = 1 def setup_class(cls): from rpython.translator.revdb.test.test_basic import compile, run + from rpython.translator.revdb.test.test_basic import fetch_rdb FOO = lltype.Struct('FOO') foo = lltype.malloc(FOO, flavor='raw', immortal=True) @@ -19,15 +20,23 @@ bar = lltype.malloc(BAR, flavor='raw', immortal=True) bar.p = foo + BAZ = lltype.Struct('BAZ', ('p', lltype.Ptr(FOO)), ('q', lltype.Signed), + hints={'union': True}) + baz = lltype.malloc(BAZ, flavor='raw', immortal=True) + baz.p = foo + def main(argv): assert bar.p == foo + assert baz.p == foo revdb.stop_point() return 9 compile(cls, main, backendopt=False) run(cls, '') + rdb = fetch_rdb(cls, [cls.exename]) + assert len(rdb.rdb_struct) >= 4 - def test_replaying_bug(self): + def test_replaying_raw(self): # This tiny test seems to always have foo at the same address # in multiple runs. Here we recompile with different options # just to change that address. From pypy.commits at gmail.com Sun Jul 3 11:50:50 2016 From: pypy.commits at gmail.com (arigo) Date: Sun, 03 Jul 2016 08:50:50 -0700 (PDT) Subject: [pypy-commit] pypy reverse-debugger: extra test Message-ID: <5779345a.03c31c0a.ae7c9.ffffce1a@mx.google.com> Author: Armin Rigo Branch: reverse-debugger Changeset: r85524:4897e2599437 Date: 2016-07-03 16:38 +0200 http://bitbucket.org/pypy/pypy/changeset/4897e2599437/ Log: extra test diff --git a/rpython/translator/revdb/test/test_raw.py b/rpython/translator/revdb/test/test_raw.py --- a/rpython/translator/revdb/test/test_raw.py +++ b/rpython/translator/revdb/test/test_raw.py @@ -25,9 +25,15 @@ baz = lltype.malloc(BAZ, flavor='raw', immortal=True) baz.p = foo + VBAR = lltype.Array(lltype.Ptr(FOO)) + vbar = lltype.malloc(VBAR, 3, flavor='raw', immortal=True) + vbar[0] = vbar[1] = vbar[2] = foo + def main(argv): assert bar.p == foo assert baz.p == foo + for i in range(3): + assert vbar[i] == foo revdb.stop_point() return 9 From pypy.commits at gmail.com Sun Jul 3 11:50:52 2016 From: pypy.commits at gmail.com (arigo) Date: Sun, 03 Jul 2016 08:50:52 -0700 (PDT) Subject: [pypy-commit] pypy reverse-debugger: Test and fix Message-ID: <5779345c.4a9bc20a.2e07b.74bf@mx.google.com> Author: Armin Rigo Branch: reverse-debugger Changeset: r85525:9fec64697d6b Date: 2016-07-03 16:46 +0200 http://bitbucket.org/pypy/pypy/changeset/9fec64697d6b/ Log: Test and fix diff --git a/rpython/translator/c/node.py b/rpython/translator/c/node.py --- a/rpython/translator/c/node.py +++ b/rpython/translator/c/node.py @@ -497,8 +497,7 @@ def get_declaration(self): name = self.name - if name.startswith('RPY_RDB_A('): - assert name.endswith(')') + if name.startswith('RPY_RDB_A(') and name.endswith(')'): name = name[len('RPY_RDB_A('):-1] if name[-2:] == '.b': # xxx fish fish diff --git a/rpython/translator/revdb/test/test_raw.py b/rpython/translator/revdb/test/test_raw.py --- a/rpython/translator/revdb/test/test_raw.py +++ b/rpython/translator/revdb/test/test_raw.py @@ -29,11 +29,18 @@ vbar = lltype.malloc(VBAR, 3, flavor='raw', immortal=True) vbar[0] = vbar[1] = vbar[2] = foo + RECBAR = lltype.Struct('RECBAR', ('super', BAR), ('q', lltype.Ptr(FOO))) + recbar = lltype.malloc(RECBAR, flavor='raw', immortal=True) + recbar.q = foo + recbar.super.p = foo + def main(argv): assert bar.p == foo assert baz.p == foo for i in range(3): assert vbar[i] == foo + assert recbar.q == foo + assert recbar.super.p == foo revdb.stop_point() return 9 From pypy.commits at gmail.com Sun Jul 3 12:41:07 2016 From: pypy.commits at gmail.com (arigo) Date: Sun, 03 Jul 2016 09:41:07 -0700 (PDT) Subject: [pypy-commit] pypy reverse-debugger: Write down how it is supposed to work (not fully implemented, must be fixed) Message-ID: <57794023.81eac20a.c8e55.fffff61a@mx.google.com> Author: Armin Rigo Branch: reverse-debugger Changeset: r85526:a0ec1eb89799 Date: 2016-07-03 18:42 +0200 http://bitbucket.org/pypy/pypy/changeset/a0ec1eb89799/ Log: Write down how it is supposed to work (not fully implemented, must be fixed) diff --git a/rpython/translator/revdb/gencsupp.py b/rpython/translator/revdb/gencsupp.py --- a/rpython/translator/revdb/gencsupp.py +++ b/rpython/translator/revdb/gencsupp.py @@ -4,6 +4,44 @@ from rpython.translator.c.support import cdecl from rpython.rlib import exports, revdb +# +# How It Works: we arrange things so that when replaying, all variables +# have the "same" content as they had during recording. More precisely, +# we divide all variables according to their type in two categories: +# +# * "moving things", whose value during recording is bitwise different +# from their value during replaying; +# +# * "fixed things", whose values are bitwise identical. +# +# Moving things are: +# +# * GC pointers. During replaying they point to locally-allocated +# memory that is an object with the "same" content as during +# recording; +# +# * pointers to RPython functions; +# +# * pointers to structures with the "static_immutable" hint, like +# vtables. +# +# Fixed things are the rest: +# +# * integers, floats; +# +# * most raw pointers, which during replaying will thus point to +# nonsense. (This pointer is not used during replaying to +# read/write memory: any write is ignored, and any read has its +# result recorded in the log.) +# +# Note an issue with prebuilt raw pointers to fixed things (i.e. all +# constants in the C sources that appear either inside the code or +# inside "static_immutable" or prebuilt GC structures). During +# replaying, they must correspond to bitwise the same value as during +# recording, and not to the local-process address of the raw +# structure, which is typically different (and should never be used). +# + def extra_files(): srcdir = py.path.local(__file__).join('..', 'src-revdb') diff --git a/rpython/translator/revdb/test/test_raw.py b/rpython/translator/revdb/test/test_raw.py --- a/rpython/translator/revdb/test/test_raw.py +++ b/rpython/translator/revdb/test/test_raw.py @@ -34,6 +34,15 @@ recbar.q = foo recbar.super.p = foo + IBAR = lltype.Struct('IBAR', ('p', lltype.Ptr(FOO)), + hints={'static_immutable': True}) + ibar = lltype.malloc(IBAR, flavor='raw', immortal=True) + ibar.p = foo + + BARI = lltype.Struct('BARI', ('b', lltype.Ptr(IBAR))) + bari = lltype.malloc(BARI, flavor='raw', immortal=True) + bari.b = ibar + def main(argv): assert bar.p == foo assert baz.p == foo @@ -41,6 +50,8 @@ assert vbar[i] == foo assert recbar.q == foo assert recbar.super.p == foo + assert ibar.p == foo + assert bari.b == ibar revdb.stop_point() return 9 From pypy.commits at gmail.com Sun Jul 3 15:10:01 2016 From: pypy.commits at gmail.com (raffael_t) Date: Sun, 03 Jul 2016 12:10:01 -0700 (PDT) Subject: [pypy-commit] pypy py3.5: Bugfix - use update for dict instead of popitem, store dict class as variable, more precise exception handling with better messages Message-ID: <57796309.6237c20a.5ad17.73c6@mx.google.com> Author: Raffael Tfirst Branch: py3.5 Changeset: r85527:f4f98e487091 Date: 2016-07-03 21:09 +0200 http://bitbucket.org/pypy/pypy/changeset/f4f98e487091/ Log: Bugfix - use update for dict instead of popitem, store dict class as variable, more precise exception handling with better messages diff --git a/pypy/interpreter/pyopcode.py b/pypy/interpreter/pyopcode.py --- a/pypy/interpreter/pyopcode.py +++ b/pypy/interpreter/pyopcode.py @@ -1356,18 +1356,19 @@ def BUILD_MAP_UNPACK_WITH_CALL(self, itemcount, next_instr): num_maps = itemcount & 0xff w_dict = self.space.newdict() + dict_class = w_dict.__class__ for i in range(num_maps, 0, -1): w_item = self.peekvalue(i-1) - if not issubclass(w_item.__class__, self.space.newdict().__class__): + if not issubclass(w_item.__class__, dict_class): raise oefmt(self.space.w_TypeError, - "%s is not a mapping", w_item.__class__.__name__) + "'%T' object is not a mapping", w_item) num_items = w_item.length() + keys = w_item.w_keys() for j in range(num_items): - (w_key, w_value) = w_item.popitem() - if self.space.is_true(self.space.contains(w_dict,w_key)): + if self.space.is_true(self.space.contains(w_dict,keys.getitem(0))): raise oefmt(self.space.w_TypeError, - "got multiple values for keyword argument %s", self.space.unicode_w(w_key)) - self.space.setitem(w_dict, w_key, w_value) + "got multiple values for keyword argument %s", self.space.unicode_w(keys.getitem(0))) + self.space.call_method(w_dict, 'update', w_item) while num_maps != 0: self.popvalue() num_maps -= 1 @@ -1375,15 +1376,13 @@ def BUILD_MAP_UNPACK(self, itemcount, next_instr): w_dict = self.space.newdict() + dict_class = w_dict.__class__ for i in range(itemcount, 0, -1): w_item = self.peekvalue(i-1) - if not issubclass(w_item.__class__, self.space.newdict().__class__): + if not issubclass(w_item.__class__, dict_class): raise oefmt(self.space.w_TypeError, - "%s is not a mapping", w_item.__class__.__name__) - num_items = w_item.length() - for j in range(num_items): - (w_key, w_value) = w_item.popitem() - self.space.setitem(w_dict, w_key, w_value) + "'%T' object is not a mapping", w_item) + self.space.call_method(w_dict, 'update', w_item) while itemcount != 0: self.popvalue() itemcount -= 1 From pypy.commits at gmail.com Mon Jul 4 03:58:43 2016 From: pypy.commits at gmail.com (arigo) Date: Mon, 04 Jul 2016 00:58:43 -0700 (PDT) Subject: [pypy-commit] pypy reverse-debugger: Don't implement what I wrote (it's a mess). Instead rely on us getting Message-ID: <577a1733.e409c20a.a597d.ffffb5b9@mx.google.com> Author: Armin Rigo Branch: reverse-debugger Changeset: r85528:a142591d6ca2 Date: 2016-07-04 10:00 +0200 http://bitbucket.org/pypy/pypy/changeset/a142591d6ca2/ Log: Don't implement what I wrote (it's a mess). Instead rely on us getting the same addresses as before. Check that by recording in the log file the addresses of a random function and a random struct. diff --git a/rpython/translator/c/node.py b/rpython/translator/c/node.py --- a/rpython/translator/c/node.py +++ b/rpython/translator/c/node.py @@ -459,23 +459,24 @@ self.implementationtypename = db.gettype( T, varlength=self.getvarlength()) parent, parentindex = parentlink(obj) - mangled = False + ## mangled = False if obj in exports.EXPORTS_obj2name: self.name = exports.EXPORTS_obj2name[obj] self.globalcontainer = 2 # meh elif parent is None: self.name = db.namespace.uniquename('g_' + self.basename()) self.globalcontainer = True - if db.reverse_debugger and T._gckind != 'gc': - from rpython.translator.revdb import gencsupp - mangled = gencsupp.mangle_name_prebuilt_raw(db, self, T) + ## if db.reverse_debugger and T._gckind != 'gc': + ## from rpython.translator.revdb import gencsupp + ## mangled = gencsupp.mangle_name_prebuilt_raw(db, self, T) else: self.globalcontainer = False parentnode = db.getcontainernode(parent) defnode = db.gettypedefnode(parentnode.getTYPE()) self.name = defnode.access_expr(parentnode.name, parentindex) if self.typename != self.implementationtypename: - if db.gettypedefnode(T).extra_union_for_varlength and not mangled: + if db.gettypedefnode(T).extra_union_for_varlength: + ## and not mangled: self.name += '.b' self._funccodegen_owner = None @@ -497,8 +498,8 @@ def get_declaration(self): name = self.name - if name.startswith('RPY_RDB_A(') and name.endswith(')'): - name = name[len('RPY_RDB_A('):-1] + ## if name.startswith('RPY_RDB_A(') and name.endswith(')'): + ## name = name[len('RPY_RDB_A('):-1] if name[-2:] == '.b': # xxx fish fish assert self.implementationtypename.startswith('struct ') @@ -802,11 +803,14 @@ expr = db.get(value) if typeOf(value) is Void: comma = '' - elif expr.startswith('(&RPY_RDB_A('): - # can't use this in static initialization code - assert db.reverse_debugger - db.late_initializations.append(('%s' % access_expr, expr)) - expr = 'NULL /* patched later with %s */' % (expr,) + ## elif expr.startswith('(&RPY_RDB_A('): + ## # can't use this in static initialization code if we + ## # are inside a GC struct or a static_immutable struct. + ## # (It is not needed inside other raw structs, but we + ## # don't try to optimize that here.) + ## assert db.reverse_debugger + ## db.late_initializations.append(('%s' % access_expr, expr)) + ## expr = 'NULL /* patched later with %s */' % (expr,) expr += comma i = expr.find('\n') if i < 0: diff --git a/rpython/translator/revdb/gencsupp.py b/rpython/translator/revdb/gencsupp.py --- a/rpython/translator/revdb/gencsupp.py +++ b/rpython/translator/revdb/gencsupp.py @@ -9,37 +9,27 @@ # have the "same" content as they had during recording. More precisely, # we divide all variables according to their type in two categories: # -# * "moving things", whose value during recording is bitwise different -# from their value during replaying; +# * non-GC pointers, whose values are bitwise identical. # -# * "fixed things", whose values are bitwise identical. +# * GC pointers: these things are "moving", in the sense that the +# bitwise value of a GC pointer during recording is different from +# its bitwise value during replaying. The object at the new +# address contains the "same" content as the original one, using +# this definition of "same". # -# Moving things are: +# Note that most non-GC pointers are not followed during replaying: we +# never call external libraries nor call malloc to get pieces of raw +# memory. Writes to raw memory are ignored, and reads return a value +# recorded in the log file. However, the following are still needed: # -# * GC pointers. During replaying they point to locally-allocated -# memory that is an object with the "same" content as during -# recording; +# * function pointers pointing to RPython functions; # -# * pointers to RPython functions; +# * "static-immutable" structs like vtables. # -# * pointers to structures with the "static_immutable" hint, like -# vtables. -# -# Fixed things are the rest: -# -# * integers, floats; -# -# * most raw pointers, which during replaying will thus point to -# nonsense. (This pointer is not used during replaying to -# read/write memory: any write is ignored, and any read has its -# result recorded in the log.) -# -# Note an issue with prebuilt raw pointers to fixed things (i.e. all -# constants in the C sources that appear either inside the code or -# inside "static_immutable" or prebuilt GC structures). During -# replaying, they must correspond to bitwise the same value as during -# recording, and not to the local-process address of the raw -# structure, which is typically different (and should never be used). +# For now. we must make sure that these are at the same address as +# they were. This is very roughly checked. On Linux, it means you +# must run with Address Space Layout Randomization disabled. This +# might be fixed in the future. # @@ -49,16 +39,16 @@ srcdir / 'revdb.c', ] -def mangle_name_prebuilt_raw(database, node, S): - if (S._gckind != 'gc' and not S._hints.get('is_excdata') - and not S._hints.get('static_immutable') - and not S._hints.get('ignore_revdb') - and not S._hints.get('gcheader')): - database.all_raw_structures.append(node) - node.name = 'RPY_RDB_A(%s)' % (node.name,) - return True - else: - return False +## def mangle_name_prebuilt_raw(database, node, S): +## if (S._gckind != 'gc' and not S._hints.get('is_excdata') +## and not S._hints.get('static_immutable') +## and not S._hints.get('ignore_revdb') +## and not S._hints.get('gcheader')): +## database.all_raw_structures.append(node) +## node.name = 'RPY_RDB_A(%s)' % (node.name,) +## return True +## else: +## return False def prepare_function(funcgen): stack_bottom = False @@ -151,13 +141,13 @@ db.get(s) db.stack_bottom_funcnames = [] - db.all_raw_structures = [] + ## db.all_raw_structures = [] def write_revdb_def_file(db, target_path): f = target_path.open('w') funcnames = sorted(db.stack_bottom_funcnames) - print >> f, "#define RDB_VERSION 0x%x" % random.randrange(0, sys.maxint) - print >> f + ## print >> f, "#define RDB_VERSION 0x%x" % random.randrange(0, sys.maxint) + ## print >> f for i, fn in enumerate(funcnames): print >> f, '#define RPY_CALLBACKLOC_%s %d' % (fn, i) print >> f @@ -169,36 +159,36 @@ else: tail = ', \\' print >> f, '\t(void *)%s%s' % (fn, tail) - print >> f + ## print >> f - def _base(name): - assert name.startswith('RPY_RDB_A(') - if name.endswith('.b'): - name = name[:-2] - name = name[len('RPY_RDB_A('):-1] - return name + ## def _base(name): + ## assert name.startswith('RPY_RDB_A(') + ## if name.endswith('.b'): + ## name = name[:-2] + ## name = name[len('RPY_RDB_A('):-1] + ## return name - rawstructs = sorted(db.all_raw_structures, key=lambda node: node.name) - print >> f, '#define RPY_RDB_A(name) (*rpy_rdb_struct.name)' - print >> f, 'struct rpy_rdb_a_s {' - for i, node in enumerate(rawstructs): - print >> f, '\t%s;' % (cdecl(node.typename, '*'+_base(node.name)),) - if not rawstructs: - print >> f, '\tchar dummy;' - print >> f, '};' - print >> f, 'RPY_EXTERN struct rpy_rdb_a_s rpy_rdb_struct;' - print >> f - print >> f, '#define RPY_RDB_STRUCT_CONTENT \\' - if not rawstructs: - print >> f, '\t0' - else: - for i, node in enumerate(rawstructs): - if i == len(rawstructs) - 1: - tail = '' - else: - tail = ', \\' - name = '&' + _base(node.name) - if node.typename != node.implementationtypename: - name = '(%s)%s' % (cdecl(node.typename, '*'), name) - print >> f, '\t%s%s' % (name, tail) + ## rawstructs = sorted(db.all_raw_structures, key=lambda node: node.name) + ## print >> f, '#define RPY_RDB_A(name) (*rpy_rdb_struct.name)' + ## print >> f, 'struct rpy_rdb_a_s {' + ## for i, node in enumerate(rawstructs): + ## print >> f, '\t%s;' % (cdecl(node.typename, '*'+_base(node.name)),) + ## if not rawstructs: + ## print >> f, '\tchar dummy;' + ## print >> f, '};' + ## print >> f, 'RPY_EXTERN struct rpy_rdb_a_s rpy_rdb_struct;' + ## print >> f + ## print >> f, '#define RPY_RDB_STRUCT_CONTENT \\' + ## if not rawstructs: + ## print >> f, '\t0' + ## else: + ## for i, node in enumerate(rawstructs): + ## if i == len(rawstructs) - 1: + ## tail = '' + ## else: + ## tail = ', \\' + ## name = '&' + _base(node.name) + ## if node.typename != node.implementationtypename: + ## name = '(%s)%s' % (cdecl(node.typename, '*'), name) + ## print >> f, '\t%s%s' % (name, tail) f.close() diff --git a/rpython/translator/revdb/src-revdb/revdb.c b/rpython/translator/revdb/src-revdb/revdb.c --- a/rpython/translator/revdb/src-revdb/revdb.c +++ b/rpython/translator/revdb/src-revdb/revdb.c @@ -22,6 +22,7 @@ #include "src-revdb/revdb_include.h" #define RDB_SIGNATURE "RevDB:" +#define RDB_VERSION 0x00FF0003 #define WEAKREF_AFTERWARDS_DEAD ((char)0xf2) #define WEAKREF_AFTERWARDS_ALIVE ((char)0xeb) @@ -36,7 +37,8 @@ typedef struct { Signed version; uint64_t reserved1, reserved2; - unsigned int size_rdb_struct; + void *ptr1, *ptr2; + int reversed3; int argc; char **argv; } rdb_header_t; @@ -164,13 +166,14 @@ memset(&h, 0, sizeof(h)); h.version = RDB_VERSION; - h.size_rdb_struct = sizeof(rpy_rdb_struct); + h.ptr1 = &rpy_reverse_db_stop_point; + h.ptr2 = &rpy_revdb; h.argc = argc; h.argv = argv; write_all((const char *)&h, sizeof(h)); /* write the whole content of rpy_rdb_struct */ - write_all((const char *)&rpy_rdb_struct, sizeof(rpy_rdb_struct)); + /*write_all((const char *)&rpy_rdb_struct, sizeof(rpy_rdb_struct));*/ fprintf(stderr, "PID %d: recording revdb log to '%s'\n", (int)getpid(), filename); @@ -665,8 +668,17 @@ (long)h.version, (long)RDB_VERSION); exit(1); } - if (h.size_rdb_struct != sizeof(rpy_rdb_struct)) { - fprintf(stderr, "bad size_rdb_struct\n"); + if (h.ptr1 != &rpy_reverse_db_stop_point || + h.ptr2 != &rpy_revdb) { + fprintf(stderr, + "\n" + "In the replaying process, the addresses are different than\n" + "in the recording process. We don't support this case for\n" + "now, sorry. On Linux, check if Address Space Layout\n" + "Randomization (ADSL) is enabled, and disable it with:\n" + "\n" + " echo 0 | sudo tee /proc/sys/kernel/randomize_va_space\n" + "\n"); exit(1); } *argc_p = h.argc; @@ -683,7 +695,7 @@ } /* read the whole content of rpy_rdb_struct */ - read_all((char *)&rpy_rdb_struct, sizeof(rpy_rdb_struct)); + /*read_all((char *)&rpy_rdb_struct, sizeof(rpy_rdb_struct));*/ rpy_revdb.buf_p = rpy_rev_buffer; rpy_revdb.buf_limit = rpy_rev_buffer; @@ -1344,10 +1356,6 @@ fq_trigger(); } -struct rpy_rdb_a_s rpy_rdb_struct = { - RPY_RDB_STRUCT_CONTENT /* macro from revdb_def.h */ -}; - static void *callbacklocs[] = { RPY_CALLBACKLOCS /* macro from revdb_def.h */ }; diff --git a/rpython/translator/revdb/src-revdb/revdb_include.h b/rpython/translator/revdb/src-revdb/revdb_include.h --- a/rpython/translator/revdb/src-revdb/revdb_include.h +++ b/rpython/translator/revdb/src-revdb/revdb_include.h @@ -29,7 +29,7 @@ RPY_EXTERN void rpy_reverse_db_setup(int *argc_p, char **argv_p[]); RPY_EXTERN void rpy_reverse_db_teardown(void); -#if 1 /* enable to print locations to stderr of all the EMITs */ +#if 0 /* enable to print locations to stderr of all the EMITs */ # define _RPY_REVDB_PRINT(mode, _e) \ fprintf(stderr, \ "%s:%d: %0*llx\n", \ @@ -37,7 +37,7 @@ ((unsigned long long)_e) & ((2ULL << (8*sizeof(_e)-1)) - 1)) #endif -#if 1 /* enable to print all mallocs to stderr */ +#if 0 /* enable to print all mallocs to stderr */ RPY_EXTERN void seeing_uid(uint64_t uid); # define _RPY_REVDB_PRUID() \ seeing_uid(uid); \ diff --git a/rpython/translator/revdb/test/test_basic.py b/rpython/translator/revdb/test/test_basic.py --- a/rpython/translator/revdb/test/test_basic.py +++ b/rpython/translator/revdb/test/test_basic.py @@ -16,20 +16,19 @@ class RDB(object): def __init__(self, filename, expected_argv): with open(filename, 'rb') as f: - header = f.readline() self.buffer = f.read() - assert header == 'RevDB:\t' + '\t'.join(expected_argv) + '\n' + self.cur = self.buffer.index('\x00') + 1 + header = self.buffer[:self.cur] + assert header == 'RevDB:\t' + '\t'.join(expected_argv) + '\n\x00' # - self.cur = 0 - x = self.read1('c'); assert x == '\x00' - x = self.read1('P'); #assert x == ...random version number... + x = self.read1('P'); assert x == 0x00FF0003 x = self.read1('P'); assert x == 0 x = self.read1('P'); assert x == 0 - size_rdb_struct = self.read1('I') + x = self.read1('P'); #assert x == &rpy_reverse_db_stop_point + x = self.read1('P'); #assert x == &rpy_revdb + x = self.read1('i'); assert x == 0 self.argc = self.read1('i') self.argv = self.read1('P') - self.rdb_struct = self.buffer[self.cur : self.cur + size_rdb_struct] - self.cur += size_rdb_struct self.current_packet_end = self.cur self.read_check_argv(expected_argv) diff --git a/rpython/translator/revdb/test/test_raw.py b/rpython/translator/revdb/test/test_raw.py --- a/rpython/translator/revdb/test/test_raw.py +++ b/rpython/translator/revdb/test/test_raw.py @@ -43,6 +43,13 @@ bari = lltype.malloc(BARI, flavor='raw', immortal=True) bari.b = ibar + class X: + pass + x = X() + x.foo = foo + x.ibar = ibar + x.bari = bari + def main(argv): assert bar.p == foo assert baz.p == foo @@ -52,22 +59,29 @@ assert recbar.super.p == foo assert ibar.p == foo assert bari.b == ibar + assert x.foo == foo + assert x.ibar == ibar + assert x.bari == bari revdb.stop_point() return 9 compile(cls, main, backendopt=False) run(cls, '') rdb = fetch_rdb(cls, [cls.exename]) - assert len(rdb.rdb_struct) >= 4 + #assert len(rdb.rdb_struct) >= 4 def test_replaying_raw(self): # This tiny test seems to always have foo at the same address # in multiple runs. Here we recompile with different options # just to change that address. - subprocess.check_call(["make", "clean"], - cwd=os.path.dirname(str(self.exename))) - subprocess.check_call(["make", "lldebug"], - cwd=os.path.dirname(str(self.exename))) + # + # NOTE: not supported right now! The executable must be + # exactly the same one with the same raw addresses. This + # might be fixed in the future. + #subprocess.check_call(["make", "clean"], + # cwd=os.path.dirname(str(self.exename))) + #subprocess.check_call(["make", "lldebug"], + # cwd=os.path.dirname(str(self.exename))) # child = self.replay() child.send(Message(CMD_FORWARD, 2)) From pypy.commits at gmail.com Mon Jul 4 04:06:27 2016 From: pypy.commits at gmail.com (arigo) Date: Mon, 04 Jul 2016 01:06:27 -0700 (PDT) Subject: [pypy-commit] pypy reverse-debugger: fix acronym Message-ID: <577a1903.05261c0a.bee6a.ffff90c2@mx.google.com> Author: Armin Rigo Branch: reverse-debugger Changeset: r85529:13ee7857c97e Date: 2016-07-04 10:07 +0200 http://bitbucket.org/pypy/pypy/changeset/13ee7857c97e/ Log: fix acronym diff --git a/rpython/translator/revdb/src-revdb/revdb.c b/rpython/translator/revdb/src-revdb/revdb.c --- a/rpython/translator/revdb/src-revdb/revdb.c +++ b/rpython/translator/revdb/src-revdb/revdb.c @@ -675,7 +675,7 @@ "In the replaying process, the addresses are different than\n" "in the recording process. We don't support this case for\n" "now, sorry. On Linux, check if Address Space Layout\n" - "Randomization (ADSL) is enabled, and disable it with:\n" + "Randomization (ASLR) is enabled, and disable it with:\n" "\n" " echo 0 | sudo tee /proc/sys/kernel/randomize_va_space\n" "\n"); From pypy.commits at gmail.com Mon Jul 4 04:30:46 2016 From: pypy.commits at gmail.com (arigo) Date: Mon, 04 Jul 2016 01:30:46 -0700 (PDT) Subject: [pypy-commit] pypy reverse-debugger: Silence warnings Message-ID: <577a1eb6.c445c20a.ff3cd.ffffbf47@mx.google.com> Author: Armin Rigo Branch: reverse-debugger Changeset: r85530:1997376fdb5d Date: 2016-07-04 10:32 +0200 http://bitbucket.org/pypy/pypy/changeset/1997376fdb5d/ Log: Silence warnings diff --git a/rpython/translator/c/gc.py b/rpython/translator/c/gc.py --- a/rpython/translator/c/gc.py +++ b/rpython/translator/c/gc.py @@ -236,7 +236,7 @@ if sys.platform == 'win32': pass # yield 'assert(GC_all_interior_pointers == 0);' else: - yield 'GC_all_interior_pointers = 0;' + yield 'GC_set_all_interior_pointers(0);' yield 'boehm_gc_startup_code();' def get_real_weakref_type(self): diff --git a/rpython/translator/c/src/mem.c b/rpython/translator/c/src/mem.c --- a/rpython/translator/c/src/mem.c +++ b/rpython/translator/c/src/mem.c @@ -117,8 +117,8 @@ void boehm_gc_startup_code(void) { GC_init(); - GC_finalizer_notifier = &boehm_gc_finalizer_notifier; - GC_finalize_on_demand = 1; + GC_set_finalizer_notifier(&boehm_gc_finalizer_notifier); + GC_set_finalize_on_demand(1); GC_set_warn_proc(mem_boehm_ignore); } From pypy.commits at gmail.com Mon Jul 4 05:03:41 2016 From: pypy.commits at gmail.com (plan_rich) Date: Mon, 04 Jul 2016 02:03:41 -0700 (PDT) Subject: [pypy-commit] pypy s390x-old-cpu-no-jit: (s390x) add a config switch, not supported cpu revisions build a no jit version. #2318 Message-ID: <577a266d.8210c20a.e28a3.fffffaa3@mx.google.com> Author: Richard Plangger Branch: s390x-old-cpu-no-jit Changeset: r85531:3a21d6a5c5bc Date: 2016-07-04 11:02 +0200 http://bitbucket.org/pypy/pypy/changeset/3a21d6a5c5bc/ Log: (s390x) add a config switch, not supported cpu revisions build a no jit version. #2318 diff --git a/pypy/goal/targetpypystandalone.py b/pypy/goal/targetpypystandalone.py --- a/pypy/goal/targetpypystandalone.py +++ b/pypy/goal/targetpypystandalone.py @@ -12,6 +12,7 @@ from pypy import pypydir from rpython.rlib import rthread from pypy.module.thread import os_thread +from platform import machine thisdir = py.path.local(__file__).dirpath() @@ -276,6 +277,17 @@ elif config.objspace.usemodules.pypyjit: config.translation.jit = True + # s390x build a no jit version on older CPUs + if config.translation.jit and machine() == 's390x': + from rpython.translator.platform.arch.s390x import s390x_cpu_revision + cpu_revision = s390x_cpu_revision() + if cpu_revision == "unknown": + # this CPU might be very old! Thus we turn off + # the JIT, ve + config.translation.jit = False + print >> sys.stderr, "(s390x) turning off JIT. " \ + "CPU revision is not supported" + if config.translation.sandbox: config.objspace.lonepycfiles = False diff --git a/rpython/translator/platform/arch/s390x.py b/rpython/translator/platform/arch/s390x.py --- a/rpython/translator/platform/arch/s390x.py +++ b/rpython/translator/platform/arch/s390x.py @@ -80,7 +80,7 @@ # gcc does not recognize z13 as a compiler flag! revision = 'zEC12' - assert revision != 'unknown' - cflags += ('-march='+revision,) + if revision != 'unknown': + cflags += ('-march='+revision,) cflags += ('-m64','-mzarch') return cflags From pypy.commits at gmail.com Mon Jul 4 06:07:18 2016 From: pypy.commits at gmail.com (arigo) Date: Mon, 04 Jul 2016 03:07:18 -0700 (PDT) Subject: [pypy-commit] pypy reverse-debugger: Fix: thread-local values are usually read directly (threadlocalref_get), Message-ID: <577a3556.45061c0a.d0579.37bc@mx.google.com> Author: Armin Rigo Branch: reverse-debugger Changeset: r85532:c2bd5822be84 Date: 2016-07-04 12:08 +0200 http://bitbucket.org/pypy/pypy/changeset/c2bd5822be84/ Log: Fix: thread-local values are usually read directly (threadlocalref_get), but written with threadlocalref_addr()+op_raw_store(), which is not redone. diff --git a/rpython/rlib/rthread.py b/rpython/rlib/rthread.py --- a/rpython/rlib/rthread.py +++ b/rpython/rlib/rthread.py @@ -324,8 +324,7 @@ def get_or_make_raw(): if we_are_translated(): _threadlocalref_seeme(self) - addr = llop.threadlocalref_addr(llmemory.Address) - return llop.raw_load(FIELDTYPE, addr, offset) + return llop.threadlocalref_load(FIELDTYPE, offset) else: return getattr(self.local, 'rawvalue', zero) @@ -333,8 +332,7 @@ def setraw(value): if we_are_translated(): _threadlocalref_seeme(self) - addr = llop.threadlocalref_addr(llmemory.Address) - llop.raw_store(lltype.Void, addr, offset, value) + llop.threadlocalref_store(lltype.Void, offset, value) else: self.local.rawvalue = value diff --git a/rpython/rtyper/llinterp.py b/rpython/rtyper/llinterp.py --- a/rpython/rtyper/llinterp.py +++ b/rpython/rtyper/llinterp.py @@ -981,6 +981,11 @@ return self.op_raw_load(RESTYPE, _address_of_thread_local(), offset) op_threadlocalref_get.need_result_type = True + op_threadlocalref_load = op_threadlocalref_get + + def op_threadlocalref_store(self, offset, value): + self.op_raw_store(_address_of_thread_local(), offset, value) + def op_threadlocalref_acquire(self, prev): raise NotImplementedError def op_threadlocalref_release(self, prev): diff --git a/rpython/rtyper/lltypesystem/lloperation.py b/rpython/rtyper/lltypesystem/lloperation.py --- a/rpython/rtyper/lltypesystem/lloperation.py +++ b/rpython/rtyper/lltypesystem/lloperation.py @@ -542,6 +542,8 @@ 'threadlocalref_addr': LLOp(), # get (or make) addr of tl 'threadlocalref_get': LLOp(sideeffects=False), # read field (no check) + 'threadlocalref_load': LLOp(sideeffects=False), # read field (with check) + 'threadlocalref_store': LLOp(), # write field (with check) 'threadlocalref_acquire': LLOp(), # lock for enum 'threadlocalref_release': LLOp(), # lock for enum 'threadlocalref_enum': LLOp(sideeffects=False), # enum all threadlocalrefs diff --git a/rpython/translator/c/funcgen.py b/rpython/translator/c/funcgen.py --- a/rpython/translator/c/funcgen.py +++ b/rpython/translator/c/funcgen.py @@ -963,8 +963,8 @@ return None # use the default def OP_THREADLOCALREF_GET(self, op): - typename = self.db.gettype(op.result.concretetype) if isinstance(op.args[0], Constant): + typename = self.db.gettype(op.result.concretetype) assert isinstance(op.args[0].value, CDefinedIntSymbolic) fieldname = op.args[0].value.expr assert fieldname.startswith('RPY_TLOFS_') @@ -974,7 +974,19 @@ cdecl(typename, ''), fieldname) else: - return 'OP_THREADLOCALREF_GET_NONCONST(%s, %s, %s);' % ( - cdecl(typename, ''), - self.expr(op.args[0]), - self.expr(op.result)) + # this is used for the fall-back path in the JIT + return self.OP_THREADLOCALREF_LOAD(op) + + def OP_THREADLOCALREF_LOAD(self, op): + typename = self.db.gettype(op.result.concretetype) + return 'OP_THREADLOCALREF_LOAD(%s, %s, %s);' % ( + cdecl(typename, ''), + self.expr(op.args[0]), + self.expr(op.result)) + + def OP_THREADLOCALREF_STORE(self, op): + typename = self.db.gettype(op.args[1].concretetype) + return 'OP_THREADLOCALREF_STORE(%s, %s, %s);' % ( + cdecl(typename, ''), + self.expr(op.args[0]), + self.expr(op.args[1])) diff --git a/rpython/translator/c/src/threadlocal.h b/rpython/translator/c/src/threadlocal.h --- a/rpython/translator/c/src/threadlocal.h +++ b/rpython/translator/c/src/threadlocal.h @@ -115,13 +115,19 @@ RPY_EXTERN pthread_key_t pypy_threadlocal_key; -/* only for the fall-back path in the JIT */ -#define OP_THREADLOCALREF_GET_NONCONST(RESTYPE, offset, r) \ +#define OP_THREADLOCALREF_LOAD(RESTYPE, offset, r) \ do { \ char *a; \ OP_THREADLOCALREF_ADDR(a); \ r = *(RESTYPE *)(a + offset); \ } while (0) +#define OP_THREADLOCALREF_STORE(VALTYPE, offset, value) \ + do { \ + char *a; \ + OP_THREADLOCALREF_ADDR(a); \ + *(VALTYPE *)(a + offset) = value; \ + } while (0) + #endif /* _SRC_THREADLOCAL_H */ From pypy.commits at gmail.com Mon Jul 4 06:36:45 2016 From: pypy.commits at gmail.com (arigo) Date: Mon, 04 Jul 2016 03:36:45 -0700 (PDT) Subject: [pypy-commit] pypy default: Windows: can call spawnv() if it is called in C '_spawnv' and process.h Message-ID: <577a3c3d.0c091c0a.442dd.ffff9bb0@mx.google.com> Author: Armin Rigo Branch: Changeset: r85533:7c031204dc57 Date: 2016-07-04 12:38 +0200 http://bitbucket.org/pypy/pypy/changeset/7c031204dc57/ Log: Windows: can call spawnv() if it is called in C '_spawnv' and process.h is included (thanks cheery) diff --git a/rpython/rlib/rposix.py b/rpython/rlib/rposix.py --- a/rpython/rlib/rposix.py +++ b/rpython/rlib/rposix.py @@ -220,7 +220,7 @@ pass if _WIN32: - includes = ['io.h', 'sys/utime.h', 'sys/types.h'] + includes = ['io.h', 'sys/utime.h', 'sys/types.h', 'process.h'] libraries = [] else: if sys.platform.startswith(('darwin', 'netbsd', 'openbsd')): @@ -704,10 +704,10 @@ c_execve = external('execve', [rffi.CCHARP, rffi.CCHARPP, rffi.CCHARPP], rffi.INT, save_err=rffi.RFFI_SAVE_ERRNO) -c_spawnv = external('spawnv', +c_spawnv = external(UNDERSCORE_ON_WIN32 + 'spawnv', [rffi.INT, rffi.CCHARP, rffi.CCHARPP], rffi.INT, save_err=rffi.RFFI_SAVE_ERRNO) -c_spawnve = external('spawnve', +c_spawnve = external(UNDERSCORE_ON_WIN32 + 'spawnve', [rffi.INT, rffi.CCHARP, rffi.CCHARPP, rffi.CCHARPP], rffi.INT, save_err=rffi.RFFI_SAVE_ERRNO) From pypy.commits at gmail.com Mon Jul 4 07:06:23 2016 From: pypy.commits at gmail.com (arigo) Date: Mon, 04 Jul 2016 04:06:23 -0700 (PDT) Subject: [pypy-commit] pypy reverse-debugger: Tweak tweak tweak Message-ID: <577a432f.a459c20a.c1e3d.ffffc4d9@mx.google.com> Author: Armin Rigo Branch: reverse-debugger Changeset: r85534:8abe003a37d0 Date: 2016-07-04 13:07 +0200 http://bitbucket.org/pypy/pypy/changeset/8abe003a37d0/ Log: Tweak tweak tweak diff --git a/pypy/interpreter/reverse_debugging.py b/pypy/interpreter/reverse_debugging.py --- a/pypy/interpreter/reverse_debugging.py +++ b/pypy/interpreter/reverse_debugging.py @@ -12,6 +12,7 @@ class DBState: extend_syntax_with_dollar_num = False + standard_code = True breakpoint_stack_id = 0 breakpoint_funcnames = None printed_objects = {} @@ -187,22 +188,35 @@ return lineno +class NonStandardCode(object): + def __enter__(self): + dbstate.standard_code = False + self.c = dbstate.space.actionflag._ticker_cache + def __exit__(self, *args): + dbstate.space.actionflag._ticker_cache = self.c + dbstate.standard_code = True +non_standard_code = NonStandardCode() + + def stop_point_at_start_of_line(): if revdb.watch_save_state(): any_watch_point = False space = dbstate.space - for prog, watch_id, expected in dbstate.watch_progs: - any_watch_point = True - try: - if _run_watch(space, prog) != expected: + with non_standard_code: + for prog, watch_id, expected in dbstate.watch_progs: + any_watch_point = True + try: + if _run_watch(space, prog) != expected: + break + except Exception: break - except OperationError: - break - else: - watch_id = -1 + else: + watch_id = -1 revdb.watch_restore_state(any_watch_point) if watch_id != -1: revdb.breakpoint(watch_id) + elif not dbstate.standard_code: + return revdb.stop_point() @@ -312,38 +326,40 @@ if frame is None: return space = dbstate.space - try: - prepare_print_environment(space) - code = compile(expression, 'single') + with non_standard_code: try: - code.exec_code(space, - frame.get_w_globals(), - frame.getdictscope()) + prepare_print_environment(space) + code = compile(expression, 'single') + try: + code.exec_code(space, + frame.get_w_globals(), + frame.getdictscope()) - except OperationError as operationerr: - # can't use sys.excepthook: it will likely try to do 'import - # traceback', which might not be doable without using I/O - tb = operationerr.get_traceback() - if tb is not None: - revdb.send_output("Traceback (most recent call last):\n") - while tb is not None: - if not isinstance(tb, pytraceback.PyTraceback): - revdb.send_output(" ??? %s\n" % tb) - break - show_frame(tb.frame, tb.get_lineno(), indent=' ') - tb = tb.next - revdb.send_output('%s\n' % operationerr.errorstr(space)) + except OperationError as operationerr: + # can't use sys.excepthook: it will likely try to do 'import + # traceback', which might not be doable without using I/O + tb = operationerr.get_traceback() + if tb is not None: + revdb.send_output("Traceback (most recent call last):\n") + while tb is not None: + if not isinstance(tb, pytraceback.PyTraceback): + revdb.send_output(" ??? %s\n" % tb) + break + show_frame(tb.frame, tb.get_lineno(), indent=' ') + tb = tb.next + revdb.send_output('%s\n' % operationerr.errorstr(space)) - # set the sys.last_xxx attributes - w_type = operationerr.w_type - w_value = operationerr.get_w_value(space) - w_tb = space.wrap(operationerr.get_traceback()) - space.setitem(space.sys.w_dict, space.wrap('last_type'), w_type) - space.setitem(space.sys.w_dict, space.wrap('last_value'), w_value) - space.setitem(space.sys.w_dict, space.wrap('last_traceback'), w_tb) + # set the sys.last_xxx attributes + w_type = operationerr.w_type + w_value = operationerr.get_w_value(space) + w_tb = space.wrap(operationerr.get_traceback()) + w_dict = space.sys.w_dict + space.setitem(w_dict, space.wrap('last_type'), w_type) + space.setitem(w_dict, space.wrap('last_value'), w_value) + space.setitem(w_dict, space.wrap('last_traceback'), w_tb) - except OperationError as e: - revdb.send_output('%s\n' % e.errorstr(space, use_repr=True)) + except OperationError as e: + revdb.send_output('%s\n' % e.errorstr(space, use_repr=True)) lambda_print = lambda: command_print @@ -522,13 +538,14 @@ def command_checkwatch(cmd, marshalled_code): space = dbstate.space - try: - code = interp_marshal.loads(space, space.wrap(marshalled_code)) - text = _run_watch(space, code) - except OperationError as e: - revdb.send_watch(e.errorstr(space), ok_flag=0) - else: - revdb.send_watch(text, ok_flag=1) + with non_standard_code: + try: + code = interp_marshal.loads(space, space.wrap(marshalled_code)) + text = _run_watch(space, code) + except OperationError as e: + revdb.send_watch(e.errorstr(space), ok_flag=0) + else: + revdb.send_watch(text, ok_flag=1) lambda_checkwatch = lambda: command_checkwatch @@ -575,7 +592,8 @@ def _update_ticker_from_signals(self): from rpython.rlib import rsignal - if rsignal.pypysig_check_and_reset(): - self.rearm_ticker() + if dbstate.standard_code: + if rsignal.pypysig_check_and_reset(): + self.rearm_ticker() return self._SIG_TICKER_COUNT _update_ticker_from_signals._dont_inline_ = True From pypy.commits at gmail.com Mon Jul 4 08:05:37 2016 From: pypy.commits at gmail.com (arigo) Date: Mon, 04 Jul 2016 05:05:37 -0700 (PDT) Subject: [pypy-commit] pypy reverse-debugger: tweaks, watchpoints still crash in pypy Message-ID: <577a5111.a758c20a.2fb20.ffffcc3c@mx.google.com> Author: Armin Rigo Branch: reverse-debugger Changeset: r85535:62906323fc67 Date: 2016-07-04 14:06 +0200 http://bitbucket.org/pypy/pypy/changeset/62906323fc67/ Log: tweaks, watchpoints still crash in pypy diff --git a/pypy/interpreter/executioncontext.py b/pypy/interpreter/executioncontext.py --- a/pypy/interpreter/executioncontext.py +++ b/pypy/interpreter/executioncontext.py @@ -494,6 +494,7 @@ """The normal class for space.actionflag. The signal module provides a different one.""" _ticker = 0 + _ticker_count = -1 # xxx only for reverse_debugger.py def get_ticker(self): return self._ticker diff --git a/pypy/interpreter/reverse_debugging.py b/pypy/interpreter/reverse_debugging.py --- a/pypy/interpreter/reverse_debugging.py +++ b/pypy/interpreter/reverse_debugging.py @@ -191,9 +191,11 @@ class NonStandardCode(object): def __enter__(self): dbstate.standard_code = False - self.c = dbstate.space.actionflag._ticker_cache + self.t = dbstate.space.actionflag._ticker + self.c = dbstate.space.actionflag._ticker_count def __exit__(self, *args): - dbstate.space.actionflag._ticker_cache = self.c + dbstate.space.actionflag._ticker = self.t + dbstate.space.actionflag._ticker_count = self.c dbstate.standard_code = True non_standard_code = NonStandardCode() @@ -206,17 +208,18 @@ for prog, watch_id, expected in dbstate.watch_progs: any_watch_point = True try: - if _run_watch(space, prog) != expected: - break + got = _run_watch(space, prog) + except OperationError as e: + got = e.errorstr(space) except Exception: break + if got != expected: + break else: watch_id = -1 revdb.watch_restore_state(any_watch_point) if watch_id != -1: revdb.breakpoint(watch_id) - elif not dbstate.standard_code: - return revdb.stop_point() @@ -566,17 +569,17 @@ # bytecodes, at the expense of not reacting to signals instantly. _SIG_TICKER_COUNT = 100 - _ticker_cache = 0 + _ticker = 0 _ticker_count = _SIG_TICKER_COUNT * 10 def get_ticker(self): - return self._ticker_cache + return self._ticker def reset_ticker(self, value): - self._ticker_cache = value + self._ticker = value def rearm_ticker(self): - self._ticker_cache = -1 + self._ticker = -1 def decrement_ticker(self, by): if we_are_translated(): @@ -588,7 +591,7 @@ print ("RDBSignalActionFlag: has_bytecode_counter: " "not supported for now") raise NotImplementedError - return self._ticker_cache + return self._ticker def _update_ticker_from_signals(self): from rpython.rlib import rsignal diff --git a/pypy/module/signal/__init__.py b/pypy/module/signal/__init__.py --- a/pypy/module/signal/__init__.py +++ b/pypy/module/signal/__init__.py @@ -46,10 +46,7 @@ space.check_signal_action = interp_signal.CheckSignalAction(space) space.actionflag.register_periodic_action(space.check_signal_action, use_bytecode_counter=False) - if space.config.translation.reverse_debugger: - from pypy.interpreter.reverse_debugging import RDBSignalActionFlag - space.actionflag.__class__ = RDBSignalActionFlag - else: + if not space.config.translation.reverse_debugger: space.actionflag.__class__ = interp_signal.SignalActionFlag # xxx yes I know the previous line is a hack diff --git a/rpython/translator/revdb/interact.py b/rpython/translator/revdb/interact.py --- a/rpython/translator/revdb/interact.py +++ b/rpython/translator/revdb/interact.py @@ -35,6 +35,7 @@ if last_time != previous_time: print self.pgroup.update_watch_values() + last_time = self.pgroup.get_current_time() if self.print_extra_pending_info: print self.print_extra_pending_info self.print_extra_pending_info = None diff --git a/rpython/translator/revdb/process.py b/rpython/translator/revdb/process.py --- a/rpython/translator/revdb/process.py +++ b/rpython/translator/revdb/process.py @@ -399,7 +399,13 @@ self.active.breakpoints_cache = self.all_breakpoints.duplicate() def update_watch_values(self): - self._update_watchpoints_uids() + try: + self._update_watchpoints_uids() + except socket.error as e: + print >> sys.stderr, "socket.error: %s" % (e,) + print >> sys.stderr, "restarting at position 1" + self.jump_in_time(1) + self._update_watchpoints_uids() seen = set() for num, name in self.all_breakpoints.num2break.items(): if name.startswith('W'): @@ -507,9 +513,9 @@ except IndexError: continue if skip_futures and uid >= self.get_currently_created_objects(): - print >> sys.stderr, ( - "note: '$%d' refers to an object that is " - "only created later in time" % nid) + #print >> sys.stderr, ( + # "note: '$%d' refers to an object that is " + # "only created later in time" % nid) continue uids.append(uid) return uids From pypy.commits at gmail.com Mon Jul 4 09:30:34 2016 From: pypy.commits at gmail.com (plan_rich) Date: Mon, 04 Jul 2016 06:30:34 -0700 (PDT) Subject: [pypy-commit] pypy ppc-vsx-support: vectorized reduction test now passes (ppc) Message-ID: <577a64fa.a459c20a.c1e3d.ffffff9a@mx.google.com> Author: Richard Plangger Branch: ppc-vsx-support Changeset: r85536:1f00adc7b0fd Date: 2016-07-04 15:29 +0200 http://bitbucket.org/pypy/pypy/changeset/1f00adc7b0fd/ Log: vectorized reduction test now passes (ppc) diff --git a/rpython/jit/backend/ppc/codebuilder.py b/rpython/jit/backend/ppc/codebuilder.py --- a/rpython/jit/backend/ppc/codebuilder.py +++ b/rpython/jit/backend/ppc/codebuilder.py @@ -713,6 +713,8 @@ vperm = VA(4, XO10=43) vsel = VA(4, XO10=42) vspltisb = VXI(4, XO8=780) + vspltisw = VXI(4, XO8=844) + vspltisw = VXI(4, XO8=908) VX_splat = Form("ivrT", "ivrB", "ivrA", "XO8") vspltb = VX_splat(4, XO8=524) diff --git a/rpython/jit/backend/ppc/condition.py b/rpython/jit/backend/ppc/condition.py --- a/rpython/jit/backend/ppc/condition.py +++ b/rpython/jit/backend/ppc/condition.py @@ -6,6 +6,10 @@ GE = 5 SO = 6 NS = 7 +VEQ = 8 +VEQI = 9 +VNE = 10 +VNEI = 11 cond_none = -1 # invalid def negate(cond): @@ -19,6 +23,8 @@ assert negate(GE) == LT assert negate(SO) == NS assert negate(NS) == SO +assert negate(VEQ) == VEQI +assert negate(VNE) == VNEI encoding = [ (2, 12), # EQ @@ -29,4 +35,8 @@ (0, 4), # GE (3, 12), # SO (3, 4), # NS + (24, 12), # VEQ + (24, 4), # VEQI + (26, 12), # VNE + (26, 4), # VNEI ] diff --git a/rpython/jit/backend/ppc/ppc_assembler.py b/rpython/jit/backend/ppc/ppc_assembler.py --- a/rpython/jit/backend/ppc/ppc_assembler.py +++ b/rpython/jit/backend/ppc/ppc_assembler.py @@ -1369,7 +1369,6 @@ allocation. This needs remapping which is done here for both normal registers and accumulation registers. """ - import pdb; pdb.set_trace() asminfo, bridge_faildescr, version, looptoken = target assert isinstance(bridge_faildescr, ResumeGuardDescr) assert isinstance(faildescr, ResumeGuardDescr) @@ -1385,7 +1384,6 @@ # if accumulation is saved at the guard, we need to update it here! guard_locs = self.rebuild_faillocs_from_descr(faildescr, version.inputargs) bridge_locs = self.rebuild_faillocs_from_descr(bridge_faildescr, version.inputargs) - #import pdb; pdb.set_trace() guard_accum_info = faildescr.rd_vector_info # O(n**2), but usually you only have at most 1 fail argument while guard_accum_info: diff --git a/rpython/jit/backend/ppc/vector_ext.py b/rpython/jit/backend/ppc/vector_ext.py --- a/rpython/jit/backend/ppc/vector_ext.py +++ b/rpython/jit/backend/ppc/vector_ext.py @@ -34,7 +34,18 @@ # "propagate it between this operation and the next guard by keeping # it in the cc". In the uncommon case, result_loc is another # register, and we emit a load from the cc into this register. + + # Possibly invert the bit in the CR + bit, invert = c.encoding[condition] + assert 24 <= bit <= 27 + if invert == 12: + pass + elif invert == 4: + asm.mc.crnor(bit, bit, bit) + else: + assert 0 assert asm.guard_success_cc == c.cond_none + # if result_loc is r.SPP: asm.guard_success_cc = condition else: @@ -386,6 +397,7 @@ self.mc.vcmpgtuwx(resloc.value, argloc.value, tmp) elif size == 8: self.mc.vcmpgtudx(resloc.value, argloc.value, tmp) + flush_vec_cc(self, regalloc, c.VNEI, op.bytesize, resloc) def emit_vec_float_eq(self, op, arglocs, regalloc): resloc, loc1, loc2, sizeloc = arglocs @@ -404,7 +416,7 @@ else: notimplemented("[ppc/assembler] float == for size %d" % size) self.mc.lvx(resloc.value, off, r.SP.value) - flush_vec_cc(self, regalloc, c.EQ, op.bytesize, resloc) + flush_vec_cc(self, regalloc, c.VEQI, op.bytesize, resloc) def emit_vec_float_xor(self, op, arglocs, regalloc): resloc, l0, l1, sizeloc = arglocs @@ -432,7 +444,7 @@ res = resloc.value self.mc.lvx(res, off, r.SP.value) self.mc.vnor(res, res, res) # complement - flush_vec_cc(self, regalloc, c.NE, op.bytesize, resloc) + flush_vec_cc(self, regalloc, c.VNEI, op.bytesize, resloc) def emit_vec_cast_int_to_float(self, op, arglocs, regalloc): res, l0 = arglocs @@ -455,7 +467,7 @@ self.mc.vcmpequwx(res.value, l0.value, l1.value) elif size == 8: self.mc.vcmpequdx(res.value, l0.value, l1.value) - flush_vec_cc(self, regalloc, c.EQ, op.bytesize, res) + flush_vec_cc(self, regalloc, c.VEQI, op.bytesize, res) def emit_vec_int_ne(self, op, arglocs, regalloc): res, l0, l1, sizeloc = arglocs @@ -471,7 +483,7 @@ elif size == 8: self.mc.vcmpequdx(res.value, res.value, tmp) self.mc.vnor(res.value, res.value, res.value) - flush_vec_cc(self, regalloc, c.NE, op.bytesize, res) + flush_vec_cc(self, regalloc, c.VEQI, op.bytesize, res) def emit_vec_expand_f(self, op, arglocs, regalloc): resloc, srcloc = arglocs @@ -549,7 +561,7 @@ if size == 8: if srcloc.is_vector_reg(): # reg <- vector assert not resloc.is_vector_reg() - self.mc.load_imm(r.SCRATCH, PARAM_SAVE_AREA_OFFSET) + self.mc.load_imm(r.SCRATCH2, PARAM_SAVE_AREA_OFFSET) self.mc.stvx(src, r.SCRATCH2.value, r.SP.value) self.mc.load(res, r.SP.value, PARAM_SAVE_AREA_OFFSET+8*idx) else: @@ -621,6 +633,17 @@ else: return self.ivrm.force_allocate_reg(op, forbidden_vars) + def force_allocate_vector_reg_or_cc(self, op): + assert op.type == INT + if self.next_op_can_accept_cc(self.operations, self.rm.position): + # hack: return the SPP location to mean "lives in CC". This + # SPP will not actually be used, and the location will be freed + # after the next op as usual. + self.rm.force_allocate_frame_reg(op) + return r.SPP + else: + return self.force_allocate_vector_reg(op) + def ensure_vector_reg(self, box): if box.type == FLOAT: return self.vrm.make_sure_var_in_reg(box, @@ -691,14 +714,25 @@ prepare_vec_int_and = prepare_vec_arith prepare_vec_int_or = prepare_vec_arith prepare_vec_int_xor = prepare_vec_arith - - prepare_vec_float_eq = prepare_vec_arith - prepare_vec_float_ne = prepare_vec_arith - prepare_vec_int_eq = prepare_vec_arith - prepare_vec_int_ne = prepare_vec_arith prepare_vec_float_xor = prepare_vec_arith del prepare_vec_arith + def prepare_vec_bool(self, op): + a0 = op.getarg(0) + a1 = op.getarg(1) + assert isinstance(op, VectorOp) + size = op.bytesize + args = op.getarglist() + loc0 = self.ensure_vector_reg(a0) + loc1 = self.ensure_vector_reg(a1) + resloc = self.force_allocate_vector_reg_or_cc(op) + return [resloc, loc0, loc1, imm(size)] + + prepare_vec_float_eq = prepare_vec_bool + prepare_vec_float_ne = prepare_vec_bool + prepare_vec_int_eq = prepare_vec_bool + prepare_vec_int_ne = prepare_vec_bool + del prepare_vec_bool def prepare_vec_store(self, op): descr = op.getdescr() @@ -826,7 +860,7 @@ arg = op.getarg(0) assert isinstance(arg, VectorOp) argloc = self.ensure_vector_reg(arg) - resloc = self.force_allocate_vector_reg(op) + resloc = self.force_allocate_vector_reg_or_cc(op) return [resloc, argloc, imm(arg.bytesize)] def _prepare_vec(self, op): @@ -843,19 +877,11 @@ prepare_vec_cast_int_to_float = prepare_vec_cast_float_to_int - def load_vector_condition_into_cc(self, box): - if self.assembler.guard_success_cc == c.cond_none: - # compare happended before - #loc = self.ensure_reg(box) - #mc = self.assembler.mc - #mc.cmp_op(0, loc.value, 0, imm=True) - self.assembler.guard_success_cc = c.NE - def prepare_vec_guard_true(self, op): - self.load_vector_condition_into_cc(op.getarg(0)) + self.assembler.guard_success_cc = c.VEQ return self._prepare_guard(op) def prepare_vec_guard_false(self, op): - self.load_vector_condition_into_cc(op.getarg(0)) + self.assembler.guard_success_cc = c.VNE return self._prepare_guard(op) diff --git a/rpython/jit/metainterp/test/test_vector.py b/rpython/jit/metainterp/test/test_vector.py --- a/rpython/jit/metainterp/test/test_vector.py +++ b/rpython/jit/metainterp/test/test_vector.py @@ -332,29 +332,29 @@ @py.test.mark.parametrize('type,func,init,insert,at,count,breaks', # all - [#(rffi.DOUBLE, lambda x: not bool(x), 1.0, None, -1,32, False), - #(rffi.DOUBLE, lambda x: x == 0.0, 1.0, None, -1,33, False), - #(rffi.DOUBLE, lambda x: x == 0.0, 1.0, 0.0, 33,34, True), - #(rffi.DOUBLE, lambda x: x == 0.0, 1.0, 0.1, 4,34, False), - #(lltype.Signed, lambda x: not bool(x), 1, None, -1,32, False), + [(rffi.DOUBLE, lambda x: not bool(x), 1.0, None, -1,32, False), + (rffi.DOUBLE, lambda x: x == 0.0, 1.0, None, -1,33, False), + (rffi.DOUBLE, lambda x: x == 0.0, 1.0, 0.0, 33,34, True), + (rffi.DOUBLE, lambda x: x == 0.0, 1.0, 0.1, 4,34, False), + (lltype.Signed, lambda x: not bool(x), 1, None, -1,32, False), (lltype.Signed, lambda x: not bool(x), 1, 0, 14,32, True), - #(lltype.Signed, lambda x: not bool(x), 1, 0, 15,31, True), - #(lltype.Signed, lambda x: not bool(x), 1, 0, 4,30, True), - #(lltype.Signed, lambda x: x == 0, 1, None, -1,33, False), - #(lltype.Signed, lambda x: x == 0, 1, 0, 33,34, True), - ## any - #(rffi.DOUBLE, lambda x: x != 0.0, 0.0, 1.0, 33,35, True), - #(rffi.DOUBLE, lambda x: x != 0.0, 0.0, 1.0, -1,36, False), - #(rffi.DOUBLE, lambda x: bool(x), 0.0, 1.0, 33,37, True), - #(rffi.DOUBLE, lambda x: bool(x), 0.0, 1.0, -1,38, False), - #(lltype.Signed, lambda x: x != 0, 0, 1, 33,35, True), - #(lltype.Signed, lambda x: x != 0, 0, 1, -1,36, False), - #(lltype.Signed, lambda x: bool(x), 0, 1, 33,37, True), - #(lltype.Signed, lambda x: bool(x), 0, 1, -1,38, False), - #(rffi.INT, lambda x: intmask(x) != 0, rffi.r_int(0), rffi.r_int(1), 33,35, True), - #(rffi.INT, lambda x: intmask(x) != 0, rffi.r_int(0), rffi.r_int(1), -1,36, False), - #(rffi.INT, lambda x: bool(intmask(x)), rffi.r_int(0), rffi.r_int(1), 33,37, True), - #(rffi.INT, lambda x: bool(intmask(x)), rffi.r_int(0), rffi.r_int(1), -1,38, False), + (lltype.Signed, lambda x: not bool(x), 1, 0, 15,31, True), + (lltype.Signed, lambda x: not bool(x), 1, 0, 4,30, True), + (lltype.Signed, lambda x: x == 0, 1, None, -1,33, False), + (lltype.Signed, lambda x: x == 0, 1, 0, 33,34, True), + # any + (rffi.DOUBLE, lambda x: x != 0.0, 0.0, 1.0, 33,35, True), + (rffi.DOUBLE, lambda x: x != 0.0, 0.0, 1.0, -1,36, False), + (rffi.DOUBLE, lambda x: bool(x), 0.0, 1.0, 33,37, True), + (rffi.DOUBLE, lambda x: bool(x), 0.0, 1.0, -1,38, False), + (lltype.Signed, lambda x: x != 0, 0, 1, 33,35, True), + (lltype.Signed, lambda x: x != 0, 0, 1, -1,36, False), + (lltype.Signed, lambda x: bool(x), 0, 1, 33,37, True), + (lltype.Signed, lambda x: bool(x), 0, 1, -1,38, False), + (rffi.INT, lambda x: intmask(x) != 0, rffi.r_int(0), rffi.r_int(1), 33,35, True), + (rffi.INT, lambda x: intmask(x) != 0, rffi.r_int(0), rffi.r_int(1), -1,36, False), + (rffi.INT, lambda x: bool(intmask(x)), rffi.r_int(0), rffi.r_int(1), 33,37, True), + (rffi.INT, lambda x: bool(intmask(x)), rffi.r_int(0), rffi.r_int(1), -1,38, False), ]) def test_bool_reduction(self, type, func, init, insert, at, count, breaks): myjitdriver = JitDriver(greens = [], reds = 'auto', vectorize=True) From pypy.commits at gmail.com Mon Jul 4 10:39:52 2016 From: pypy.commits at gmail.com (plan_rich) Date: Mon, 04 Jul 2016 07:39:52 -0700 (PDT) Subject: [pypy-commit] pypy default: (mattip, arigato, plan_rich) solve issue #2339 Message-ID: <577a7538.e809c30a.a0e32.ffffee0b@mx.google.com> Author: Richard Plangger Branch: Changeset: r85537:f24ba12ad3d2 Date: 2016-07-04 16:39 +0200 http://bitbucket.org/pypy/pypy/changeset/f24ba12ad3d2/ Log: (mattip, arigato, plan_rich) solve issue #2339 diff --git a/pypy/module/cpyext/test/test_cpyext.py b/pypy/module/cpyext/test/test_cpyext.py --- a/pypy/module/cpyext/test/test_cpyext.py +++ b/pypy/module/cpyext/test/test_cpyext.py @@ -78,7 +78,7 @@ else: libraries = [] if sys.platform.startswith('linux'): - compile_extra = ["-Werror", "-g", "-O0", "-fPIC"] + compile_extra = ["-Werror", "-g", "-O0", "-Wp,-U_FORTIFY_SOURCE", "-fPIC"] link_extra = ["-g"] else: compile_extra = link_extra = None From pypy.commits at gmail.com Mon Jul 4 11:24:42 2016 From: pypy.commits at gmail.com (plan_rich) Date: Mon, 04 Jul 2016 08:24:42 -0700 (PDT) Subject: [pypy-commit] pypy ppc-vsx-support: updates the ppc opcode for vec_int_xor (veqv was used instead of vxor) Message-ID: <577a7fba.c5ddc20a.ed641.ffff8a7b@mx.google.com> Author: Richard Plangger Branch: ppc-vsx-support Changeset: r85538:2a82f60bb1d6 Date: 2016-07-04 17:24 +0200 http://bitbucket.org/pypy/pypy/changeset/2a82f60bb1d6/ Log: updates the ppc opcode for vec_int_xor (veqv was used instead of vxor) diff --git a/rpython/jit/backend/ppc/vector_ext.py b/rpython/jit/backend/ppc/vector_ext.py --- a/rpython/jit/backend/ppc/vector_ext.py +++ b/rpython/jit/backend/ppc/vector_ext.py @@ -251,7 +251,7 @@ def emit_vec_int_xor(self, op, arglocs, regalloc): resloc, loc0, loc1, sizeloc = arglocs - self.mc.veqv(resloc.value, loc0.value, loc1.value) + self.mc.vxor(resloc.value, loc0.value, loc1.value) def emit_vec_int_signext(self, op, arglocs, regalloc): resloc, loc0 = arglocs diff --git a/rpython/jit/metainterp/test/test_vector.py b/rpython/jit/metainterp/test/test_vector.py --- a/rpython/jit/metainterp/test/test_vector.py +++ b/rpython/jit/metainterp/test/test_vector.py @@ -207,9 +207,9 @@ bits = size*8 integers = st.integers(min_value=-2**(bits-1), max_value=2**(bits-1)-1) la = data.draw(st.lists(integers, min_size=10, max_size=150)) - #la = [1,2,3,4,5,6,7,8,9,10,11,12,13] + #la = [0] * 10 #1,2,3,4,5,6,7,8,9,10,11,12,13] l = len(la) - #lb = [1,2,3,4,5,6,7,8,9,10,11,12,13] + #lb = [0] * 10 # [1,2,3,4,5,6,7,8,9,10,11,12,13] lb = data.draw(st.lists(integers, min_size=l, max_size=l)) rawstorage = RawStorage() From pypy.commits at gmail.com Mon Jul 4 11:46:39 2016 From: pypy.commits at gmail.com (raffael_t) Date: Mon, 04 Jul 2016 08:46:39 -0700 (PDT) Subject: [pypy-commit] pypy py3.5: Check key type in map_unpack_with_call, return function name in error messages Message-ID: <577a84df.c8a61c0a.40e82.ffff86fa@mx.google.com> Author: Raffael Tfirst Branch: py3.5 Changeset: r85539:5242a11c98d3 Date: 2016-07-04 17:46 +0200 http://bitbucket.org/pypy/pypy/changeset/5242a11c98d3/ Log: Check key type in map_unpack_with_call, return function name in error messages diff --git a/pypy/interpreter/pyopcode.py b/pypy/interpreter/pyopcode.py --- a/pypy/interpreter/pyopcode.py +++ b/pypy/interpreter/pyopcode.py @@ -1352,9 +1352,18 @@ def BUILD_LIST_UNPACK(self, itemcount, next_instr): w_sum = self.unpack_helper(itemcount, next_instr) self.pushvalue(self.space.newlist(w_sum)) - + + def getFuncDesc(func): + if self.space.type(aaa).name.decode('utf-8') == 'method': + return "()" + elif self.space.type(aaa).name.decode('utf-8') == 'function': + return "()" + else: + return " object"; + def BUILD_MAP_UNPACK_WITH_CALL(self, itemcount, next_instr): num_maps = itemcount & 0xff + function_location = (itemcount>>8) & 0xff w_dict = self.space.newdict() dict_class = w_dict.__class__ for i in range(num_maps, 0, -1): @@ -1365,9 +1374,15 @@ num_items = w_item.length() keys = w_item.w_keys() for j in range(num_items): - if self.space.is_true(self.space.contains(w_dict,keys.getitem(0))): + if self.space.type(keys.getitem(j)).name.decode('utf-8') == 'method': + err_fun = self.peekvalue(num_maps + function_location-1) raise oefmt(self.space.w_TypeError, - "got multiple values for keyword argument %s", self.space.unicode_w(keys.getitem(0))) + "%N%s keywords must be strings", err_fun, getFuncDesc(err_fun)) + if self.space.is_true(self.space.contains(w_dict,keys.getitem(j))): + err_fun = self.peekvalue(num_maps + function_location-1) + err_arg = self.space.unicode_w(keys.getitem(j)) + raise oefmt(self.space.w_TypeError, + "%N%s got multiple values for keyword argument %s", err_fun, getFuncDesc(err_fun), err_arg) self.space.call_method(w_dict, 'update', w_item) while num_maps != 0: self.popvalue() From pypy.commits at gmail.com Mon Jul 4 11:52:47 2016 From: pypy.commits at gmail.com (raffael_t) Date: Mon, 04 Jul 2016 08:52:47 -0700 (PDT) Subject: [pypy-commit] pypy py3.5-async: (copy unpack fixes from py3.5 branch) Message-ID: <577a864f.ca9c1c0a.114b2.fffff66c@mx.google.com> Author: Raffael Tfirst Branch: py3.5-async Changeset: r85540:1232cebac106 Date: 2016-07-04 17:52 +0200 http://bitbucket.org/pypy/pypy/changeset/1232cebac106/ Log: (copy unpack fixes from py3.5 branch) diff --git a/pypy/interpreter/pyopcode.py b/pypy/interpreter/pyopcode.py --- a/pypy/interpreter/pyopcode.py +++ b/pypy/interpreter/pyopcode.py @@ -1353,21 +1353,37 @@ w_sum = self.unpack_helper(itemcount, next_instr) self.pushvalue(self.space.newlist(w_sum)) + def getFuncDesc(func): + if self.space.type(aaa).name.decode('utf-8') == 'method': + return "()" + elif self.space.type(aaa).name.decode('utf-8') == 'function': + return "()" + else: + return " object"; + def BUILD_MAP_UNPACK_WITH_CALL(self, itemcount, next_instr): num_maps = itemcount & 0xff + function_location = (itemcount>>8) & 0xff w_dict = self.space.newdict() + dict_class = w_dict.__class__ for i in range(num_maps, 0, -1): w_item = self.peekvalue(i-1) - if not issubclass(w_item.__class__, self.space.newdict().__class__): + if not issubclass(w_item.__class__, dict_class): raise oefmt(self.space.w_TypeError, - "%s is not a mapping", w_item.__class__.__name__) + "'%T' object is not a mapping", w_item) num_items = w_item.length() + keys = w_item.w_keys() for j in range(num_items): - (w_key, w_value) = w_item.popitem() - if self.space.is_true(self.space.contains(w_dict,w_key)): + if self.space.type(keys.getitem(j)).name.decode('utf-8') == 'method': + err_fun = self.peekvalue(num_maps + function_location-1) raise oefmt(self.space.w_TypeError, - "got multiple values for keyword argument %s", self.space.unicode_w(w_key)) - self.space.setitem(w_dict, w_key, w_value) + "%N%s keywords must be strings", err_fun, getFuncDesc(err_fun)) + if self.space.is_true(self.space.contains(w_dict,keys.getitem(j))): + err_fun = self.peekvalue(num_maps + function_location-1) + err_arg = self.space.unicode_w(keys.getitem(j)) + raise oefmt(self.space.w_TypeError, + "%N%s got multiple values for keyword argument %s", err_fun, getFuncDesc(err_fun), err_arg) + self.space.call_method(w_dict, 'update', w_item) while num_maps != 0: self.popvalue() num_maps -= 1 @@ -1375,15 +1391,13 @@ def BUILD_MAP_UNPACK(self, itemcount, next_instr): w_dict = self.space.newdict() + dict_class = w_dict.__class__ for i in range(itemcount, 0, -1): w_item = self.peekvalue(i-1) - if not issubclass(w_item.__class__, self.space.newdict().__class__): + if not issubclass(w_item.__class__, dict_class): raise oefmt(self.space.w_TypeError, - "%s is not a mapping", w_item.__class__.__name__) - num_items = w_item.length() - for j in range(num_items): - (w_key, w_value) = w_item.popitem() - self.space.setitem(w_dict, w_key, w_value) + "'%T' object is not a mapping", w_item) + self.space.call_method(w_dict, 'update', w_item) while itemcount != 0: self.popvalue() itemcount -= 1 From pypy.commits at gmail.com Mon Jul 4 12:10:43 2016 From: pypy.commits at gmail.com (arigo) Date: Mon, 04 Jul 2016 09:10:43 -0700 (PDT) Subject: [pypy-commit] pypy reverse-debugger: Add space._side_effects_ok(). See doc. Still not working. Message-ID: <577a8a83.cdcf1c0a.b7f13.49ef@mx.google.com> Author: Armin Rigo Branch: reverse-debugger Changeset: r85541:d0a272e97616 Date: 2016-07-04 18:12 +0200 http://bitbucket.org/pypy/pypy/changeset/d0a272e97616/ Log: Add space._side_effects_ok(). See doc. Still not working. diff --git a/pypy/interpreter/astcompiler/codegen.py b/pypy/interpreter/astcompiler/codegen.py --- a/pypy/interpreter/astcompiler/codegen.py +++ b/pypy/interpreter/astcompiler/codegen.py @@ -1199,7 +1199,7 @@ def visit_RevDBMetaVar(self, node): if self.space.config.translation.reverse_debugger: from pypy.interpreter.reverse_debugging import dbstate - if dbstate.extend_syntax_with_dollar_num: + if not dbstate.standard_code: self.emit_op_arg(ops.LOAD_REVDB_VAR, node.metavar) return self.error("$NUM is only valid in the reverse-debugger", node) diff --git a/pypy/interpreter/astcompiler/test/test_compiler.py b/pypy/interpreter/astcompiler/test/test_compiler.py --- a/pypy/interpreter/astcompiler/test/test_compiler.py +++ b/pypy/interpreter/astcompiler/test/test_compiler.py @@ -944,10 +944,11 @@ from pypy.interpreter.reverse_debugging import dbstate, setup_revdb self.space.config.translation.reverse_debugger = True setup_revdb(self.space) - dbstate.extend_syntax_with_dollar_num = True + dbstate.standard_code = False dbstate.metavars = [self.space.wrap(6)] self.simple_test("x = 7*$0", "x", 42) - dbstate.extend_syntax_with_dollar_num = False + dbstate.standard_code = True + self.error_test("x = 7*$0", SyntaxError) class AppTestCompiler: diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py --- a/pypy/interpreter/baseobjspace.py +++ b/pypy/interpreter/baseobjspace.py @@ -811,7 +811,8 @@ w_s1 = self.interned_strings.get(s) if w_s1 is None: w_s1 = w_s - self.interned_strings.set(s, w_s1) + if self._side_effects_ok(): + self.interned_strings.set(s, w_s1) return w_s1 def new_interned_str(self, s): @@ -820,9 +821,30 @@ w_s1 = self.interned_strings.get(s) if w_s1 is None: w_s1 = self.wrap(s) - self.interned_strings.set(s, w_s1) + if self._side_effects_ok(): + self.interned_strings.set(s, w_s1) return w_s1 + def _side_effects_ok(self): + # For the reverse debugger: we run compiled watchpoint + # expressions in a fast way that will crash if they have + # side-effects. The obvious Python code with side-effects is + # documented "don't do that"; but some non-obvious side + # effects are also common, like interning strings (from + # unmarshalling the code object containing the watchpoint + # expression) to the two attribute caches in mapdict.py and + # typeobject.py. For now, we have to identify such places + # that are not acceptable for "reasonable" read-only + # watchpoint expressions, and write: + # + # if not space._side_effects_ok(): + # don't cache. + # + if self.config.translation.reverse_debugger: + from pypy.interpreter.reverse_debugging import dbstate + return dbstate.standard_code + return True + def is_interned_str(self, s): # interface for marshal_impl if not we_are_translated(): diff --git a/pypy/interpreter/reverse_debugging.py b/pypy/interpreter/reverse_debugging.py --- a/pypy/interpreter/reverse_debugging.py +++ b/pypy/interpreter/reverse_debugging.py @@ -11,7 +11,6 @@ class DBState: - extend_syntax_with_dollar_num = False standard_code = True breakpoint_stack_id = 0 breakpoint_funcnames = None @@ -258,12 +257,8 @@ def compile(source, mode): space = dbstate.space compiler = space.createcompiler() - dbstate.extend_syntax_with_dollar_num = True - try: - code = compiler.compile(source, '', mode, 0, - hidden_applevel=True) - finally: - dbstate.extend_syntax_with_dollar_num = False + code = compiler.compile(source, '', mode, 0, + hidden_applevel=True) return code @@ -526,17 +521,16 @@ def command_compilewatch(cmd, expression): space = dbstate.space - try: - code = compile(expression, 'eval') - # Note: using version 0 to marshal watchpoints, in order to - # avoid space.new_interned_str() on unmarshal. This is - # forbidden because it comes with lasting side-effects. - marshalled_code = space.str_w(interp_marshal.dumps( - space, space.wrap(code), space.wrap(0))) - except OperationError as e: - revdb.send_watch(e.errorstr(space), ok_flag=0) - else: - revdb.send_watch(marshalled_code, ok_flag=1) + with non_standard_code: + try: + code = compile(expression, 'eval') + marshalled_code = space.str_w(interp_marshal.dumps( + space, space.wrap(code), + space.wrap(interp_marshal.Py_MARSHAL_VERSION))) + except OperationError as e: + revdb.send_watch(e.errorstr(space), ok_flag=0) + else: + revdb.send_watch(marshalled_code, ok_flag=1) lambda_compilewatch = lambda: command_compilewatch def command_checkwatch(cmd, marshalled_code): diff --git a/pypy/objspace/std/mapdict.py b/pypy/objspace/std/mapdict.py --- a/pypy/objspace/std/mapdict.py +++ b/pypy/objspace/std/mapdict.py @@ -94,12 +94,13 @@ cache.hits[name] = cache.hits.get(name, 0) + 1 return attr attr = self._find_map_attr(name, index) - cache.attrs[attr_hash] = self - cache.names[attr_hash] = name - cache.indexes[attr_hash] = index - cache.cached_attrs[attr_hash] = attr - if space.config.objspace.std.withmethodcachecounter: - cache.misses[name] = cache.misses.get(name, 0) + 1 + if space._side_effects_ok(): + cache.attrs[attr_hash] = self + cache.names[attr_hash] = name + cache.indexes[attr_hash] = index + cache.cached_attrs[attr_hash] = attr + if space.config.objspace.std.withmethodcachecounter: + cache.misses[name] = cache.misses.get(name, 0) + 1 return attr def _find_map_attr(self, name, index): @@ -932,6 +933,8 @@ @jit.dont_look_inside def _fill_cache(pycode, nameindex, map, version_tag, storageindex, w_method=None): + if not pycode.space._side_effects_ok(): + return entry = pycode._mapdict_caches[nameindex] if entry is INVALID_CACHE_ENTRY: entry = CacheEntry() diff --git a/pypy/objspace/std/test/test_dictmultiobject.py b/pypy/objspace/std/test/test_dictmultiobject.py --- a/pypy/objspace/std/test/test_dictmultiobject.py +++ b/pypy/objspace/std/test/test_dictmultiobject.py @@ -1100,6 +1100,9 @@ def fromcache(self, cls): return cls(self) + def _side_effects_ok(self): + return True + w_StopIteration = StopIteration w_None = None w_NoneType = type(None, None) diff --git a/pypy/objspace/std/typeobject.py b/pypy/objspace/std/typeobject.py --- a/pypy/objspace/std/typeobject.py +++ b/pypy/objspace/std/typeobject.py @@ -248,7 +248,8 @@ # if it was not actually overriden in the class, we remember this # fact for the next time. if w_descr is object_getattribute(self.space): - self.uses_object_getattribute = True + if self.space._side_effects_ok(): + self.uses_object_getattribute = True else: return w_descr return None @@ -275,10 +276,12 @@ self.lookup('__cmp__') or self.lookup('__hash__') is not default_hash) if overrides_eq_cmp_or_hash: - self.compares_by_identity_status = OVERRIDES_EQ_CMP_OR_HASH + result = OVERRIDES_EQ_CMP_OR_HASH else: - self.compares_by_identity_status = COMPARES_BY_IDENTITY - return self.compares_by_identity_status == COMPARES_BY_IDENTITY + result = COMPARES_BY_IDENTITY + if self.space._side_effects_ok(): + self.compares_by_identity_status = result + return result == COMPARES_BY_IDENTITY def ready(self): for w_base in self.bases_w: @@ -454,11 +457,12 @@ # print "hit", self, name return tup tup = self._lookup_where_all_typeobjects(name) - cache.versions[method_hash] = version_tag - cache.names[method_hash] = name - cache.lookup_where[method_hash] = tup - if space.config.objspace.std.withmethodcachecounter: - cache.misses[name] = cache.misses.get(name, 0) + 1 + if space._side_effects_ok(): + cache.versions[method_hash] = version_tag + cache.names[method_hash] = name + cache.lookup_where[method_hash] = tup + if space.config.objspace.std.withmethodcachecounter: + cache.misses[name] = cache.misses.get(name, 0) + 1 # print "miss", self, name return tup @@ -608,7 +612,7 @@ self) w_newfunc = space.get(w_newdescr, self) if (space.config.objspace.std.newshortcut and - not we_are_jitted() and + not we_are_jitted() and space._side_effects_ok() and isinstance(w_newtype, W_TypeObject)): self.w_new_function = w_newfunc w_newobject = space.call_obj_args(w_newfunc, self, __args__) From pypy.commits at gmail.com Mon Jul 4 12:50:48 2016 From: pypy.commits at gmail.com (arigo) Date: Mon, 04 Jul 2016 09:50:48 -0700 (PDT) Subject: [pypy-commit] pypy reverse-debugger: Next fix Message-ID: <577a93e8.a758c20a.2fb20.3d9c@mx.google.com> Author: Armin Rigo Branch: reverse-debugger Changeset: r85542:79bf6fc209bb Date: 2016-07-04 18:52 +0200 http://bitbucket.org/pypy/pypy/changeset/79bf6fc209bb/ Log: Next fix diff --git a/pypy/interpreter/reverse_debugging.py b/pypy/interpreter/reverse_debugging.py --- a/pypy/interpreter/reverse_debugging.py +++ b/pypy/interpreter/reverse_debugging.py @@ -458,14 +458,15 @@ dbstate.breakpoint_stack_id = cmd.c_arg1 funcnames = None watch_progs = [] - for i, kind, name in revdb.split_breakpoints_arg(extra): - if kind == 'B': - if funcnames is None: - funcnames = {} - funcnames[name] = i - elif kind == 'W': - code = interp_marshal.loads(space, space.wrap(name)) - watch_progs.append((code, i, '')) + with non_standard_code: + for i, kind, name in revdb.split_breakpoints_arg(extra): + if kind == 'B': + if funcnames is None: + funcnames = {} + funcnames[name] = i + elif kind == 'W': + code = interp_marshal.loads(space, space.wrap(name)) + watch_progs.append((code, i, '')) dbstate.breakpoint_funcnames = funcnames dbstate.watch_progs = watch_progs[:] lambda_breakpoints = lambda: command_breakpoints diff --git a/pypy/module/__pypy__/__init__.py b/pypy/module/__pypy__/__init__.py --- a/pypy/module/__pypy__/__init__.py +++ b/pypy/module/__pypy__/__init__.py @@ -91,6 +91,7 @@ 'interp_magic.save_module_content_for_future_reload', 'decode_long' : 'interp_magic.decode_long', '_promote' : 'interp_magic._promote', + 'side_effects_ok' : 'interp_magic.side_effects_ok', } if sys.platform == 'win32': interpleveldefs['get_console_cp'] = 'interp_magic.get_console_cp' diff --git a/pypy/module/__pypy__/interp_magic.py b/pypy/module/__pypy__/interp_magic.py --- a/pypy/module/__pypy__/interp_magic.py +++ b/pypy/module/__pypy__/interp_magic.py @@ -188,3 +188,19 @@ else: jit.promote(w_obj) return w_obj + +def side_effects_ok(space): + """For use with the reverse-debugger: this function normally returns + True, but will return False if we are evaluating a debugging command + like a watchpoint. You are responsible for not doing any side effect + at all (including no caching) when evaluating watchpoints. This + function is meant to help a bit---you can write: + + if not __pypy__.side_effects_ok(): + skip the caching logic + + inside getter methods or properties, to make them usable from + watchpoints. Note that you need to re-run ``PYPYRDB=.. pypy'' + after changing the Python code. + """ + return space.wrap(space._side_effects_ok()) diff --git a/rpython/translator/revdb/process.py b/rpython/translator/revdb/process.py --- a/rpython/translator/revdb/process.py +++ b/rpython/translator/revdb/process.py @@ -136,6 +136,7 @@ breakpoints_cache=self.breakpoints_cache, printed_objects=self.printed_objects) other.expect_ready() + #print >> sys.stderr, 'CLONED', self.current_time return other def close(self): From pypy.commits at gmail.com Mon Jul 4 13:01:21 2016 From: pypy.commits at gmail.com (arigo) Date: Mon, 04 Jul 2016 10:01:21 -0700 (PDT) Subject: [pypy-commit] pypy reverse-debugger: Add some "assert not standard_code". Add a "with non_standard_code" in Message-ID: <577a9661.c8a61c0a.40e82.ffffa350@mx.google.com> Author: Armin Rigo Branch: reverse-debugger Changeset: r85543:1e996f85a807 Date: 2016-07-04 19:02 +0200 http://bitbucket.org/pypy/pypy/changeset/1e996f85a807/ Log: Add some "assert not standard_code". Add a "with non_standard_code" in command_locals(). I think that it's not actually needed here and in command_print(), because these commands make the process tainted, but better safe than sorry. diff --git a/pypy/interpreter/reverse_debugging.py b/pypy/interpreter/reverse_debugging.py --- a/pypy/interpreter/reverse_debugging.py +++ b/pypy/interpreter/reverse_debugging.py @@ -84,8 +84,8 @@ # When we see a jump backward, we set 'f_revdb_nextline_instr' in # such a way that the next instruction, at 'jumpto', will trigger # stop_point_at_start_of_line(). We have to trigger it even if - # 'jumpto' is not actually a start of line. For example, in a - # 'while foo: body', the body ends with a JUMP_ABSOLUTE which + # 'jumpto' is not actually a start of line. For example, after a + # 'while foo:', the body ends with a JUMP_ABSOLUTE which # jumps back to the *second* opcode of the while. frame.f_revdb_nextline_instr = jumpto @@ -248,13 +248,17 @@ def fetch_cur_frame(): - ec = dbstate.space.getexecutioncontext() - frame = ec.topframeref() + ec = dbstate.space.threadlocals.get_ec() + if ec is None: + frame = None + else: + frame = ec.topframeref() if frame is None: revdb.send_output("No stack.\n") return frame def compile(source, mode): + assert not dbstate.standard_code space = dbstate.space compiler = space.createcompiler() code = compiler.compile(source, '', mode, 0, @@ -269,6 +273,7 @@ self.space = space def descr_write(self, w_buffer): + assert not dbstate.standard_code space = self.space if space.isinstance_w(w_buffer, space.w_unicode): w_buffer = space.call_method(w_buffer, 'encode', @@ -293,6 +298,7 @@ for non-prebuilt objects. Such objects are then recorded in 'printed_objects'. """ + assert not dbstate.standard_code if space.is_w(w_obj, space.w_None): return uid = revdb.get_unique_id(w_obj) @@ -313,6 +319,7 @@ def prepare_print_environment(space): + assert not dbstate.standard_code w_revdb_output = space.wrap(W_RevDBOutput(space)) w_displayhook = get_revdb_displayhook(space) space.sys.setdictvalue(space, 'stdout', w_revdb_output) @@ -430,26 +437,27 @@ if frame is None: return space = dbstate.space - try: - prepare_print_environment(space) - space.appexec([space.wrap(space.sys), - frame.getdictscope()], """(sys, locals): - lst = locals.keys() - lst.sort() - print 'Locals:' - for key in lst: - try: - print ' %s =' % key, - s = '%r' % locals[key] - if len(s) > 140: - s = s[:100] + '...' + s[-30:] - print s - except: - exc, val, tb = sys.exc_info() - print '!<%s: %r>' % (exc, val) - """) - except OperationError as e: - revdb.send_output('%s\n' % e.errorstr(space, use_repr=True)) + with non_standard_code: + try: + prepare_print_environment(space) + space.appexec([space.wrap(space.sys), + frame.getdictscope()], """(sys, locals): + lst = locals.keys() + lst.sort() + print 'Locals:' + for key in lst: + try: + print ' %s =' % key, + s = '%r' % locals[key] + if len(s) > 140: + s = s[:100] + '...' + s[-30:] + print s + except: + exc, val, tb = sys.exc_info() + print '!<%s: %r>' % (exc, val) + """) + except OperationError as e: + revdb.send_output('%s\n' % e.errorstr(space, use_repr=True)) lambda_locals = lambda: command_locals @@ -548,6 +556,7 @@ def _run_watch(space, prog): + # must be called from non_standard_code! w_dict = space.builtin.w_dict w_res = prog.exec_code(space, w_dict, w_dict) return space.str_w(space.repr(w_res)) From pypy.commits at gmail.com Mon Jul 4 13:17:59 2016 From: pypy.commits at gmail.com (raffael_t) Date: Mon, 04 Jul 2016 10:17:59 -0700 (PDT) Subject: [pypy-commit] pypy py3.5-async: Generate ast for async Message-ID: <577a9a47.c61f1c0a.3a0bb.ffffdc19@mx.google.com> Author: Raffael Tfirst Branch: py3.5-async Changeset: r85544:367468a1d202 Date: 2016-07-04 19:17 +0200 http://bitbucket.org/pypy/pypy/changeset/367468a1d202/ Log: Generate ast for async diff --git a/pypy/interpreter/astcompiler/ast.py b/pypy/interpreter/astcompiler/ast.py --- a/pypy/interpreter/astcompiler/ast.py +++ b/pypy/interpreter/astcompiler/ast.py @@ -314,6 +314,8 @@ return None if space.isinstance_w(w_node, get(space).w_FunctionDef): return FunctionDef.from_object(space, w_node) + if space.isinstance_w(w_node, get(space).w_AsyncFunctionDef): + return AsyncFunctionDef.from_object(space, w_node) if space.isinstance_w(w_node, get(space).w_ClassDef): return ClassDef.from_object(space, w_node) if space.isinstance_w(w_node, get(space).w_Return): @@ -326,12 +328,16 @@ return AugAssign.from_object(space, w_node) if space.isinstance_w(w_node, get(space).w_For): return For.from_object(space, w_node) + if space.isinstance_w(w_node, get(space).w_AsyncFor): + return AsyncFor.from_object(space, w_node) if space.isinstance_w(w_node, get(space).w_While): return While.from_object(space, w_node) if space.isinstance_w(w_node, get(space).w_If): return If.from_object(space, w_node) if space.isinstance_w(w_node, get(space).w_With): return With.from_object(space, w_node) + if space.isinstance_w(w_node, get(space).w_AsyncWith): + return AsyncWith.from_object(space, w_node) if space.isinstance_w(w_node, get(space).w_Raise): return Raise.from_object(space, w_node) if space.isinstance_w(w_node, get(space).w_Try): @@ -434,6 +440,82 @@ State.ast_type('FunctionDef', 'stmt', ['name', 'args', 'body', 'decorator_list', 'returns']) +class AsyncFunctionDef(stmt): + + def __init__(self, name, args, body, decorator_list, returns, lineno, col_offset): + self.name = name + self.args = args + self.body = body + self.decorator_list = decorator_list + self.returns = returns + stmt.__init__(self, lineno, col_offset) + + def walkabout(self, visitor): + visitor.visit_AsyncFunctionDef(self) + + def mutate_over(self, visitor): + self.args = self.args.mutate_over(visitor) + if self.body: + visitor._mutate_sequence(self.body) + if self.decorator_list: + visitor._mutate_sequence(self.decorator_list) + if self.returns: + self.returns = self.returns.mutate_over(visitor) + return visitor.visit_AsyncFunctionDef(self) + + def to_object(self, space): + w_node = space.call_function(get(space).w_AsyncFunctionDef) + w_name = space.wrap(self.name.decode('utf-8')) # identifier + space.setattr(w_node, space.wrap('name'), w_name) + w_args = self.args.to_object(space) # arguments + space.setattr(w_node, space.wrap('args'), w_args) + if self.body is None: + body_w = [] + else: + body_w = [node.to_object(space) for node in self.body] # stmt + w_body = space.newlist(body_w) + space.setattr(w_node, space.wrap('body'), w_body) + if self.decorator_list is None: + decorator_list_w = [] + else: + decorator_list_w = [node.to_object(space) for node in self.decorator_list] # expr + w_decorator_list = space.newlist(decorator_list_w) + space.setattr(w_node, space.wrap('decorator_list'), w_decorator_list) + w_returns = self.returns.to_object(space) if self.returns is not None else space.w_None # expr + space.setattr(w_node, space.wrap('returns'), w_returns) + w_lineno = space.wrap(self.lineno) # int + space.setattr(w_node, space.wrap('lineno'), w_lineno) + w_col_offset = space.wrap(self.col_offset) # int + space.setattr(w_node, space.wrap('col_offset'), w_col_offset) + return w_node + + @staticmethod + def from_object(space, w_node): + w_name = get_field(space, w_node, 'name', False) + w_args = get_field(space, w_node, 'args', False) + w_body = get_field(space, w_node, 'body', False) + w_decorator_list = get_field(space, w_node, 'decorator_list', False) + w_returns = get_field(space, w_node, 'returns', True) + w_lineno = get_field(space, w_node, 'lineno', False) + w_col_offset = get_field(space, w_node, 'col_offset', False) + _name = space.identifier_w(w_name) + if _name is None: + raise_required_value(space, w_node, 'name') + _args = arguments.from_object(space, w_args) + if _args is None: + raise_required_value(space, w_node, 'args') + body_w = space.unpackiterable(w_body) + _body = [stmt.from_object(space, w_item) for w_item in body_w] + decorator_list_w = space.unpackiterable(w_decorator_list) + _decorator_list = [expr.from_object(space, w_item) for w_item in decorator_list_w] + _returns = expr.from_object(space, w_returns) if w_returns is not None else None + _lineno = space.int_w(w_lineno) + _col_offset = space.int_w(w_col_offset) + return AsyncFunctionDef(_name, _args, _body, _decorator_list, _returns, _lineno, _col_offset) + +State.ast_type('AsyncFunctionDef', 'stmt', ['name', 'args', 'body', 'decorator_list', 'returns']) + + class ClassDef(stmt): def __init__(self, name, bases, keywords, body, decorator_list, lineno, col_offset): @@ -771,6 +853,76 @@ State.ast_type('For', 'stmt', ['target', 'iter', 'body', 'orelse']) +class AsyncFor(stmt): + + def __init__(self, target, iter, body, orelse, lineno, col_offset): + self.target = target + self.iter = iter + self.body = body + self.orelse = orelse + stmt.__init__(self, lineno, col_offset) + + def walkabout(self, visitor): + visitor.visit_AsyncFor(self) + + def mutate_over(self, visitor): + self.target = self.target.mutate_over(visitor) + self.iter = self.iter.mutate_over(visitor) + if self.body: + visitor._mutate_sequence(self.body) + if self.orelse: + visitor._mutate_sequence(self.orelse) + return visitor.visit_AsyncFor(self) + + def to_object(self, space): + w_node = space.call_function(get(space).w_AsyncFor) + w_target = self.target.to_object(space) # expr + space.setattr(w_node, space.wrap('target'), w_target) + w_iter = self.iter.to_object(space) # expr + space.setattr(w_node, space.wrap('iter'), w_iter) + if self.body is None: + body_w = [] + else: + body_w = [node.to_object(space) for node in self.body] # stmt + w_body = space.newlist(body_w) + space.setattr(w_node, space.wrap('body'), w_body) + if self.orelse is None: + orelse_w = [] + else: + orelse_w = [node.to_object(space) for node in self.orelse] # stmt + w_orelse = space.newlist(orelse_w) + space.setattr(w_node, space.wrap('orelse'), w_orelse) + w_lineno = space.wrap(self.lineno) # int + space.setattr(w_node, space.wrap('lineno'), w_lineno) + w_col_offset = space.wrap(self.col_offset) # int + space.setattr(w_node, space.wrap('col_offset'), w_col_offset) + return w_node + + @staticmethod + def from_object(space, w_node): + w_target = get_field(space, w_node, 'target', False) + w_iter = get_field(space, w_node, 'iter', False) + w_body = get_field(space, w_node, 'body', False) + w_orelse = get_field(space, w_node, 'orelse', False) + w_lineno = get_field(space, w_node, 'lineno', False) + w_col_offset = get_field(space, w_node, 'col_offset', False) + _target = expr.from_object(space, w_target) + if _target is None: + raise_required_value(space, w_node, 'target') + _iter = expr.from_object(space, w_iter) + if _iter is None: + raise_required_value(space, w_node, 'iter') + body_w = space.unpackiterable(w_body) + _body = [stmt.from_object(space, w_item) for w_item in body_w] + orelse_w = space.unpackiterable(w_orelse) + _orelse = [stmt.from_object(space, w_item) for w_item in orelse_w] + _lineno = space.int_w(w_lineno) + _col_offset = space.int_w(w_col_offset) + return AsyncFor(_target, _iter, _body, _orelse, _lineno, _col_offset) + +State.ast_type('AsyncFor', 'stmt', ['target', 'iter', 'body', 'orelse']) + + class While(stmt): def __init__(self, test, body, orelse, lineno, col_offset): @@ -949,6 +1101,60 @@ State.ast_type('With', 'stmt', ['items', 'body']) +class AsyncWith(stmt): + + def __init__(self, items, body, lineno, col_offset): + self.items = items + self.body = body + stmt.__init__(self, lineno, col_offset) + + def walkabout(self, visitor): + visitor.visit_AsyncWith(self) + + def mutate_over(self, visitor): + if self.items: + visitor._mutate_sequence(self.items) + if self.body: + visitor._mutate_sequence(self.body) + return visitor.visit_AsyncWith(self) + + def to_object(self, space): + w_node = space.call_function(get(space).w_AsyncWith) + if self.items is None: + items_w = [] + else: + items_w = [node.to_object(space) for node in self.items] # withitem + w_items = space.newlist(items_w) + space.setattr(w_node, space.wrap('items'), w_items) + if self.body is None: + body_w = [] + else: + body_w = [node.to_object(space) for node in self.body] # stmt + w_body = space.newlist(body_w) + space.setattr(w_node, space.wrap('body'), w_body) + w_lineno = space.wrap(self.lineno) # int + space.setattr(w_node, space.wrap('lineno'), w_lineno) + w_col_offset = space.wrap(self.col_offset) # int + space.setattr(w_node, space.wrap('col_offset'), w_col_offset) + return w_node + + @staticmethod + def from_object(space, w_node): + w_items = get_field(space, w_node, 'items', False) + w_body = get_field(space, w_node, 'body', False) + w_lineno = get_field(space, w_node, 'lineno', False) + w_col_offset = get_field(space, w_node, 'col_offset', False) + items_w = space.unpackiterable(w_items) + _items = [withitem.from_object(space, w_item) for w_item in items_w] + body_w = space.unpackiterable(w_body) + _body = [stmt.from_object(space, w_item) for w_item in body_w] + _lineno = space.int_w(w_lineno) + _col_offset = space.int_w(w_col_offset) + return AsyncWith(_items, _body, _lineno, _col_offset) + +State.ast_type('AsyncWith', 'stmt', ['items', 'body']) + + class Raise(stmt): def __init__(self, exc, cause, lineno, col_offset): @@ -1450,6 +1656,8 @@ return DictComp.from_object(space, w_node) if space.isinstance_w(w_node, get(space).w_GeneratorExp): return GeneratorExp.from_object(space, w_node) + if space.isinstance_w(w_node, get(space).w_Await): + return Await.from_object(space, w_node) if space.isinstance_w(w_node, get(space).w_Yield): return Yield.from_object(space, w_node) if space.isinstance_w(w_node, get(space).w_YieldFrom): @@ -2037,6 +2245,44 @@ State.ast_type('GeneratorExp', 'expr', ['elt', 'generators']) +class Await(expr): + + def __init__(self, value, lineno, col_offset): + self.value = value + expr.__init__(self, lineno, col_offset) + + def walkabout(self, visitor): + visitor.visit_Await(self) + + def mutate_over(self, visitor): + self.value = self.value.mutate_over(visitor) + return visitor.visit_Await(self) + + def to_object(self, space): + w_node = space.call_function(get(space).w_Await) + w_value = self.value.to_object(space) # expr + space.setattr(w_node, space.wrap('value'), w_value) + w_lineno = space.wrap(self.lineno) # int + space.setattr(w_node, space.wrap('lineno'), w_lineno) + w_col_offset = space.wrap(self.col_offset) # int + space.setattr(w_node, space.wrap('col_offset'), w_col_offset) + return w_node + + @staticmethod + def from_object(space, w_node): + w_value = get_field(space, w_node, 'value', False) + w_lineno = get_field(space, w_node, 'lineno', False) + w_col_offset = get_field(space, w_node, 'col_offset', False) + _value = expr.from_object(space, w_value) + if _value is None: + raise_required_value(space, w_node, 'value') + _lineno = space.int_w(w_lineno) + _col_offset = space.int_w(w_col_offset) + return Await(_value, _lineno, _col_offset) + +State.ast_type('Await', 'expr', ['value']) + + class Yield(expr): def __init__(self, value, lineno, col_offset): @@ -3596,6 +3842,8 @@ return self.default_visitor(node) def visit_FunctionDef(self, node): return self.default_visitor(node) + def visit_AsyncFunctionDef(self, node): + return self.default_visitor(node) def visit_ClassDef(self, node): return self.default_visitor(node) def visit_Return(self, node): @@ -3608,12 +3856,16 @@ return self.default_visitor(node) def visit_For(self, node): return self.default_visitor(node) + def visit_AsyncFor(self, node): + return self.default_visitor(node) def visit_While(self, node): return self.default_visitor(node) def visit_If(self, node): return self.default_visitor(node) def visit_With(self, node): return self.default_visitor(node) + def visit_AsyncWith(self, node): + return self.default_visitor(node) def visit_Raise(self, node): return self.default_visitor(node) def visit_Try(self, node): @@ -3658,6 +3910,8 @@ return self.default_visitor(node) def visit_GeneratorExp(self, node): return self.default_visitor(node) + def visit_Await(self, node): + return self.default_visitor(node) def visit_Yield(self, node): return self.default_visitor(node) def visit_YieldFrom(self, node): @@ -3732,6 +3986,13 @@ if node.returns: node.returns.walkabout(self) + def visit_AsyncFunctionDef(self, node): + node.args.walkabout(self) + self.visit_sequence(node.body) + self.visit_sequence(node.decorator_list) + if node.returns: + node.returns.walkabout(self) + def visit_ClassDef(self, node): self.visit_sequence(node.bases) self.visit_sequence(node.keywords) @@ -3759,6 +4020,12 @@ self.visit_sequence(node.body) self.visit_sequence(node.orelse) + def visit_AsyncFor(self, node): + node.target.walkabout(self) + node.iter.walkabout(self) + self.visit_sequence(node.body) + self.visit_sequence(node.orelse) + def visit_While(self, node): node.test.walkabout(self) self.visit_sequence(node.body) @@ -3773,6 +4040,10 @@ self.visit_sequence(node.items) self.visit_sequence(node.body) + def visit_AsyncWith(self, node): + self.visit_sequence(node.items) + self.visit_sequence(node.body) + def visit_Raise(self, node): if node.exc: node.exc.walkabout(self) @@ -3857,6 +4128,9 @@ node.elt.walkabout(self) self.visit_sequence(node.generators) + def visit_Await(self, node): + node.value.walkabout(self) + def visit_Yield(self, node): if node.value: node.value.walkabout(self) From pypy.commits at gmail.com Mon Jul 4 13:59:00 2016 From: pypy.commits at gmail.com (arigo) Date: Mon, 04 Jul 2016 10:59:00 -0700 (PDT) Subject: [pypy-commit] pypy reverse-debugger: os._exit() should close the revdb log file Message-ID: <577aa3e4.4f8d1c0a.75f11.1eb2@mx.google.com> Author: Armin Rigo Branch: reverse-debugger Changeset: r85545:1167f883bd80 Date: 2016-07-04 20:00 +0200 http://bitbucket.org/pypy/pypy/changeset/1167f883bd80/ Log: os._exit() should close the revdb log file diff --git a/rpython/rlib/debug.py b/rpython/rlib/debug.py --- a/rpython/rlib/debug.py +++ b/rpython/rlib/debug.py @@ -158,7 +158,9 @@ def debug_flush(): - """ Flushes the debug file + """ Flushes the debug file. + + With the reverse-debugger, it also closes the output log. """ pass diff --git a/rpython/translator/c/src/debug_print.h b/rpython/translator/c/src/debug_print.h --- a/rpython/translator/c/src/debug_print.h +++ b/rpython/translator/c/src/debug_print.h @@ -36,7 +36,13 @@ #define OP_DEBUG_OFFSET(res) res = pypy_debug_offset() #define OP_DEBUG_FORKED(ofs, _) pypy_debug_forked(ofs) #define OP_HAVE_DEBUG_PRINTS(r) r = (pypy_have_debug_prints & 1) -#define OP_DEBUG_FLUSH() fflush(pypy_debug_file) + +#ifdef RPY_REVERSE_DEBUGGER +RPY_EXTERN void rpy_reverse_db_teardown(void); +# define OP_DEBUG_FLUSH() fflush(pypy_debug_file); rpy_reverse_db_teardown() +#else +# define OP_DEBUG_FLUSH() fflush(pypy_debug_file) +#endif /************************************************************/ diff --git a/rpython/translator/revdb/src-revdb/revdb.c b/rpython/translator/revdb/src-revdb/revdb.c --- a/rpython/translator/revdb/src-revdb/revdb.c +++ b/rpython/translator/revdb/src-revdb/revdb.c @@ -89,8 +89,13 @@ RPY_REVDB_EMIT(stop_points = rpy_revdb.stop_point_seen; , uint64_t _e, stop_points); - if (!RPY_RDB_REPLAY) + if (!RPY_RDB_REPLAY) { rpy_reverse_db_flush(); + if (rpy_rev_fileno >= 0) { + close(rpy_rev_fileno); + rpy_rev_fileno = -1; + } + } else check_at_end(stop_points); } From pypy.commits at gmail.com Mon Jul 4 14:31:34 2016 From: pypy.commits at gmail.com (arigo) Date: Mon, 04 Jul 2016 11:31:34 -0700 (PDT) Subject: [pypy-commit] pypy reverse-debugger: Fix location of the stop on backward watchpoints Message-ID: <577aab86.8210c20a.f82a4.00ec@mx.google.com> Author: Armin Rigo Branch: reverse-debugger Changeset: r85546:3c09a4bbcbc2 Date: 2016-07-04 20:32 +0200 http://bitbucket.org/pypy/pypy/changeset/3c09a4bbcbc2/ Log: Fix location of the stop on backward watchpoints diff --git a/rpython/translator/revdb/interact.py b/rpython/translator/revdb/interact.py --- a/rpython/translator/revdb/interact.py +++ b/rpython/translator/revdb/interact.py @@ -191,7 +191,11 @@ kind, num, name)) self.print_extra_pending_info = '\n'.join(printing) if self.pgroup.get_current_time() != b.time: - self.pgroup.jump_in_time(b.time) + target_time = b.time + if backward and any(self._bp_kind(num)[0] == 'watchpoint' + for num in b.regular_breakpoint_nums()): + target_time += 1 + self.pgroup.jump_in_time(target_time) def remove_tainting(self): if self.pgroup.is_tainted(): From pypy.commits at gmail.com Mon Jul 4 15:15:43 2016 From: pypy.commits at gmail.com (mattip) Date: Mon, 04 Jul 2016 12:15:43 -0700 (PDT) Subject: [pypy-commit] pypy call-via-pyobj: make test_binop_mul_impl fail since tp_as_number slot is not inherited properly Message-ID: <577ab5df.a251c20a.97e6d.0d59@mx.google.com> Author: Matti Picus Branch: call-via-pyobj Changeset: r85547:f987c3b171e3 Date: 2016-07-04 19:02 +0300 http://bitbucket.org/pypy/pypy/changeset/f987c3b171e3/ Log: make test_binop_mul_impl fail since tp_as_number slot is not inherited properly diff --git a/pypy/module/cpyext/test/array.c b/pypy/module/cpyext/test/array.c --- a/pypy/module/cpyext/test/array.c +++ b/pypy/module/cpyext/test/array.c @@ -1901,13 +1901,6 @@ return Py_NotImplemented; } -static PyNumberMethods array_as_number = { - (binaryfunc)NULL, /* nb_add*/ - (binaryfunc)NULL, /* nb_subtract */ - (binaryfunc)array_multiply, /* nb_multiply */ - (binaryfunc)NULL, /* nb_divide */ -}; - static PyObject* array_base_multiply(PyObject* obj1, PyObject* obj2) { @@ -1958,7 +1951,7 @@ static PyNumberMethods array_base_as_number = { (binaryfunc)NULL, /* nb_add*/ (binaryfunc)NULL, /* nb_subtract */ - (binaryfunc)array_base_multiply, /* nb_multiply */ + (binaryfunc)array_multiply, /* nb_multiply */ (binaryfunc)NULL, /* nb_divide */ }; @@ -2273,7 +2266,7 @@ 0, /* tp_setattr */ 0, /* tp_compare */ (reprfunc)array_repr, /* tp_repr */ - &array_as_number, /* tp_as_number*/ + 0, /* tp_as_number*/ &array_as_sequence, /* tp_as_sequence*/ &array_as_mapping, /* tp_as_mapping*/ 0, /* tp_hash */ From pypy.commits at gmail.com Mon Jul 4 15:15:45 2016 From: pypy.commits at gmail.com (mattip) Date: Mon, 04 Jul 2016 12:15:45 -0700 (PDT) Subject: [pypy-commit] pypy call-via-pyobj: a hack to fix the test in f987c3b171e3, not clear to me why int is special Message-ID: <577ab5e1.d48e1c0a.dc72a.ffff99ae@mx.google.com> Author: Matti Picus Branch: call-via-pyobj Changeset: r85548:3c9d24675b27 Date: 2016-07-04 22:09 +0300 http://bitbucket.org/pypy/pypy/changeset/3c9d24675b27/ Log: a hack to fix the test in f987c3b171e3, not clear to me why int is special diff --git a/pypy/module/cpyext/typeobject.py b/pypy/module/cpyext/typeobject.py --- a/pypy/module/cpyext/typeobject.py +++ b/pypy/module/cpyext/typeobject.py @@ -794,15 +794,27 @@ # While this is a hack, cpython does it as well. w_metatype = space.w_type + base = py_type.c_tp_base + bases_set = False + if base and not space.is_w(from_ref(space, base), space.w_int): + # inheriting tp_as_* slots + # set these now so the functions in add_operators() will have offsets + # XXX why is int special (inifinite recursion if these are set on + # a subclass of int, i.e. Enum in test_int_subtype in test_intobject.py) + if not py_type.c_tp_as_number: py_type.c_tp_as_number = base.c_tp_as_number + if not py_type.c_tp_as_sequence: py_type.c_tp_as_sequence = base.c_tp_as_sequence + if not py_type.c_tp_as_mapping: py_type.c_tp_as_mapping = base.c_tp_as_mapping + if not py_type.c_tp_as_buffer: py_type.c_tp_as_buffer = base.c_tp_as_buffer + bases_set = True + w_obj = space.allocate_instance(W_PyCTypeObject, w_metatype) track_reference(space, py_obj, w_obj) w_obj.__init__(space, py_type) w_obj.ready() finish_type_2(space, py_type, w_obj) - # inheriting tp_as_* slots - base = py_type.c_tp_base - if base: + if base and not bases_set: + # inheriting tp_as_* slots if not py_type.c_tp_as_number: py_type.c_tp_as_number = base.c_tp_as_number if not py_type.c_tp_as_sequence: py_type.c_tp_as_sequence = base.c_tp_as_sequence if not py_type.c_tp_as_mapping: py_type.c_tp_as_mapping = base.c_tp_as_mapping From pypy.commits at gmail.com Mon Jul 4 15:19:57 2016 From: pypy.commits at gmail.com (raffael_t) Date: Mon, 04 Jul 2016 12:19:57 -0700 (PDT) Subject: [pypy-commit] pypy py3.5-async: Change funcdef handling, return AsyncFunctionDef if needed Message-ID: <577ab6dd.aa6ec20a.f2f62.6be5@mx.google.com> Author: Raffael Tfirst Branch: py3.5-async Changeset: r85549:38271294f870 Date: 2016-07-04 21:19 +0200 http://bitbucket.org/pypy/pypy/changeset/38271294f870/ Log: Change funcdef handling, return AsyncFunctionDef if needed diff --git a/pypy/interpreter/astcompiler/astbuilder.py b/pypy/interpreter/astcompiler/astbuilder.py --- a/pypy/interpreter/astcompiler/astbuilder.py +++ b/pypy/interpreter/astcompiler/astbuilder.py @@ -475,7 +475,7 @@ return [self.handle_expr(bases_node.get_child(0))] return self.get_expression_list(bases_node) - def handle_funcdef(self, funcdef_node, decorators=None): + def handle_funcdef_impl(self, funcdef_node, is_async, decorators=None): name_node = funcdef_node.get_child(1) name = self.new_identifier(name_node.get_value()) self.check_forbidden_name(name, name_node) @@ -486,8 +486,12 @@ returns = self.handle_expr(funcdef_node.get_child(4)) suite += 2 body = self.handle_suite(funcdef_node.get_child(suite)) - return ast.FunctionDef(name, args, body, decorators, returns, - funcdef_node.get_lineno(), funcdef_node.get_column()) + if is_async: + return ast.AsyncFunctionDef(name, args, body, decorators, returns, + funcdef_node.get_lineno(), funcdef_node.get_column()) + else: + return ast.FunctionDef(name, args, body, decorators, returns, + funcdef_node.get_lineno(), funcdef_node.get_column()) def handle_decorated(self, decorated_node): decorators = self.handle_decorators(decorated_node.get_child(0)) From pypy.commits at gmail.com Mon Jul 4 15:54:56 2016 From: pypy.commits at gmail.com (raffael_t) Date: Mon, 04 Jul 2016 12:54:56 -0700 (PDT) Subject: [pypy-commit] pypy py3.5-async: Handle async_funcdef Message-ID: <577abf10.081ac20a.e187e.ffffc321@mx.google.com> Author: Raffael Tfirst Branch: py3.5-async Changeset: r85550:4b920008b181 Date: 2016-07-04 21:54 +0200 http://bitbucket.org/pypy/pypy/changeset/4b920008b181/ Log: Handle async_funcdef diff --git a/pypy/interpreter/astcompiler/astbuilder.py b/pypy/interpreter/astcompiler/astbuilder.py --- a/pypy/interpreter/astcompiler/astbuilder.py +++ b/pypy/interpreter/astcompiler/astbuilder.py @@ -493,6 +493,9 @@ return ast.FunctionDef(name, args, body, decorators, returns, funcdef_node.get_lineno(), funcdef_node.get_column()) + def handle_async_funcdef(self, node, decorators=None): + return handle_funcdef_impl(c, node.get_child(1), decorators, 1) + def handle_decorated(self, decorated_node): decorators = self.handle_decorators(decorated_node.get_child(0)) definition = decorated_node.get_child(1) From pypy.commits at gmail.com Mon Jul 4 16:00:55 2016 From: pypy.commits at gmail.com (raffael_t) Date: Mon, 04 Jul 2016 13:00:55 -0700 (PDT) Subject: [pypy-commit] pypy py3.5-async: Restore function lost because of rename (handle_funcdef without async) Message-ID: <577ac077.8323c20a.bbb2.305a@mx.google.com> Author: Raffael Tfirst Branch: py3.5-async Changeset: r85551:288c22e89431 Date: 2016-07-04 22:00 +0200 http://bitbucket.org/pypy/pypy/changeset/288c22e89431/ Log: Restore function lost because of rename (handle_funcdef without async) diff --git a/pypy/interpreter/astcompiler/astbuilder.py b/pypy/interpreter/astcompiler/astbuilder.py --- a/pypy/interpreter/astcompiler/astbuilder.py +++ b/pypy/interpreter/astcompiler/astbuilder.py @@ -494,7 +494,10 @@ funcdef_node.get_lineno(), funcdef_node.get_column()) def handle_async_funcdef(self, node, decorators=None): - return handle_funcdef_impl(c, node.get_child(1), decorators, 1) + return handle_funcdef_impl(node.get_child(1), decorators, 1) + + def handle_funcdef(self, node, decorators=None): + return handle_funcdef_impl(node, decorators, 0) def handle_decorated(self, decorated_node): decorators = self.handle_decorators(decorated_node.get_child(0)) From pypy.commits at gmail.com Tue Jul 5 02:31:38 2016 From: pypy.commits at gmail.com (cfbolz) Date: Mon, 04 Jul 2016 23:31:38 -0700 (PDT) Subject: [pypy-commit] pypy default: don't use eval on user input Message-ID: <577b544a.2237c20a.f77dc.ffffaee7@mx.google.com> Author: Carl Friedrich Bolz Branch: Changeset: r85552:7f5d1ceedd1d Date: 2016-07-05 08:29 +0200 http://bitbucket.org/pypy/pypy/changeset/7f5d1ceedd1d/ Log: don't use eval on user input diff --git a/dotviewer/graphparse.py b/dotviewer/graphparse.py --- a/dotviewer/graphparse.py +++ b/dotviewer/graphparse.py @@ -85,10 +85,11 @@ pass def splitline(line, re_word = re.compile(r'[^\s"]\S*|["]["]|["].*?[^\\]["]')): + import ast result = [] for word in re_word.findall(line): if word.startswith('"'): - word = eval(word) + word = ast.literal_eval(word) result.append(word) return result From pypy.commits at gmail.com Tue Jul 5 05:57:10 2016 From: pypy.commits at gmail.com (plan_rich) Date: Tue, 05 Jul 2016 02:57:10 -0700 (PDT) Subject: [pypy-commit] pypy default: (s390x) apply same changes as in 2269d1d, push/pop gcmap callsites/calls modified/added Message-ID: <577b8476.c445c20a.6659b.ffff9eaa@mx.google.com> Author: Richard Plangger Branch: Changeset: r85553:316ea0980c19 Date: 2016-07-05 11:55 +0200 http://bitbucket.org/pypy/pypy/changeset/316ea0980c19/ Log: (s390x) apply same changes as in 2269d1d, push/pop gcmap callsites/calls modified/added diff --git a/rpython/jit/backend/zarch/arch.py b/rpython/jit/backend/zarch/arch.py --- a/rpython/jit/backend/zarch/arch.py +++ b/rpython/jit/backend/zarch/arch.py @@ -45,10 +45,7 @@ # +------------------------------+ <- assembler begin # | SAVE CONTEXT | # +------------------------------+ -# +--+| BRANCH (saves addr of pool | -# | | in r13) | -# | +------------------------------+ -# | | ... | +#start| ... | # | | LITERAL POOL | <---+ # | | ... | <-+ | # +-->+------------------------------+ | | diff --git a/rpython/jit/backend/zarch/assembler.py b/rpython/jit/backend/zarch/assembler.py --- a/rpython/jit/backend/zarch/assembler.py +++ b/rpython/jit/backend/zarch/assembler.py @@ -309,11 +309,9 @@ # signature of this _frame_realloc_slowpath function: # * on entry, r0 is the new size - # * on entry, r1 is the gcmap # * no managed register must be modified - ofs2 = self.cpu.get_ofs_of_frame_field('jf_gcmap') - mc.STG(r.SCRATCH, l.addr(ofs2, r.SPP)) + # caller already did push_gcmap(store=True) self._push_core_regs_to_jitframe(mc, r.MANAGED_REGS) self._push_fp_regs_to_jitframe(mc) @@ -347,6 +345,7 @@ mc.load(r.r5, r.r5, 0) mc.store(r.r2, r.r5, -WORD) + self.pop_gcmap(mc) # cancel the push_gcmap(store=True) in the caller self._pop_core_regs_from_jitframe(mc, r.MANAGED_REGS) self._pop_fp_regs_from_jitframe(mc) @@ -411,6 +410,7 @@ reg is not r.r4 and reg is not r.r5 and reg is not r.r11] + # the caller already did push_gcmap(store=True) self._push_core_regs_to_jitframe(mc, regs) if supports_floats: self._push_fp_regs_to_jitframe(mc) @@ -420,6 +420,7 @@ # Finish self._reload_frame_if_necessary(mc) + self.pop_gcmap(mc) # cancel the push_gcmap(store=True) in the caller self._pop_core_regs_from_jitframe(mc, saved_regs) if supports_floats: self._pop_fp_regs_from_jitframe(mc) @@ -449,12 +450,11 @@ mc.store_link() mc.push_std_frame() # - ofs2 = self.cpu.get_ofs_of_frame_field('jf_gcmap') - mc.STG(r.r1, l.addr(ofs2, r.SPP)) saved_regs = [reg for reg in r.MANAGED_REGS if reg is not r.RES and reg is not r.RSZ] self._push_core_regs_to_jitframe(mc, saved_regs) self._push_fp_regs_to_jitframe(mc) + # the caller already did push_gcmap(store=True) # if kind == 'fixed': addr = self.cpu.gc_ll_descr.get_malloc_slowpath_addr() @@ -502,6 +502,7 @@ # r.RSZ is loaded from [r1], to make the caller's store a no-op here mc.load(r.RSZ, r.r1, 0) # + self.pop_gcmap(mc) # push_gcmap(store=True) done by the caller mc.restore_link() mc.BCR(c.ANY, r.r14) self.mc = None @@ -588,7 +589,7 @@ # sum -> (14 bytes) mc.write('\x00'*14) mc.load_imm(r.RETURN, self._frame_realloc_slowpath) - self.load_gcmap(mc, r.r1, gcmap) + self.push_gcmap(mc, gcmap, store=True) mc.raw_call() self.frame_depth_to_patch.append((patch_pos, mc.currpos())) @@ -685,6 +686,8 @@ # name = "Loop # %s: %s" % (looptoken.number, loopname) # self.cpu.profile_agent.native_code_written(name, # rawstart, full_size) + #print(hex(rawstart+looppos)) + #import pdb; pdb.set_trace() return AsmInfo(ops_offset, rawstart + looppos, size_excluding_failure_stuff - looppos, rawstart) @@ -867,6 +870,10 @@ ofs = self.cpu.get_ofs_of_frame_field('jf_gcmap') mc.STG(r.SCRATCH, l.addr(ofs, r.SPP)) + def pop_gcmap(self, mc): + ofs = self.cpu.get_ofs_of_frame_field('jf_gcmap') + mc.LG(r.SCRATCH, l.addr(ofs, r.SPP)) + def break_long_loop(self): # If the loop is too long, the guards in it will jump forward # more than 32 KB. We use an approximate hack to know if we @@ -1339,7 +1346,7 @@ # new value of nursery_free_adr in RSZ and the adr of the new object # in RES. - self.load_gcmap(mc, r.r1, gcmap) + self.push_gcmap(mc, gcmap, store=True) mc.branch_absolute(self.malloc_slowpath) # here r1 holds nursery_free_addr @@ -1375,7 +1382,7 @@ # new value of nursery_free_adr in RSZ and the adr of the new object # in RES. - self.load_gcmap(mc, r.r1, gcmap) + self.push_gcmap(mc, gcmap, store=True) mc.branch_absolute(self.malloc_slowpath) offset = mc.currpos() - fast_jmp_pos @@ -1468,7 +1475,7 @@ pmc.overwrite() # # save the gcmap - self.load_gcmap(mc, r.r1, gcmap) + self.push_gcmap(mc, gcmap, store=True) # # load the function into r14 and jump if kind == rewrite.FLAG_ARRAY: From pypy.commits at gmail.com Tue Jul 5 07:07:47 2016 From: pypy.commits at gmail.com (plan_rich) Date: Tue, 05 Jul 2016 04:07:47 -0700 (PDT) Subject: [pypy-commit] pypy default: (s390x, ppc) resolve issue in test_gc_integration (test_malloc_slowpath), register allocator will not use register slots, but the frame slots on the gcmap Message-ID: <577b9503.949a1c0a.f1dbf.5ec6@mx.google.com> Author: Richard Plangger Branch: Changeset: r85554:a09c99e17a27 Date: 2016-07-05 13:06 +0200 http://bitbucket.org/pypy/pypy/changeset/a09c99e17a27/ Log: (s390x,ppc) resolve issue in test_gc_integration (test_malloc_slowpath), register allocator will not use register slots, but the frame slots on the gcmap diff --git a/rpython/jit/backend/llsupport/test/test_gc_integration.py b/rpython/jit/backend/llsupport/test/test_gc_integration.py --- a/rpython/jit/backend/llsupport/test/test_gc_integration.py +++ b/rpython/jit/backend/llsupport/test/test_gc_integration.py @@ -330,6 +330,13 @@ expected_size = 2 idx = 1 fixed_size -= 32 + if self.cpu.backend_name.startswith('zarch') or \ + self.cpu.backend_name.startswith('ppc'): + # the allocation always allocates the register + # into the return register. (e.g. r3 on ppc) + # the next malloc_nursery will move r3 to the + # frame manager, thus the two bits will be on the frame + fixed_size += 4 assert len(frame.jf_gcmap) == expected_size # check that we have two bits set, and that they are in two # registers (p0 and p1 are moved away when doing p2, but not From pypy.commits at gmail.com Tue Jul 5 07:08:52 2016 From: pypy.commits at gmail.com (arigo) Date: Tue, 05 Jul 2016 04:08:52 -0700 (PDT) Subject: [pypy-commit] pypy default: Found out how to reduce this division to a simple algorithm instead of Message-ID: <577b9544.09f6c20a.836ba.2814@mx.google.com> Author: Armin Rigo Branch: Changeset: r85556:676d2fb4524b Date: 2016-07-05 13:10 +0200 http://bitbucket.org/pypy/pypy/changeset/676d2fb4524b/ Log: Found out how to reduce this division to a simple algorithm instead of using rbigint. Avoids troubles if the full RPython program is also using parts of rbigint. diff --git a/rpython/jit/metainterp/optimizeopt/intdiv.py b/rpython/jit/metainterp/optimizeopt/intdiv.py --- a/rpython/jit/metainterp/optimizeopt/intdiv.py +++ b/rpython/jit/metainterp/optimizeopt/intdiv.py @@ -1,5 +1,5 @@ from rpython.rlib.rarithmetic import LONG_BIT, intmask, r_uint -from rpython.rlib.rbigint import rbigint, ONERBIGINT + from rpython.jit.metainterp.history import ConstInt from rpython.jit.metainterp.resoperation import ResOperation, rop @@ -17,10 +17,19 @@ while (r_uint(1) << (i+1)) < r_uint(m): i += 1 - # k = 2**(64+i) // m + 1, computed manually using rbigint - # because that's the easiest - k1 = ONERBIGINT.lshift(LONG_BIT + i).floordiv(rbigint.fromint(m)) - k = k1.touint() + r_uint(1) + # quotient = 2**(64+i) // m + high_word_dividend = r_uint(1) << i + quotient = r_uint(0) + for bit in range(LONG_BIT-1, -1, -1): + t = quotient + (r_uint(1) << bit) + # check: is 't * m' small enough to be < 2**(64+i), or not? + # note that we're really computing (2**(64+i)-1) // m, but the result + # is the same, because powers of two are not multiples of m. + if unsigned_mul_high(t, m) < high_word_dividend: + quotient = t # yes, small enough + + # k = 2**(64+i) // m + 1 + k = quotient + r_uint(1) assert k != r_uint(0) # Proof that k < 2**64 holds in all cases, even with the "+1": From pypy.commits at gmail.com Tue Jul 5 07:08:51 2016 From: pypy.commits at gmail.com (arigo) Date: Tue, 05 Jul 2016 04:08:51 -0700 (PDT) Subject: [pypy-commit] pypy reverse-debugger: Generally useful to run most pypy tests on top of a thread-less pypy Message-ID: <577b9543.cc9d1c0a.f6a85.ffffb993@mx.google.com> Author: Armin Rigo Branch: reverse-debugger Changeset: r85555:0efcaa2d00e4 Date: 2016-07-05 09:10 +0200 http://bitbucket.org/pypy/pypy/changeset/0efcaa2d00e4/ Log: Generally useful to run most pypy tests on top of a thread-less pypy diff --git a/rpython/rlib/rthread.py b/rpython/rlib/rthread.py --- a/rpython/rlib/rthread.py +++ b/rpython/rlib/rthread.py @@ -96,15 +96,17 @@ if we_are_translated(): return tlfield_thread_ident.getraw() else: - import thread - return thread.get_ident() + try: + import thread + return thread.get_ident() + except ImportError: + return 1 def get_or_make_ident(): if we_are_translated(): return tlfield_thread_ident.get_or_make_raw() else: - import thread - return thread.get_ident() + return get_ident() @specialize.arg(0) def start_new_thread(x, y): From pypy.commits at gmail.com Tue Jul 5 07:16:01 2016 From: pypy.commits at gmail.com (arigo) Date: Tue, 05 Jul 2016 04:16:01 -0700 (PDT) Subject: [pypy-commit] pypy default: Fix Message-ID: <577b96f1.c5461c0a.e7979.ffffced4@mx.google.com> Author: Armin Rigo Branch: Changeset: r85557:85427fd0119b Date: 2016-07-05 13:17 +0200 http://bitbucket.org/pypy/pypy/changeset/85427fd0119b/ Log: Fix diff --git a/rpython/jit/metainterp/optimizeopt/intdiv.py b/rpython/jit/metainterp/optimizeopt/intdiv.py --- a/rpython/jit/metainterp/optimizeopt/intdiv.py +++ b/rpython/jit/metainterp/optimizeopt/intdiv.py @@ -25,7 +25,7 @@ # check: is 't * m' small enough to be < 2**(64+i), or not? # note that we're really computing (2**(64+i)-1) // m, but the result # is the same, because powers of two are not multiples of m. - if unsigned_mul_high(t, m) < high_word_dividend: + if unsigned_mul_high(t, r_uint(m)) < high_word_dividend: quotient = t # yes, small enough # k = 2**(64+i) // m + 1 From pypy.commits at gmail.com Tue Jul 5 08:40:12 2016 From: pypy.commits at gmail.com (arigo) Date: Tue, 05 Jul 2016 05:40:12 -0700 (PDT) Subject: [pypy-commit] pypy reverse-debugger: Next fix (in clibffi.py), improve the debugging support Message-ID: <577baaac.06a81c0a.47527.6435@mx.google.com> Author: Armin Rigo Branch: reverse-debugger Changeset: r85558:edc44ccff552 Date: 2016-07-05 14:41 +0200 http://bitbucket.org/pypy/pypy/changeset/edc44ccff552/ Log: Next fix (in clibffi.py), improve the debugging support diff --git a/rpython/rlib/clibffi.py b/rpython/rlib/clibffi.py --- a/rpython/rlib/clibffi.py +++ b/rpython/rlib/clibffi.py @@ -420,6 +420,7 @@ """ userdata = rffi.cast(USERDATA_P, ll_userdata) userdata.callback(ll_args, ll_res, userdata) +_ll_callback._revdb_do_all_calls_ = True def ll_callback(ffi_cif, ll_res, ll_args, ll_userdata): rposix._errno_after(rffi.RFFI_ERR_ALL | rffi.RFFI_ALT_ERRNO) diff --git a/rpython/translator/revdb/gencsupp.py b/rpython/translator/revdb/gencsupp.py --- a/rpython/translator/revdb/gencsupp.py +++ b/rpython/translator/revdb/gencsupp.py @@ -60,7 +60,8 @@ name = funcgen.functionname funcgen.db.stack_bottom_funcnames.append(name) extra_enter_text = '\n'.join( - ['RPY_REVDB_CALLBACKLOC(RPY_CALLBACKLOC_%s);' % name] + + ['/* this function is a callback */', + 'RPY_REVDB_CALLBACKLOC(RPY_CALLBACKLOC_%s);' % name] + ['\t' + emit('/*arg*/', funcgen.lltypename(v), funcgen.expr(v)) for v in funcgen.graph.getargs()]) extra_return_text = '/* RPY_CALLBACK_LEAVE(); */' diff --git a/rpython/translator/revdb/pplog.py b/rpython/translator/revdb/pplog.py --- a/rpython/translator/revdb/pplog.py +++ b/rpython/translator/revdb/pplog.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2 +#!/usr/bin/env pypy # Post-process log files to make them diff-able. # @@ -19,7 +19,9 @@ r_hide_tail = re.compile(r"revdb[.]c:\d+: ([0-9a-f]+)") -r_remove = re.compile("\w+[.]c:\d+: obj 92233720368") +r_remove = re.compile(r"\w+[.]c:\d+: obj 92233720368|" + r"PID \d+ starting, log file disabled|" + r"\[") def post_process(fin, fout): diff --git a/rpython/translator/revdb/src-revdb/revdb.c b/rpython/translator/revdb/src-revdb/revdb.c --- a/rpython/translator/revdb/src-revdb/revdb.c +++ b/rpython/translator/revdb/src-revdb/revdb.c @@ -46,7 +46,7 @@ rpy_revdb_t rpy_revdb; static char rpy_rev_buffer[16384]; /* max. 32768 */ -static int rpy_rev_fileno = -1; +int rpy_rev_fileno = -1; static char flag_io_disabled = FID_REGULAR_MODE; diff --git a/rpython/translator/revdb/src-revdb/revdb_include.h b/rpython/translator/revdb/src-revdb/revdb_include.h --- a/rpython/translator/revdb/src-revdb/revdb_include.h +++ b/rpython/translator/revdb/src-revdb/revdb_include.h @@ -22,6 +22,7 @@ } rpy_revdb_t; RPY_EXTERN rpy_revdb_t rpy_revdb; +RPY_EXTERN int rpy_rev_fileno; /* ------------------------------------------------------------ */ @@ -31,19 +32,23 @@ #if 0 /* enable to print locations to stderr of all the EMITs */ # define _RPY_REVDB_PRINT(mode, _e) \ - fprintf(stderr, \ - "%s:%d: %0*llx\n", \ - __FILE__, __LINE__, 2 * sizeof(_e), \ - ((unsigned long long)_e) & ((2ULL << (8*sizeof(_e)-1)) - 1)) + if (rpy_rev_fileno >= 0) { \ + fprintf(stderr, \ + "%s:%d: %0*llx\n", \ + __FILE__, __LINE__, 2 * sizeof(_e), \ + ((unsigned long long)_e) & ((2ULL << (8*sizeof(_e)-1)) - 1)); \ + } #endif #if 0 /* enable to print all mallocs to stderr */ RPY_EXTERN void seeing_uid(uint64_t uid); -# define _RPY_REVDB_PRUID() \ - seeing_uid(uid); \ - fprintf(stderr, \ - "%s:%d: obj %llu\n", \ - __FILE__, __LINE__, (unsigned long long) uid) +# define _RPY_REVDB_PRUID() \ + if (rpy_rev_fileno >= 0) { \ + seeing_uid(uid); \ + fprintf(stderr, \ + "%s:%d: obj %llu\n", \ + __FILE__, __LINE__, (unsigned long long) uid); \ + } #endif #ifndef _RPY_REVDB_PRINT From pypy.commits at gmail.com Tue Jul 5 08:49:25 2016 From: pypy.commits at gmail.com (plan_rich) Date: Tue, 05 Jul 2016 05:49:25 -0700 (PDT) Subject: [pypy-commit] pypy ppc-vsx-support: merge default Message-ID: <577bacd5.4553c20a.c5823.4807@mx.google.com> Author: Richard Plangger Branch: ppc-vsx-support Changeset: r85559:ac89a39486d5 Date: 2016-07-05 13:16 +0200 http://bitbucket.org/pypy/pypy/changeset/ac89a39486d5/ Log: merge default diff --git a/dotviewer/graphparse.py b/dotviewer/graphparse.py --- a/dotviewer/graphparse.py +++ b/dotviewer/graphparse.py @@ -85,10 +85,11 @@ pass def splitline(line, re_word = re.compile(r'[^\s"]\S*|["]["]|["].*?[^\\]["]')): + import ast result = [] for word in re_word.findall(line): if word.startswith('"'): - word = eval(word) + word = ast.literal_eval(word) result.append(word) return result diff --git a/lib_pypy/datetime.py b/lib_pypy/datetime.py --- a/lib_pypy/datetime.py +++ b/lib_pypy/datetime.py @@ -839,7 +839,7 @@ month = self._month if day is None: day = self._day - return date(year, month, day) + return date.__new__(type(self), year, month, day) # Comparisons of date objects with other. @@ -1356,7 +1356,8 @@ microsecond = self.microsecond if tzinfo is True: tzinfo = self.tzinfo - return time(hour, minute, second, microsecond, tzinfo) + return time.__new__(type(self), + hour, minute, second, microsecond, tzinfo) def __nonzero__(self): if self.second or self.microsecond: @@ -1566,8 +1567,9 @@ microsecond = self.microsecond if tzinfo is True: tzinfo = self.tzinfo - return datetime(year, month, day, hour, minute, second, microsecond, - tzinfo) + return datetime.__new__(type(self), + year, month, day, hour, minute, second, + microsecond, tzinfo) def astimezone(self, tz): if not isinstance(tz, tzinfo): diff --git a/pypy/module/cpyext/test/test_cpyext.py b/pypy/module/cpyext/test/test_cpyext.py --- a/pypy/module/cpyext/test/test_cpyext.py +++ b/pypy/module/cpyext/test/test_cpyext.py @@ -78,7 +78,7 @@ else: libraries = [] if sys.platform.startswith('linux'): - compile_extra = ["-Werror", "-g", "-O0", "-fPIC"] + compile_extra = ["-Werror", "-g", "-O0", "-Wp,-U_FORTIFY_SOURCE", "-fPIC"] link_extra = ["-g"] else: compile_extra = link_extra = None diff --git a/pypy/module/test_lib_pypy/test_datetime.py b/pypy/module/test_lib_pypy/test_datetime.py --- a/pypy/module/test_lib_pypy/test_datetime.py +++ b/pypy/module/test_lib_pypy/test_datetime.py @@ -315,6 +315,51 @@ class sub(datetime.timedelta): pass assert type(+sub()) is datetime.timedelta + def test_subclass_date(self): + # replace() should return a subclass but not call __new__ or __init__. + class MyDate(datetime.date): + forbidden = False + def __new__(cls): + if cls.forbidden: FAIL + return datetime.date.__new__(cls, 2016, 2, 3) + def __init__(self, *args): + if self.forbidden: FAIL + d = MyDate() + d.forbidden = True + d2 = d.replace(day=5) + assert type(d2) is MyDate + assert d2 == datetime.date(2016, 2, 5) + + def test_subclass_time(self): + # replace() should return a subclass but not call __new__ or __init__. + class MyTime(datetime.time): + forbidden = False + def __new__(cls): + if cls.forbidden: FAIL + return datetime.time.__new__(cls, 1, 2, 3) + def __init__(self, *args): + if self.forbidden: FAIL + d = MyTime() + d.forbidden = True + d2 = d.replace(hour=5) + assert type(d2) is MyTime + assert d2 == datetime.time(5, 2, 3) + + def test_subclass_datetime(self): + # replace() should return a subclass but not call __new__ or __init__. + class MyDatetime(datetime.datetime): + forbidden = False + def __new__(cls): + if cls.forbidden: FAIL + return datetime.datetime.__new__(cls, 2016, 4, 5, 1, 2, 3) + def __init__(self, *args): + if self.forbidden: FAIL + d = MyDatetime() + d.forbidden = True + d2 = d.replace(hour=7) + assert type(d2) is MyDatetime + assert d2 == datetime.datetime(2016, 4, 5, 7, 2, 3) + class TestDatetimeHost(BaseTestDatetime): def setup_class(cls): diff --git a/rpython/jit/backend/llsupport/test/test_gc_integration.py b/rpython/jit/backend/llsupport/test/test_gc_integration.py --- a/rpython/jit/backend/llsupport/test/test_gc_integration.py +++ b/rpython/jit/backend/llsupport/test/test_gc_integration.py @@ -330,6 +330,13 @@ expected_size = 2 idx = 1 fixed_size -= 32 + if self.cpu.backend_name.startswith('zarch') or \ + self.cpu.backend_name.startswith('ppc'): + # the allocation always allocates the register + # into the return register. (e.g. r3 on ppc) + # the next malloc_nursery will move r3 to the + # frame manager, thus the two bits will be on the frame + fixed_size += 4 assert len(frame.jf_gcmap) == expected_size # check that we have two bits set, and that they are in two # registers (p0 and p1 are moved away when doing p2, but not diff --git a/rpython/jit/backend/zarch/arch.py b/rpython/jit/backend/zarch/arch.py --- a/rpython/jit/backend/zarch/arch.py +++ b/rpython/jit/backend/zarch/arch.py @@ -45,10 +45,7 @@ # +------------------------------+ <- assembler begin # | SAVE CONTEXT | # +------------------------------+ -# +--+| BRANCH (saves addr of pool | -# | | in r13) | -# | +------------------------------+ -# | | ... | +#start| ... | # | | LITERAL POOL | <---+ # | | ... | <-+ | # +-->+------------------------------+ | | diff --git a/rpython/jit/backend/zarch/assembler.py b/rpython/jit/backend/zarch/assembler.py --- a/rpython/jit/backend/zarch/assembler.py +++ b/rpython/jit/backend/zarch/assembler.py @@ -309,11 +309,9 @@ # signature of this _frame_realloc_slowpath function: # * on entry, r0 is the new size - # * on entry, r1 is the gcmap # * no managed register must be modified - ofs2 = self.cpu.get_ofs_of_frame_field('jf_gcmap') - mc.STG(r.SCRATCH, l.addr(ofs2, r.SPP)) + # caller already did push_gcmap(store=True) self._push_core_regs_to_jitframe(mc, r.MANAGED_REGS) self._push_fp_regs_to_jitframe(mc) @@ -347,6 +345,7 @@ mc.load(r.r5, r.r5, 0) mc.store(r.r2, r.r5, -WORD) + self.pop_gcmap(mc) # cancel the push_gcmap(store=True) in the caller self._pop_core_regs_from_jitframe(mc, r.MANAGED_REGS) self._pop_fp_regs_from_jitframe(mc) @@ -411,6 +410,7 @@ reg is not r.r4 and reg is not r.r5 and reg is not r.r11] + # the caller already did push_gcmap(store=True) self._push_core_regs_to_jitframe(mc, regs) if supports_floats: self._push_fp_regs_to_jitframe(mc) @@ -420,6 +420,7 @@ # Finish self._reload_frame_if_necessary(mc) + self.pop_gcmap(mc) # cancel the push_gcmap(store=True) in the caller self._pop_core_regs_from_jitframe(mc, saved_regs) if supports_floats: self._pop_fp_regs_from_jitframe(mc) @@ -449,12 +450,11 @@ mc.store_link() mc.push_std_frame() # - ofs2 = self.cpu.get_ofs_of_frame_field('jf_gcmap') - mc.STG(r.r1, l.addr(ofs2, r.SPP)) saved_regs = [reg for reg in r.MANAGED_REGS if reg is not r.RES and reg is not r.RSZ] self._push_core_regs_to_jitframe(mc, saved_regs) self._push_fp_regs_to_jitframe(mc) + # the caller already did push_gcmap(store=True) # if kind == 'fixed': addr = self.cpu.gc_ll_descr.get_malloc_slowpath_addr() @@ -502,6 +502,7 @@ # r.RSZ is loaded from [r1], to make the caller's store a no-op here mc.load(r.RSZ, r.r1, 0) # + self.pop_gcmap(mc) # push_gcmap(store=True) done by the caller mc.restore_link() mc.BCR(c.ANY, r.r14) self.mc = None @@ -588,7 +589,7 @@ # sum -> (14 bytes) mc.write('\x00'*14) mc.load_imm(r.RETURN, self._frame_realloc_slowpath) - self.load_gcmap(mc, r.r1, gcmap) + self.push_gcmap(mc, gcmap, store=True) mc.raw_call() self.frame_depth_to_patch.append((patch_pos, mc.currpos())) @@ -685,6 +686,8 @@ # name = "Loop # %s: %s" % (looptoken.number, loopname) # self.cpu.profile_agent.native_code_written(name, # rawstart, full_size) + #print(hex(rawstart+looppos)) + #import pdb; pdb.set_trace() return AsmInfo(ops_offset, rawstart + looppos, size_excluding_failure_stuff - looppos, rawstart) @@ -867,6 +870,10 @@ ofs = self.cpu.get_ofs_of_frame_field('jf_gcmap') mc.STG(r.SCRATCH, l.addr(ofs, r.SPP)) + def pop_gcmap(self, mc): + ofs = self.cpu.get_ofs_of_frame_field('jf_gcmap') + mc.LG(r.SCRATCH, l.addr(ofs, r.SPP)) + def break_long_loop(self): # If the loop is too long, the guards in it will jump forward # more than 32 KB. We use an approximate hack to know if we @@ -1339,7 +1346,7 @@ # new value of nursery_free_adr in RSZ and the adr of the new object # in RES. - self.load_gcmap(mc, r.r1, gcmap) + self.push_gcmap(mc, gcmap, store=True) mc.branch_absolute(self.malloc_slowpath) # here r1 holds nursery_free_addr @@ -1375,7 +1382,7 @@ # new value of nursery_free_adr in RSZ and the adr of the new object # in RES. - self.load_gcmap(mc, r.r1, gcmap) + self.push_gcmap(mc, gcmap, store=True) mc.branch_absolute(self.malloc_slowpath) offset = mc.currpos() - fast_jmp_pos @@ -1468,7 +1475,7 @@ pmc.overwrite() # # save the gcmap - self.load_gcmap(mc, r.r1, gcmap) + self.push_gcmap(mc, gcmap, store=True) # # load the function into r14 and jump if kind == rewrite.FLAG_ARRAY: diff --git a/rpython/rlib/ropenssl.py b/rpython/rlib/ropenssl.py --- a/rpython/rlib/ropenssl.py +++ b/rpython/rlib/ropenssl.py @@ -586,11 +586,6 @@ HASH_MALLOC_SIZE = EVP_MD_SIZE + EVP_MD_CTX_SIZE \ + rffi.sizeof(EVP_MD) * 2 + 208 -OBJ_NAME_CALLBACK = lltype.Ptr(lltype.FuncType( - [OBJ_NAME, rffi.VOIDP], lltype.Void)) -OBJ_NAME_do_all = external( - 'OBJ_NAME_do_all', [rffi.INT, OBJ_NAME_CALLBACK, rffi.VOIDP], lltype.Void) - def init_ssl(): libssl_SSL_load_error_strings() libssl_SSL_library_init() diff --git a/rpython/rlib/rposix.py b/rpython/rlib/rposix.py --- a/rpython/rlib/rposix.py +++ b/rpython/rlib/rposix.py @@ -220,7 +220,7 @@ pass if _WIN32: - includes = ['io.h', 'sys/utime.h', 'sys/types.h'] + includes = ['io.h', 'sys/utime.h', 'sys/types.h', 'process.h'] libraries = [] else: if sys.platform.startswith(('darwin', 'netbsd', 'openbsd')): @@ -704,10 +704,10 @@ c_execve = external('execve', [rffi.CCHARP, rffi.CCHARPP, rffi.CCHARPP], rffi.INT, save_err=rffi.RFFI_SAVE_ERRNO) -c_spawnv = external('spawnv', +c_spawnv = external(UNDERSCORE_ON_WIN32 + 'spawnv', [rffi.INT, rffi.CCHARP, rffi.CCHARPP], rffi.INT, save_err=rffi.RFFI_SAVE_ERRNO) -c_spawnve = external('spawnve', +c_spawnve = external(UNDERSCORE_ON_WIN32 + 'spawnve', [rffi.INT, rffi.CCHARP, rffi.CCHARPP, rffi.CCHARPP], rffi.INT, save_err=rffi.RFFI_SAVE_ERRNO) From pypy.commits at gmail.com Tue Jul 5 08:49:27 2016 From: pypy.commits at gmail.com (plan_rich) Date: Tue, 05 Jul 2016 05:49:27 -0700 (PDT) Subject: [pypy-commit] pypy ppc-vsx-support: add intmask to arith in test (vec_int_sub), implement stitch guard Message-ID: <577bacd7.d48e1c0a.180f1.393f@mx.google.com> Author: Richard Plangger Branch: ppc-vsx-support Changeset: r85560:b45d23fda408 Date: 2016-07-05 14:48 +0200 http://bitbucket.org/pypy/pypy/changeset/b45d23fda408/ Log: add intmask to arith in test (vec_int_sub), implement stitch guard diff --git a/rpython/jit/backend/ppc/locations.py b/rpython/jit/backend/ppc/locations.py --- a/rpython/jit/backend/ppc/locations.py +++ b/rpython/jit/backend/ppc/locations.py @@ -153,9 +153,6 @@ def __repr__(self): return 'FP(%s)+%d' % (self.type, self.value) - def location_code(self): - return 'b' - def get_position(self): return self.position diff --git a/rpython/jit/backend/ppc/ppc_assembler.py b/rpython/jit/backend/ppc/ppc_assembler.py --- a/rpython/jit/backend/ppc/ppc_assembler.py +++ b/rpython/jit/backend/ppc/ppc_assembler.py @@ -14,6 +14,7 @@ from rpython.jit.backend.ppc.helper.regalloc import _check_imm_arg import rpython.jit.backend.ppc.register as r import rpython.jit.backend.ppc.condition as c +from rpython.jit.metainterp.compile import ResumeGuardDescr from rpython.jit.backend.ppc.register import JITFRAME_FIXED_SIZE from rpython.jit.metainterp.history import AbstractFailDescr from rpython.jit.backend.llsupport import jitframe, rewrite @@ -811,7 +812,7 @@ #print(hex(rawstart)) #import pdb; pdb.set_trace() return AsmInfo(ops_offset, rawstart + looppos, - size_excluding_failure_stuff - looppos) + size_excluding_failure_stuff - looppos, rawstart + looppos) def _assemble(self, regalloc, inputargs, operations): self._regalloc = regalloc @@ -876,7 +877,8 @@ self.fixup_target_tokens(rawstart) self.update_frame_depth(frame_depth) self.teardown() - return AsmInfo(ops_offset, startpos + rawstart, codeendpos - startpos) + return AsmInfo(ops_offset, startpos + rawstart, codeendpos - startpos, + startpos + rawstart) def reserve_gcref_table(self, allgcrefs): # allocate the gc table right now. We write absolute loads in @@ -1373,14 +1375,13 @@ assert isinstance(bridge_faildescr, ResumeGuardDescr) assert isinstance(faildescr, ResumeGuardDescr) assert asminfo.rawstart != 0 - self.mc = codebuf.MachineCodeBlockWrapper() + self.mc = PPCBuilder() allblocks = self.get_asmmemmgr_blocks(looptoken) self.datablockwrapper = MachineDataBlockWrapper(self.cpu.asmmemmgr, allblocks) frame_info = self.datablockwrapper.malloc_aligned( jitframe.JITFRAMEINFO_SIZE, alignment=WORD) - self.mc.force_frame_size(DEFAULT_FRAME_BYTES) # if accumulation is saved at the guard, we need to update it here! guard_locs = self.rebuild_faillocs_from_descr(faildescr, version.inputargs) bridge_locs = self.rebuild_faillocs_from_descr(bridge_faildescr, version.inputargs) @@ -1392,7 +1393,7 @@ if bridge_accum_info.failargs_pos == guard_accum_info.failargs_pos: # the mapping might be wrong! if bridge_accum_info.location is not guard_accum_info.location: - self.mov(guard_accum_info.location, bridge_accum_info.location) + self.regalloc_mov(guard_accum_info.location, bridge_accum_info.location) bridge_accum_info = bridge_accum_info.next() guard_accum_info = guard_accum_info.next() @@ -1401,19 +1402,14 @@ assert len(guard_locs) == len(bridge_locs) for i,gloc in enumerate(guard_locs): bloc = bridge_locs[i] - bstack = bloc.location_code() == 'b' - gstack = gloc.location_code() == 'b' - if bstack and gstack: + if bloc.is_stack() and gloc.is_stack(): pass elif gloc is not bloc: - self.mov(gloc, bloc) + self.regalloc_mov(gloc, bloc) offset = self.mc.get_relative_pos() - self.mc.JMP_l(0) - self.mc.writeimm32(0) - self.mc.force_frame_size(DEFAULT_FRAME_BYTES) + self.mc.b_abs(asminfo.rawstart) + rawstart = self.materialize_loop(looptoken) - # update the jump (above) to the real trace - self._patch_jump_to(rawstart + offset, asminfo.rawstart) # update the guard to jump right to this custom piece of assembler self.patch_jump_for_descr(faildescr, rawstart) diff --git a/rpython/jit/backend/ppc/vector_ext.py b/rpython/jit/backend/ppc/vector_ext.py --- a/rpython/jit/backend/ppc/vector_ext.py +++ b/rpython/jit/backend/ppc/vector_ext.py @@ -88,8 +88,7 @@ def emit_vec_load_f(self, op, arglocs, regalloc): resloc, baseloc, indexloc, size_loc, ofs, integer_loc, aligned_loc = arglocs - #src_addr = addr_add(baseloc, ofs_loc, ofs.value, 0) - assert ofs.value == 0 + indexloc = self._apply_offset(indexloc, ofs) itemsize = size_loc.value if itemsize == 4: self.mc.lxvw4x(resloc.value, indexloc.value, baseloc.value) @@ -99,8 +98,7 @@ def emit_vec_load_i(self, op, arglocs, regalloc): resloc, baseloc, indexloc, size_loc, ofs, \ Vhiloc, Vloloc, Vploc, tloc = arglocs - #src_addr = addr_add(base_loc, ofs_loc, ofs.value, 0) - assert ofs.value == 0 + indexloc = self._apply_offset(indexloc, ofs) Vlo = Vloloc.value Vhi = Vhiloc.value self.mc.lvx(Vhi, indexloc.value, baseloc.value) @@ -117,21 +115,10 @@ else: self.mc.vperm(resloc.value, Vlo, Vhi, Vp) - def _emit_vec_setitem(self, op, arglocs, regalloc): - # prepares item scale (raw_store does not) - base_loc, ofs_loc, value_loc, size_loc, baseofs, integer_loc, aligned_loc = arglocs - scale = get_scale(size_loc.value) - dest_loc = addr_add(base_loc, ofs_loc, baseofs.value, scale) - self._vec_store(dest_loc, value_loc, integer_loc.value, - size_loc.value, aligned_loc.value) - - genop_discard_vec_setarrayitem_raw = _emit_vec_setitem - genop_discard_vec_setarrayitem_gc = _emit_vec_setitem - def emit_vec_store(self, op, arglocs, regalloc): baseloc, indexloc, valueloc, sizeloc, baseofs, \ integer_loc, aligned_loc = arglocs - #dest_loc = addr_add(base_loc, ofs_loc, baseofs.value, 0) + indexloc = self._apply_offset(indexloc, baseofs) assert baseofs.value == 0 if integer_loc.value: Vloloc = regalloc.ivrm.get_scratch_reg() diff --git a/rpython/jit/metainterp/test/test_vector.py b/rpython/jit/metainterp/test/test_vector.py --- a/rpython/jit/metainterp/test/test_vector.py +++ b/rpython/jit/metainterp/test/test_vector.py @@ -207,9 +207,7 @@ bits = size*8 integers = st.integers(min_value=-2**(bits-1), max_value=2**(bits-1)-1) la = data.draw(st.lists(integers, min_size=10, max_size=150)) - #la = [0] * 10 #1,2,3,4,5,6,7,8,9,10,11,12,13] l = len(la) - #lb = [0] * 10 # [1,2,3,4,5,6,7,8,9,10,11,12,13] lb = data.draw(st.lists(integers, min_size=l, max_size=l)) rawstorage = RawStorage() @@ -238,11 +236,11 @@ test_vec_short_add = \ vec_int_arith(lambda a,b: r_int(a)+r_int(b), rffi.SHORT) - test_vec_signed_sub = \ - vec_int_arith(lambda a,b: r_int(a)-r_int(b), rffi.SIGNED) - test_vec_int_sub = \ + test_vec_sub_signed = \ + vec_int_arith(lambda a,b: intmask(a-b), rffi.SIGNED) + test_vec_sub_int = \ vec_int_arith(lambda a,b: r_int(a)-r_int(b), rffi.INT) - test_vec_short_sub = \ + test_vec_sub_short = \ vec_int_arith(lambda a,b: r_int(a)-r_int(b), rffi.SHORT) test_vec_signed_and = \ From pypy.commits at gmail.com Tue Jul 5 09:43:00 2016 From: pypy.commits at gmail.com (plan_rich) Date: Tue, 05 Jul 2016 06:43:00 -0700 (PDT) Subject: [pypy-commit] pypy ppc-vsx-support: implement more unpack_i combinations, conversion function float -> int Message-ID: <577bb964.4aa6c20a.dbb48.3176@mx.google.com> Author: Richard Plangger Branch: ppc-vsx-support Changeset: r85561:e9942271f923 Date: 2016-07-05 15:42 +0200 http://bitbucket.org/pypy/pypy/changeset/e9942271f923/ Log: implement more unpack_i combinations, conversion function float -> int diff --git a/rpython/jit/backend/ppc/codebuilder.py b/rpython/jit/backend/ppc/codebuilder.py --- a/rpython/jit/backend/ppc/codebuilder.py +++ b/rpython/jit/backend/ppc/codebuilder.py @@ -635,6 +635,7 @@ # conversion from/to xvcvsxddp = XX2(60, XO6=504) + xvcvdpsxds = XX2(60, XO6=472) # compare greater than unsigned int vcmpgtubx = VC(4, XO12=518, OE=1) diff --git a/rpython/jit/backend/ppc/vector_ext.py b/rpython/jit/backend/ppc/vector_ext.py --- a/rpython/jit/backend/ppc/vector_ext.py +++ b/rpython/jit/backend/ppc/vector_ext.py @@ -545,14 +545,27 @@ res = resloc.value src = srcloc.value size = op.bytesize - if size == 8: - if srcloc.is_vector_reg(): # reg <- vector - assert not resloc.is_vector_reg() - self.mc.load_imm(r.SCRATCH2, PARAM_SAVE_AREA_OFFSET) - self.mc.stvx(src, r.SCRATCH2.value, r.SP.value) - self.mc.load(res, r.SP.value, PARAM_SAVE_AREA_OFFSET+8*idx) - else: - notimplemented("[ppc/vec_unpack_i] 64 bit integer") + count = countloc.value + if count == 1: + assert srcloc.is_vector_reg() + assert not resloc.is_vector_reg() + off = PARAM_SAVE_AREA_OFFSET + self.mc.load_imm(r.SCRATCH2, off) + off = off + size*idx + self.mc.stvx(src, r.SCRATCH2.value, r.SP.value) + if size == 8: + self.mc.load(res, r.SP.value, off+size*idx) + elif size == 4: + self.mc.lwa(res, r.SP.value, off) + elif size == 2: + self.mc.lha(res, r.SP.value, off) + elif size == 1: + self.mc.lbz(res, r.SP.value, off) + self.mc.extsb(res, res) + return + + notimplemented("[ppc/vec_unpack_i] %d bit integer, count %d" % \ + (size*8, count)) def emit_vec_pack_f(self, op, arglocs, regalloc): resloc, vloc, srcloc, residxloc, srcidxloc, countloc = arglocs @@ -600,9 +613,18 @@ resloc, srcloc, idxloc, countloc = arglocs self.emit_vec_pack_f(op, [resloc, srcloc, srcloc, imm(0), idxloc, countloc], regalloc) + def emit_vec_cast_float_to_int(self, op, arglocs, regalloc): + res, l0 = arglocs + offloc = regalloc.rm.get_scratch_reg() + v0 = regalloc.vrm.get_scratch_reg() + off = offloc.value + # SP is always 16 byte aligned, and PARAM_SAVE_AREA_OFFSET % 16 == 0 + self.mc.load_imm(offloc, PARAM_SAVE_AREA_OFFSET) + self.mc.xvcvdpsxds(v0.value, l0.value) + self.mc.stxvd2x(v0.value, off, r.SP.value) + self.mc.lvx(res.value, off, r.SP.value) + # needed as soon as PPC's support_singlefloat is implemented! - #def genop_vec_cast_float_to_int(self, op, arglocs, regalloc): - # self.mc.CVTPD2DQ(resloc, arglocs[0]) #def genop_vec_cast_singlefloat_to_float(self, op, arglocs, regalloc): # self.mc.CVTPS2PD(resloc, arglocs[0]) From pypy.commits at gmail.com Tue Jul 5 10:29:11 2016 From: pypy.commits at gmail.com (plan_rich) Date: Tue, 05 Jul 2016 07:29:11 -0700 (PDT) Subject: [pypy-commit] pypy ppc-vsx-support: remove the integer register allocator. float and int vector register are colocated (i.e. float 32 -> int 0, float 33 -> int 1, ...), this makes things much easier Message-ID: <577bc437.4a9bc20a.d005.313a@mx.google.com> Author: Richard Plangger Branch: ppc-vsx-support Changeset: r85562:c97b2b5608a4 Date: 2016-07-05 16:27 +0200 http://bitbucket.org/pypy/pypy/changeset/c97b2b5608a4/ Log: remove the integer register allocator. float and int vector register are colocated (i.e. float 32 -> int 0, float 33 -> int 1, ...), this makes things much easier diff --git a/rpython/jit/backend/ppc/regalloc.py b/rpython/jit/backend/ppc/regalloc.py --- a/rpython/jit/backend/ppc/regalloc.py +++ b/rpython/jit/backend/ppc/regalloc.py @@ -142,9 +142,9 @@ self.temp_boxes.append(box) return reg -class IntegerVectorRegisterManager(RegisterManager): - all_regs = r.MANAGED_INTEGER_VECTOR_REGS - box_types = [INT] +class VectorRegisterManager(RegisterManager): + all_regs = r.MANAGED_FLOAT_VECTOR_REGS + box_types = [FLOAT, INT] save_around_call_regs = [] # ??? lookup the ABI assert set(save_around_call_regs).issubset(all_regs) @@ -154,26 +154,11 @@ def ensure_reg(self, box): raise NotImplementedError - def get_scratch_reg(self): - box = TempInt() - reg = self.force_allocate_reg(box, forbidden_vars=self.temp_boxes) - self.temp_boxes.append(box) - return reg - -class FloatVectorRegisterManager(IntegerVectorRegisterManager): - all_regs = r.MANAGED_FLOAT_VECTOR_REGS - box_types = [FLOAT] - save_around_call_regs = [] # ??? lookup the ABI - assert set(save_around_call_regs).issubset(all_regs) - - def __init__(self, longevity, frame_manager=None, assembler=None): - RegisterManager.__init__(self, longevity, frame_manager, assembler) - - def ensure_reg(self, box): - raise NotImplementedError - - def get_scratch_reg(self): - box = TempFloat() + def get_scratch_reg(self, type=INT): + if type == FLOAT: + box = TempFloat() + else: + box = TempInt() reg = self.force_allocate_reg(box, forbidden_vars=self.temp_boxes) self.temp_boxes.append(box) return reg @@ -221,10 +206,8 @@ assembler = self.assembler) self.fprm = FPRegisterManager(self.longevity, frame_manager = self.fm, assembler = self.assembler) - self.vrm = FloatVectorRegisterManager(self.longevity, frame_manager = self.fm, - assembler = self.assembler) - self.ivrm = IntegerVectorRegisterManager(self.longevity, frame_manager = self.fm, - assembler = self.assembler) + self.vrm = VectorRegisterManager(self.longevity, frame_manager = self.fm, + assembler = self.assembler) return operations def prepare_loop(self, inputargs, operations, looptoken, allgcrefs): @@ -286,16 +269,13 @@ def possibly_free_var(self, var): if var is not None: - if var.type == FLOAT: - if var.is_vector(): + if var.is_vector(): + if var.type != VOID: self.vrm.possibly_free_var(var) - else: - self.fprm.possibly_free_var(var) + elif var.type == FLOAT: + self.fprm.possibly_free_var(var) elif var.type == INT: - if var.is_vector(): - self.ivrm.possibly_free_var(var) - else: - self.rm.possibly_free_var(var) + self.rm.possibly_free_var(var) def possibly_free_vars(self, vars): for var in vars: @@ -339,7 +319,6 @@ self.rm.position = i self.fprm.position = i self.vrm.position = i - self.ivrm.position = i opnum = op.opnum if rop.has_no_side_effect(opnum) and op not in self.longevity: i += 1 @@ -348,16 +327,13 @@ # for j in range(op.numargs()): box = op.getarg(j) - if box.type != FLOAT: - if box.is_vector(): - self.ivrm.temp_boxes.append(box) - else: - self.rm.temp_boxes.append(box) + if box.is_vector(): + if box.type != VOID: + self.vrm.temp_boxes.append(box) + elif box.type != FLOAT: + self.rm.temp_boxes.append(box) else: - if box.is_vector(): - self.vrm.temp_boxes.append(box) - else: - self.fprm.temp_boxes.append(box) + self.fprm.temp_boxes.append(box) # if not we_are_translated() and opnum == rop.FORCE_SPILL: self._consider_force_spill(op) @@ -369,7 +345,6 @@ self.rm._check_invariants() self.fprm._check_invariants() self.vrm._check_invariants() - self.ivrm._check_invariants() if self.assembler.mc.get_relative_pos() > self.limit_loop_break: self.assembler.break_long_loop(self) self.limit_loop_break = (self.assembler.mc.get_relative_pos() + @@ -413,10 +388,7 @@ def loc(self, var): if var.is_vector(): - if var.type == FLOAT: - return self.vrm.loc(var) - else: - return self.ivrm.loc(var) + return self.vrm.loc(var) else: if var.type == FLOAT: return self.fprm.loc(var) @@ -487,7 +459,6 @@ self.rm.free_temp_vars() self.fprm.free_temp_vars() self.vrm.free_temp_vars() - self.ivrm.free_temp_vars() # ****************************************************** # * P R E P A R E O P E R A T I O N S * diff --git a/rpython/jit/backend/ppc/vector_ext.py b/rpython/jit/backend/ppc/vector_ext.py --- a/rpython/jit/backend/ppc/vector_ext.py +++ b/rpython/jit/backend/ppc/vector_ext.py @@ -51,8 +51,8 @@ else: resval = result_loc.value # either doubleword integer 1 (2x) or word integer 1 (4x) - ones = regalloc.ivrm.get_scratch_reg().value - zeros = regalloc.ivrm.get_scratch_reg().value + ones = regalloc.vrm.get_scratch_reg(type=INT).value + zeros = regalloc.vrm.get_scratch_reg(type=INT).value asm.mc.vxor(zeros, zeros, zeros) if size == 4: asm.mc.vspltisw(ones, 1) @@ -121,15 +121,15 @@ indexloc = self._apply_offset(indexloc, baseofs) assert baseofs.value == 0 if integer_loc.value: - Vloloc = regalloc.ivrm.get_scratch_reg() - Vhiloc = regalloc.ivrm.get_scratch_reg() - Vploc = regalloc.ivrm.get_scratch_reg() + Vloloc = regalloc.vrm.get_scratch_reg(type=INT) + Vhiloc = regalloc.vrm.get_scratch_reg(type=INT) + Vploc = regalloc.vrm.get_scratch_reg(type=INT) tloc = regalloc.rm.get_scratch_reg() - V1sloc = regalloc.ivrm.get_scratch_reg() + V1sloc = regalloc.vrm.get_scratch_reg(type=INT) V1s = V1sloc.value - V0sloc = regalloc.ivrm.get_scratch_reg() + V0sloc = regalloc.vrm.get_scratch_reg(type=INT) V0s = V0sloc.value - Vmaskloc = regalloc.ivrm.get_scratch_reg() + Vmaskloc = regalloc.vrm.get_scratch_reg(type=INT) Vmask = Vmaskloc.value Vlo = Vhiloc.value Vhi = Vloloc.value @@ -372,7 +372,7 @@ def emit_vec_int_is_true(self, op, arglocs, regalloc): resloc, argloc, sizeloc = arglocs size = sizeloc.value - tmp = regalloc.ivrm.get_scratch_reg().value + tmp = regalloc.vrm.get_scratch_reg(type=INT).value self.mc.vxor(tmp, tmp, tmp) # argloc[i] > 0: # For an unsigned integer that is equivalent to argloc[i] != 0 @@ -616,13 +616,11 @@ def emit_vec_cast_float_to_int(self, op, arglocs, regalloc): res, l0 = arglocs offloc = regalloc.rm.get_scratch_reg() - v0 = regalloc.vrm.get_scratch_reg() + v0 = regalloc.vrm.get_scratch_reg(type=INT) off = offloc.value # SP is always 16 byte aligned, and PARAM_SAVE_AREA_OFFSET % 16 == 0 self.mc.load_imm(offloc, PARAM_SAVE_AREA_OFFSET) - self.mc.xvcvdpsxds(v0.value, l0.value) - self.mc.stxvd2x(v0.value, off, r.SP.value) - self.mc.lvx(res.value, off, r.SP.value) + self.mc.xvcvdpsxds(res.value, l0.value) # needed as soon as PPC's support_singlefloat is implemented! #def genop_vec_cast_singlefloat_to_float(self, op, arglocs, regalloc): @@ -637,10 +635,7 @@ def force_allocate_vector_reg(self, op): forbidden_vars = self.vrm.temp_boxes - if op.type == FLOAT: - return self.vrm.force_allocate_reg(op, forbidden_vars) - else: - return self.ivrm.force_allocate_reg(op, forbidden_vars) + return self.vrm.force_allocate_reg(op, forbidden_vars) def force_allocate_vector_reg_or_cc(self, op): assert op.type == INT @@ -654,12 +649,8 @@ return self.force_allocate_vector_reg(op) def ensure_vector_reg(self, box): - if box.type == FLOAT: - return self.vrm.make_sure_var_in_reg(box, - forbidden_vars=self.vrm.temp_boxes) - else: - return self.ivrm.make_sure_var_in_reg(box, - forbidden_vars=self.ivrm.temp_boxes) + return self.vrm.make_sure_var_in_reg(box, + forbidden_vars=self.vrm.temp_boxes) def _prepare_load(self, op): descr = op.getdescr() @@ -691,9 +682,9 @@ ofs_loc = self.ensure_reg(a1) result_loc = self.force_allocate_vector_reg(op) tloc = self.rm.get_scratch_reg() - Vhiloc = self.ivrm.get_scratch_reg() - Vloloc = self.ivrm.get_scratch_reg() - Vploc = self.ivrm.get_scratch_reg() + Vhiloc = self.vrm.get_scratch_reg(type=INT) + Vloloc = self.vrm.get_scratch_reg(type=INT) + Vploc = self.vrm.get_scratch_reg(type=INT) return [result_loc, base_loc, ofs_loc, imm(itemsize), imm(ofs), Vhiloc, Vloloc, Vploc, tloc] diff --git a/rpython/jit/metainterp/test/test_vector.py b/rpython/jit/metainterp/test/test_vector.py --- a/rpython/jit/metainterp/test/test_vector.py +++ b/rpython/jit/metainterp/test/test_vector.py @@ -613,7 +613,7 @@ yield (size, Typ(*t1), Typ(*t2), Typ(*t3), op[0], op[1]) types = [('rffi.DOUBLE', 'float', 'float'), ('rffi.SIGNED', 'int', 'int'), - ('rffi.FLOAT', 'rffi.r_singlefloat', 'float'), + #('rffi.FLOAT', 'rffi.r_singlefloat', 'float'), ] operators = [('add', '+'), ] @@ -629,11 +629,11 @@ vector_b = lltype.malloc(T2, size, flavor='raw') vector_c = lltype.malloc(T3, size, flavor='raw') for i in range(size): - vector_a[i] = {type_a_storecast}(i+1) + vector_a[i] = {type_a_storecast}(1) for i in range(size): - vector_b[i] = {type_b_storecast}(i+1) + vector_b[i] = {type_b_storecast}(1) for i in range(size): - vector_c[i] = {type_c_storecast}(i+1) + vector_c[i] = {type_c_storecast}(1) i = 0 while i < size: myjitdriver.jit_merge_point() From pypy.commits at gmail.com Tue Jul 5 10:49:56 2016 From: pypy.commits at gmail.com (arigo) Date: Tue, 05 Jul 2016 07:49:56 -0700 (PDT) Subject: [pypy-commit] pypy default: Potential fix for the rare bug that shows up occasionally with cpython Message-ID: <577bc914.571b1c0a.15e4e.ffff8866@mx.google.com> Author: Armin Rigo Branch: Changeset: r85563:bd220c268bc9 Date: 2016-07-05 16:51 +0200 http://bitbucket.org/pypy/pypy/changeset/bd220c268bc9/ Log: Potential fix for the rare bug that shows up occasionally with cpython (and more often with pypy). According to reverse-debugger, it seems to be caused by a PyObject created in some earlier test that is freed in the current test. diff --git a/pypy/module/cpyext/test/test_cpyext.py b/pypy/module/cpyext/test/test_cpyext.py --- a/pypy/module/cpyext/test/test_cpyext.py +++ b/pypy/module/cpyext/test/test_cpyext.py @@ -129,6 +129,7 @@ return str(soname) def freeze_refcnts(self): + rawrefcount._dont_free_any_more() return #ZZZ state = self.space.fromcache(RefcountState) self.frozen_refcounts = {} From pypy.commits at gmail.com Tue Jul 5 11:03:37 2016 From: pypy.commits at gmail.com (arigo) Date: Tue, 05 Jul 2016 08:03:37 -0700 (PDT) Subject: [pypy-commit] pypy reverse-debugger: minor fixes Message-ID: <577bcc49.c61f1c0a.f5c8.ffff8e5a@mx.google.com> Author: Armin Rigo Branch: reverse-debugger Changeset: r85564:f14a0ab01aa2 Date: 2016-07-05 17:04 +0200 http://bitbucket.org/pypy/pypy/changeset/f14a0ab01aa2/ Log: minor fixes diff --git a/rpython/translator/revdb/interact.py b/rpython/translator/revdb/interact.py --- a/rpython/translator/revdb/interact.py +++ b/rpython/translator/revdb/interact.py @@ -162,7 +162,7 @@ kind, name = self._bp_kind(num) print '\t%s %d: %s' % (kind, num, name) else: - print 'no breakpoints.' + print 'no breakpoints/watchpoints.' cmd_info_watchpoints = cmd_info_breakpoints def move_forward(self, steps): diff --git a/rpython/translator/revdb/process.py b/rpython/translator/revdb/process.py --- a/rpython/translator/revdb/process.py +++ b/rpython/translator/revdb/process.py @@ -203,6 +203,7 @@ if nid == new_nid: pgroup.all_printed_objects_lst.append(uid) sys.stdout.write('$%d = ' % nid) + sys.stdout.flush() else: print >> sys.stderr, "unexpected %r" % (msg,) From pypy.commits at gmail.com Tue Jul 5 11:57:30 2016 From: pypy.commits at gmail.com (raffael_t) Date: Tue, 05 Jul 2016 08:57:30 -0700 (PDT) Subject: [pypy-commit] pypy py3.5-async: Implement handle_async_stmt, Parameter bugfix Message-ID: <577bd8ea.c4cb1c0a.ab92c.20a0@mx.google.com> Author: Raffael Tfirst Branch: py3.5-async Changeset: r85565:aedfbe75e2b9 Date: 2016-07-05 16:54 +0200 http://bitbucket.org/pypy/pypy/changeset/aedfbe75e2b9/ Log: Implement handle_async_stmt, Parameter bugfix diff --git a/pypy/interpreter/astcompiler/astbuilder.py b/pypy/interpreter/astcompiler/astbuilder.py --- a/pypy/interpreter/astcompiler/astbuilder.py +++ b/pypy/interpreter/astcompiler/astbuilder.py @@ -494,10 +494,21 @@ funcdef_node.get_lineno(), funcdef_node.get_column()) def handle_async_funcdef(self, node, decorators=None): - return handle_funcdef_impl(node.get_child(1), decorators, 1) + return handle_funcdef_impl(node.get_child(1), 1, decorators) def handle_funcdef(self, node, decorators=None): - return handle_funcdef_impl(node, decorators, 0) + return handle_funcdef_impl(node, 0, decorators) + + def handle_async_stmt(self, node): + ch = node.get_child(1) + if ch.type == syms.funcdef: + return handle_funcdef_impl(ch, 1) + elif ch.type == syms.with_stmt: + return handle_with_stmt(ch, 1) + elif ch.type == syms.for_stmt: + return handle_for_stmt(ch, 1) + else: + raise AssertionError("invalid async statement") def handle_decorated(self, decorated_node): decorators = self.handle_decorators(decorated_node.get_child(0)) From pypy.commits at gmail.com Tue Jul 5 11:57:32 2016 From: pypy.commits at gmail.com (raffael_t) Date: Tue, 05 Jul 2016 08:57:32 -0700 (PDT) Subject: [pypy-commit] pypy py3.5-async: Add async_funcdef to decorated Message-ID: <577bd8ec.09f6c20a.836ba.ffffa769@mx.google.com> Author: Raffael Tfirst Branch: py3.5-async Changeset: r85566:52897255462a Date: 2016-07-05 17:13 +0200 http://bitbucket.org/pypy/pypy/changeset/52897255462a/ Log: Add async_funcdef to decorated diff --git a/pypy/interpreter/astcompiler/astbuilder.py b/pypy/interpreter/astcompiler/astbuilder.py --- a/pypy/interpreter/astcompiler/astbuilder.py +++ b/pypy/interpreter/astcompiler/astbuilder.py @@ -517,6 +517,8 @@ node = self.handle_funcdef(definition, decorators) elif definition.type == syms.classdef: node = self.handle_classdef(definition, decorators) + elif definition.type == syms.async_funcdef: + node = self.handle_async_funcdef(definition, decorators) else: raise AssertionError("unkown decorated") node.lineno = decorated_node.get_lineno() From pypy.commits at gmail.com Tue Jul 5 11:57:34 2016 From: pypy.commits at gmail.com (raffael_t) Date: Tue, 05 Jul 2016 08:57:34 -0700 (PDT) Subject: [pypy-commit] pypy py3.5-async: Change handle_power to handle_atom_expr (with await) Message-ID: <577bd8ee.48371c0a.f2b81.ffff940a@mx.google.com> Author: Raffael Tfirst Branch: py3.5-async Changeset: r85567:744aee598f7a Date: 2016-07-05 17:56 +0200 http://bitbucket.org/pypy/pypy/changeset/744aee598f7a/ Log: Change handle_power to handle_atom_expr (with await) diff --git a/pypy/interpreter/astcompiler/astbuilder.py b/pypy/interpreter/astcompiler/astbuilder.py --- a/pypy/interpreter/astcompiler/astbuilder.py +++ b/pypy/interpreter/astcompiler/astbuilder.py @@ -946,11 +946,18 @@ raise AssertionError("invalid factor node") return ast.UnaryOp(op, expr, factor_node.get_lineno(), factor_node.get_column()) - def handle_power(self, power_node): - atom_expr = self.handle_atom(power_node.get_child(0)) - if power_node.num_children() == 1: + def handle_atom_expr(self, power_node): + start = 0 + num_ch = power_node.num_children() + if power_node.get_child(0).type == tokens.AWAIT: + start = 1 + atom_expr = self.handle_atom(power_node.get_child(start)) + if num_ch == 1: return atom_expr - for i in range(1, power_node.num_children()): + if start and num_ch == 2: + return ast.Await(atom_expr, power_node.get_lineno(), + power_node.get_column()) + for i in range(start+1, num_ch): trailer = power_node.get_child(i) if trailer.type != syms.trailer: break @@ -962,7 +969,11 @@ right = self.handle_expr(power_node.get_child(-1)) atom_expr = ast.BinOp(atom_expr, ast.Pow, right, power_node.get_lineno(), power_node.get_column()) - return atom_expr + if start: + return ast.Await(atom_expr, power_node.get_lineno(), + power_node.get_column()) + else: + return atom_expr def handle_slice(self, slice_node): first_child = slice_node.get_child(0) From pypy.commits at gmail.com Tue Jul 5 12:09:24 2016 From: pypy.commits at gmail.com (raffael_t) Date: Tue, 05 Jul 2016 09:09:24 -0700 (PDT) Subject: [pypy-commit] pypy py3.5-async: Add handle_power with atom item Message-ID: <577bdbb4.901e1c0a.eea3e.3688@mx.google.com> Author: Raffael Tfirst Branch: py3.5-async Changeset: r85568:6d1701e65706 Date: 2016-07-05 18:08 +0200 http://bitbucket.org/pypy/pypy/changeset/6d1701e65706/ Log: Add handle_power with atom item diff --git a/pypy/interpreter/astcompiler/astbuilder.py b/pypy/interpreter/astcompiler/astbuilder.py --- a/pypy/interpreter/astcompiler/astbuilder.py +++ b/pypy/interpreter/astcompiler/astbuilder.py @@ -946,34 +946,40 @@ raise AssertionError("invalid factor node") return ast.UnaryOp(op, expr, factor_node.get_lineno(), factor_node.get_column()) - def handle_atom_expr(self, power_node): + def handle_atom_expr(self, atom_node): start = 0 - num_ch = power_node.num_children() - if power_node.get_child(0).type == tokens.AWAIT: + num_ch = atom_node.num_children() + if atom_node.get_child(0).type == tokens.AWAIT: start = 1 - atom_expr = self.handle_atom(power_node.get_child(start)) + atom_expr = self.handle_atom(atom_node.get_child(start)) if num_ch == 1: return atom_expr if start and num_ch == 2: - return ast.Await(atom_expr, power_node.get_lineno(), - power_node.get_column()) + return ast.Await(atom_expr, atom_node.get_lineno(), + atom_node.get_column()) for i in range(start+1, num_ch): - trailer = power_node.get_child(i) + trailer = atom_node.get_child(i) if trailer.type != syms.trailer: break tmp_atom_expr = self.handle_trailer(trailer, atom_expr) tmp_atom_expr.lineno = atom_expr.lineno tmp_atom_expr.col_offset = atom_expr.col_offset atom_expr = tmp_atom_expr + if start: + return ast.Await(atom_expr, atom_node.get_lineno(), + atom_node.get_column()) + else: + return atom_expr + + def handle_power(self, power_node): + atom_expr = self.handle_atom_expr(power_node.get_child(0)) + if power_node.num_children() == 1: + return atom_expr if power_node.get_child(-1).type == syms.factor: right = self.handle_expr(power_node.get_child(-1)) atom_expr = ast.BinOp(atom_expr, ast.Pow, right, power_node.get_lineno(), power_node.get_column()) - if start: - return ast.Await(atom_expr, power_node.get_lineno(), - power_node.get_column()) - else: - return atom_expr + return atom_expr def handle_slice(self, slice_node): first_child = slice_node.get_child(0) From pypy.commits at gmail.com Tue Jul 5 12:23:11 2016 From: pypy.commits at gmail.com (raffael_t) Date: Tue, 05 Jul 2016 09:23:11 -0700 (PDT) Subject: [pypy-commit] pypy py3.5-async: Add is_async check to for_stmt and with_stmt Message-ID: <577bdeef.e5acc20a.34847.ffff840d@mx.google.com> Author: Raffael Tfirst Branch: py3.5-async Changeset: r85569:3376dafd7080 Date: 2016-07-05 18:22 +0200 http://bitbucket.org/pypy/pypy/changeset/3376dafd7080/ Log: Add is_async check to for_stmt and with_stmt diff --git a/pypy/interpreter/astcompiler/astbuilder.py b/pypy/interpreter/astcompiler/astbuilder.py --- a/pypy/interpreter/astcompiler/astbuilder.py +++ b/pypy/interpreter/astcompiler/astbuilder.py @@ -353,7 +353,7 @@ return ast.While(loop_test, body, otherwise, while_node.get_lineno(), while_node.get_column()) - def handle_for_stmt(self, for_node): + def handle_for_stmt(self, for_node, is_async): target_node = for_node.get_child(1) target_as_exprlist = self.handle_exprlist(target_node, ast.Store) if target_node.num_children() == 1: @@ -367,8 +367,12 @@ otherwise = self.handle_suite(for_node.get_child(8)) else: otherwise = None - return ast.For(target, expr, body, otherwise, for_node.get_lineno(), - for_node.get_column()) + if is_async: + return ast.AsyncFor(target, expr, body, otherwise, for_node.get_lineno(), + for_node.get_column()) + else: + return ast.For(target, expr, body, otherwise, for_node.get_lineno(), + for_node.get_column()) def handle_except_clause(self, exc, body): test = None @@ -411,7 +415,7 @@ return ast.Try(body, handlers, otherwise, finally_suite, try_node.get_lineno(), try_node.get_column()) - def handle_with_stmt(self, with_node): + def handle_with_stmt(self, with_node, is_async): body = self.handle_suite(with_node.get_child(-1)) i = with_node.num_children() - 1 while True: @@ -423,8 +427,12 @@ self.set_context(target, ast.Store) else: target = None - wi = ast.With(test, target, body, with_node.get_lineno(), - with_node.get_column()) + if is_async: + wi = ast.AsyncWith(test, target, body, with_node.get_lineno(), + with_node.get_column()) + else: + wi = ast.With(test, target, body, with_node.get_lineno(), + with_node.get_column()) if i == 1: break body = [wi] @@ -488,7 +496,7 @@ body = self.handle_suite(funcdef_node.get_child(suite)) if is_async: return ast.AsyncFunctionDef(name, args, body, decorators, returns, - funcdef_node.get_lineno(), funcdef_node.get_column()) + funcdef_node.get_lineno(), funcdef_node.get_column()) else: return ast.FunctionDef(name, args, body, decorators, returns, funcdef_node.get_lineno(), funcdef_node.get_column()) @@ -717,17 +725,19 @@ elif stmt_type == syms.while_stmt: return self.handle_while_stmt(stmt) elif stmt_type == syms.for_stmt: - return self.handle_for_stmt(stmt) + return self.handle_for_stmt(stmt, 0) elif stmt_type == syms.try_stmt: return self.handle_try_stmt(stmt) elif stmt_type == syms.with_stmt: - return self.handle_with_stmt(stmt) + return self.handle_with_stmt(stmt, 0) elif stmt_type == syms.funcdef: return self.handle_funcdef(stmt) elif stmt_type == syms.classdef: return self.handle_classdef(stmt) elif stmt_type == syms.decorated: return self.handle_decorated(stmt) + elif stmt_type == syms.async_stmt: + return self.handle_async_stmt(stmt) else: raise AssertionError("unhandled compound statement") else: From pypy.commits at gmail.com Tue Jul 5 15:44:48 2016 From: pypy.commits at gmail.com (arigo) Date: Tue, 05 Jul 2016 12:44:48 -0700 (PDT) Subject: [pypy-commit] pypy.org extradoc: update the values Message-ID: <577c0e30.85c11c0a.38396.0291@mx.google.com> Author: Armin Rigo Branch: extradoc Changeset: r765:b1d29a113eec Date: 2016-07-05 21:46 +0200 http://bitbucket.org/pypy/pypy.org/changeset/b1d29a113eec/ Log: update the values diff --git a/don1.html b/don1.html --- a/don1.html +++ b/don1.html @@ -15,7 +15,7 @@ - $64643 of $105000 (61.6%) + $64677 of $105000 (61.6%)
    @@ -23,7 +23,7 @@
  • From pypy.commits at gmail.com Wed Jul 6 04:31:29 2016 From: pypy.commits at gmail.com (plan_rich) Date: Wed, 06 Jul 2016 01:31:29 -0700 (PDT) Subject: [pypy-commit] pypy new-jit-log: merge default Message-ID: <577cc1e1.8210c20a.f730d.7e16@mx.google.com> Author: Richard Plangger Branch: new-jit-log Changeset: r85570:4c3925486347 Date: 2016-07-05 17:13 +0200 http://bitbucket.org/pypy/pypy/changeset/4c3925486347/ Log: merge default diff too long, truncating to 2000 out of 5472 lines diff --git a/dotviewer/graphparse.py b/dotviewer/graphparse.py --- a/dotviewer/graphparse.py +++ b/dotviewer/graphparse.py @@ -85,10 +85,11 @@ pass def splitline(line, re_word = re.compile(r'[^\s"]\S*|["]["]|["].*?[^\\]["]')): + import ast result = [] for word in re_word.findall(line): if word.startswith('"'): - word = eval(word) + word = ast.literal_eval(word) result.append(word) return result diff --git a/lib_pypy/datetime.py b/lib_pypy/datetime.py --- a/lib_pypy/datetime.py +++ b/lib_pypy/datetime.py @@ -839,7 +839,7 @@ month = self._month if day is None: day = self._day - return date(year, month, day) + return date.__new__(type(self), year, month, day) # Comparisons of date objects with other. @@ -1356,7 +1356,8 @@ microsecond = self.microsecond if tzinfo is True: tzinfo = self.tzinfo - return time(hour, minute, second, microsecond, tzinfo) + return time.__new__(type(self), + hour, minute, second, microsecond, tzinfo) def __nonzero__(self): if self.second or self.microsecond: @@ -1566,8 +1567,9 @@ microsecond = self.microsecond if tzinfo is True: tzinfo = self.tzinfo - return datetime(year, month, day, hour, minute, second, microsecond, - tzinfo) + return datetime.__new__(type(self), + year, month, day, hour, minute, second, + microsecond, tzinfo) def astimezone(self, tz): if not isinstance(tz, tzinfo): diff --git a/pypy/doc/config/commandline.txt b/pypy/doc/config/commandline.txt --- a/pypy/doc/config/commandline.txt +++ b/pypy/doc/config/commandline.txt @@ -9,7 +9,7 @@ PyPy Python interpreter options ------------------------------- -The following options can be used after ``translate.py +The following options can be used after ``rpython targetpypystandalone`` or as options to ``py.py``. .. GENERATE: objspace @@ -22,7 +22,7 @@ General translation options --------------------------- -The following are options of ``translate.py``. They must be +The following are options of ``bin/rpython``. They must be given before the ``targetxxx`` on the command line. * `--opt -O:`__ set the optimization level `[0, 1, size, mem, 2, 3]` diff --git a/pypy/doc/config/index.rst b/pypy/doc/config/index.rst --- a/pypy/doc/config/index.rst +++ b/pypy/doc/config/index.rst @@ -15,12 +15,12 @@ ./py.py <`objspace options`_> -and the ``translate.py`` translation entry +and the ``rpython/bin/rpython`` translation entry point which takes arguments of this form: .. parsed-literal:: - ./translate.py <`translation options`_> + ./rpython/bin/rpython <`translation options`_> For the common case of ```` being ``targetpypystandalone.py``, you can then pass the `object space options`_ after @@ -28,7 +28,7 @@ .. parsed-literal:: - ./translate.py <`translation options`_> targetpypystandalone.py <`objspace options`_> + ./rpython/bin/rpython <`translation options`_> targetpypystandalone.py <`objspace options`_> There is an `overview`_ of all command line arguments that can be passed in either position. diff --git a/pypy/doc/config/opt.rst b/pypy/doc/config/opt.rst --- a/pypy/doc/config/opt.rst +++ b/pypy/doc/config/opt.rst @@ -4,8 +4,8 @@ This meta-option selects a default set of optimization settings to use during a translation. Usage:: - translate.py --opt=# - translate.py -O# + bin/rpython --opt=# + bin/rpython -O# where ``#`` is the desired optimization level. The valid choices are: diff --git a/pypy/doc/config/translation.dont_write_c_files.txt b/pypy/doc/config/translation.dont_write_c_files.txt --- a/pypy/doc/config/translation.dont_write_c_files.txt +++ b/pypy/doc/config/translation.dont_write_c_files.txt @@ -1,4 +1,4 @@ write the generated C files to ``/dev/null`` instead of to the disk. Useful if -you want to use translate.py as a benchmark and don't want to access the disk. +you want to use translation as a benchmark and don't want to access the disk. .. _`translation documentation`: ../translation.html diff --git a/pypy/doc/config/translation.fork_before.txt b/pypy/doc/config/translation.fork_before.txt --- a/pypy/doc/config/translation.fork_before.txt +++ b/pypy/doc/config/translation.fork_before.txt @@ -1,4 +1,4 @@ This is an option mostly useful when working on the PyPy toolchain. If you use -it, translate.py will fork before the specified phase. If the translation +it, translation will fork before the specified phase. If the translation crashes after that fork, you can fix the bug in the toolchain, and continue translation at the fork-point. diff --git a/pypy/doc/cppyy.rst b/pypy/doc/cppyy.rst --- a/pypy/doc/cppyy.rst +++ b/pypy/doc/cppyy.rst @@ -122,7 +122,7 @@ $ hg up reflex-support # optional # This example shows python, but using pypy-c is faster and uses less memory - $ python rpython/translator/goal/translate.py --opt=jit pypy/goal/targetpypystandalone --withmod-cppyy + $ python rpython/bin/rpython --opt=jit pypy/goal/targetpypystandalone --withmod-cppyy This will build a ``pypy-c`` that includes the cppyy module, and through that, Reflex support. diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -48,3 +48,8 @@ Bug fix: if ``socket.socket()`` failed, the ``socket.error`` did not show the errno of the failing system call, but instead some random previous errno. + +.. branch: PyTuple_Type-subclass + +Refactor PyTupleObject to look like cpython's and allow subclassing +PyTuple_Type diff --git a/pypy/interpreter/astcompiler/tools/asdl_py.py b/pypy/interpreter/astcompiler/tools/asdl_py.py --- a/pypy/interpreter/astcompiler/tools/asdl_py.py +++ b/pypy/interpreter/astcompiler/tools/asdl_py.py @@ -400,7 +400,7 @@ if not (space.isinstance_w(w_obj, space.w_str) or space.isinstance_w(w_obj, space.w_unicode)): raise oefmt(space.w_TypeError, - "AST string must be of type str or unicode") + "AST string must be of type str or unicode") return w_obj def get_field(space, w_node, name, optional): diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py --- a/pypy/interpreter/baseobjspace.py +++ b/pypy/interpreter/baseobjspace.py @@ -1034,7 +1034,7 @@ return (None, None) def newlist_bytes(self, list_s): - return self.newlist([self.wrap(s) for s in list_s]) + return self.newlist([self.newbytes(s) for s in list_s]) def newlist_unicode(self, list_u): return self.newlist([self.wrap(u) for u in list_u]) @@ -1533,7 +1533,7 @@ # unclear if there is any use at all for getting the bytes in # the unicode buffer.) try: - return self.str_w(w_obj) + return self.bytes_w(w_obj) except OperationError as e: if not e.match(self, self.w_TypeError): raise diff --git a/pypy/interpreter/pycode.py b/pypy/interpreter/pycode.py --- a/pypy/interpreter/pycode.py +++ b/pypy/interpreter/pycode.py @@ -408,14 +408,14 @@ w(self.co_nlocals), w(self.co_stacksize), w(self.co_flags), - w(self.co_code), + space.newbytes(self.co_code), space.newtuple(self.co_consts_w), space.newtuple(self.co_names_w), space.newtuple([w(v) for v in self.co_varnames]), w(self.co_filename), w(self.co_name), w(self.co_firstlineno), - w(self.co_lnotab), + space.newbytes(self.co_lnotab), space.newtuple([w(v) for v in self.co_freevars]), space.newtuple([w(v) for v in self.co_cellvars]), w(self.magic), diff --git a/pypy/interpreter/pyparser/automata.py b/pypy/interpreter/pyparser/automata.py --- a/pypy/interpreter/pyparser/automata.py +++ b/pypy/interpreter/pyparser/automata.py @@ -13,12 +13,11 @@ # PYPY Modification: removed the EMPTY class as it's not needed here -# PYPY Modification: we don't need a particuliar DEFAULT class here -# a simple None works fine. -# (Having a DefaultClass inheriting from str makes -# the annotator crash) -DEFAULT = "\00default" # XXX hack, the rtyper does not support dict of with str|None keys - # anyway using dicts doesn't seem the best final way to store these char indexed tables +# PYPY Modification: DEFAULT is a singleton, used only in the pre-RPython +# dicts (see pytokenize.py). Then DFA.__init__() turns these dicts into +# more compact strings. +DEFAULT = object() + # PYPY Modification : removed all automata functions (any, maybe, # newArcPair, etc.) diff --git a/pypy/interpreter/pyparser/genpytokenize.py b/pypy/interpreter/pyparser/genpytokenize.py --- a/pypy/interpreter/pyparser/genpytokenize.py +++ b/pypy/interpreter/pyparser/genpytokenize.py @@ -293,7 +293,7 @@ i = 0 for k, v in sorted(state.items()): i += 1 - if k == '\x00default': + if k == DEFAULT: k = "automata.DEFAULT" else: k = repr(k) diff --git a/pypy/interpreter/pyparser/parsestring.py b/pypy/interpreter/pyparser/parsestring.py --- a/pypy/interpreter/pyparser/parsestring.py +++ b/pypy/interpreter/pyparser/parsestring.py @@ -81,7 +81,7 @@ if need_encoding: enc = encoding v = PyString_DecodeEscape(space, substr, 'strict', enc) - return space.wrap(v) + return space.newbytes(v) def decode_unicode_utf8(space, s, ps, q): # ****The Python 2.7 version, producing UTF-32 escapes**** diff --git a/pypy/module/_cffi_backend/ctypeprim.py b/pypy/module/_cffi_backend/ctypeprim.py --- a/pypy/module/_cffi_backend/ctypeprim.py +++ b/pypy/module/_cffi_backend/ctypeprim.py @@ -84,7 +84,7 @@ if self.size == 1: with cdataobj as ptr: s = ptr[0] - return self.space.wrap(s) + return self.space.newbytes(s) return W_CType.string(self, cdataobj, maxlen) def unpack_ptr(self, w_ctypeptr, ptr, length): @@ -126,12 +126,12 @@ return self.space.wrap(ord(cdata[0])) def convert_to_object(self, cdata): - return self.space.wrap(cdata[0]) + return self.space.newbytes(cdata[0]) def _convert_to_char(self, w_ob): space = self.space if space.isinstance_w(w_ob, space.w_str): - s = space.str_w(w_ob) + s = space.bytes_w(w_ob) if len(s) == 1: return s[0] if (isinstance(w_ob, cdataobj.W_CData) and @@ -146,7 +146,7 @@ def unpack_ptr(self, w_ctypeptr, ptr, length): s = rffi.charpsize2str(ptr, length) - return self.space.wrapbytes(s) + return self.space.newbytes(s) # XXX explicitly use an integer type instead of lltype.UniChar here, diff --git a/pypy/module/_cffi_backend/ctypeptr.py b/pypy/module/_cffi_backend/ctypeptr.py --- a/pypy/module/_cffi_backend/ctypeptr.py +++ b/pypy/module/_cffi_backend/ctypeptr.py @@ -120,7 +120,7 @@ s = rffi.charp2str(ptr) else: s = rffi.charp2strn(ptr, length) - return space.wrapbytes(s) + return space.newbytes(s) # # pointer to a wchar_t: builds and returns a unicode if self.is_unichar_ptr_or_array(): @@ -129,7 +129,7 @@ u = rffi.wcharp2unicode(cdata) else: u = rffi.wcharp2unicoden(cdata, length) - return space.wrap(u) + return space.newunicode(u) # return W_CType.string(self, cdataobj, maxlen) diff --git a/pypy/module/_codecs/interp_codecs.py b/pypy/module/_codecs/interp_codecs.py --- a/pypy/module/_codecs/interp_codecs.py +++ b/pypy/module/_codecs/interp_codecs.py @@ -36,12 +36,14 @@ w_errorhandler = lookup_error(space, errors) if decode: w_cls = space.w_UnicodeDecodeError + w_input = space.newbytes(input) else: w_cls = space.w_UnicodeEncodeError + w_input = space.newunicode(input) w_exc = space.call_function( w_cls, space.wrap(encoding), - space.wrap(input), + w_input, space.wrap(startpos), space.wrap(endpos), space.wrap(reason)) @@ -314,7 +316,7 @@ @unwrap_spec(errors='str_or_None') def readbuffer_encode(space, w_data, errors='strict'): s = space.getarg_w('s#', w_data) - return space.newtuple([space.wrap(s), space.wrap(len(s))]) + return space.newtuple([space.newbytes(s), space.wrap(len(s))]) @unwrap_spec(errors='str_or_None') def charbuffer_encode(space, w_data, errors='strict'): @@ -377,7 +379,7 @@ state = space.fromcache(CodecState) func = getattr(runicode, rname) result = func(uni, len(uni), errors, state.encode_error_handler) - return space.newtuple([space.wrap(result), space.wrap(len(uni))]) + return space.newtuple([space.newbytes(result), space.wrap(len(uni))]) wrap_encoder.func_name = rname globals()[name] = wrap_encoder @@ -398,7 +400,7 @@ wrap_decoder.func_name = rname globals()[name] = wrap_decoder -for encoders in [ +for encoder in [ "ascii_encode", "latin_1_encode", "utf_7_encode", @@ -412,9 +414,9 @@ "raw_unicode_escape_encode", "unicode_internal_encode", ]: - make_encoder_wrapper(encoders) + make_encoder_wrapper(encoder) -for decoders in [ +for decoder in [ "ascii_decode", "latin_1_decode", "utf_7_decode", @@ -426,7 +428,7 @@ "utf_32_le_decode", "raw_unicode_escape_decode", ]: - make_decoder_wrapper(decoders) + make_decoder_wrapper(decoder) if hasattr(runicode, 'str_decode_mbcs'): make_encoder_wrapper('mbcs_encode') @@ -560,7 +562,7 @@ if space.isinstance_w(w_ch, space.w_str): # Charmap may return a string - return space.str_w(w_ch) + return space.bytes_w(w_ch) elif space.isinstance_w(w_ch, space.w_int): # Charmap may return a number x = space.int_w(w_ch) @@ -608,7 +610,7 @@ result = runicode.unicode_encode_charmap( uni, len(uni), errors, state.encode_error_handler, mapping) - return space.newtuple([space.wrap(result), space.wrap(len(uni))]) + return space.newtuple([space.newbytes(result), space.wrap(len(uni))]) @unwrap_spec(chars=unicode) @@ -696,4 +698,4 @@ def escape_decode(space, data, errors='strict'): from pypy.interpreter.pyparser.parsestring import PyString_DecodeEscape result = PyString_DecodeEscape(space, data, errors, None) - return space.newtuple([space.wrap(result), space.wrap(len(data))]) + return space.newtuple([space.newbytes(result), space.wrap(len(data))]) diff --git a/pypy/module/_hashlib/interp_hashlib.py b/pypy/module/_hashlib/interp_hashlib.py --- a/pypy/module/_hashlib/interp_hashlib.py +++ b/pypy/module/_hashlib/interp_hashlib.py @@ -111,7 +111,7 @@ def digest(self, space): "Return the digest value as a string of binary data." digest = self._digest(space) - return space.wrap(digest) + return space.newbytes(digest) def hexdigest(self, space): "Return the digest value as a string of hexadecimal digits." diff --git a/pypy/module/_io/interp_bufferedio.py b/pypy/module/_io/interp_bufferedio.py --- a/pypy/module/_io/interp_bufferedio.py +++ b/pypy/module/_io/interp_bufferedio.py @@ -343,7 +343,7 @@ self._writer_reset_buf() def _write(self, space, data): - w_data = space.wrap(data) + w_data = space.newbytes(data) while True: try: w_written = space.call_method(self.w_raw, "write", w_data) @@ -415,7 +415,7 @@ else: raise oefmt(space.w_ValueError, "read length must be positive or -1") - return space.wrap(res) + return space.newbytes(res) @unwrap_spec(size=int) def peek_w(self, space, size=0): @@ -432,7 +432,7 @@ have = self._readahead() if have > 0: data = ''.join(self.buffer[self.pos:self.pos+have]) - return space.wrap(data) + return space.newbytes(data) # Fill the buffer from the raw stream, and copy it to the result self._reader_reset_buf() @@ -442,7 +442,7 @@ size = 0 self.pos = 0 data = ''.join(self.buffer[:size]) - return space.wrap(data) + return space.newbytes(data) @unwrap_spec(size=int) def read1_w(self, space, size): @@ -452,7 +452,7 @@ if size < 0: raise oefmt(space.w_ValueError, "read length must be positive") if size == 0: - return space.wrap("") + return space.newbytes("") with self.lock: # Return up to n bytes. If at least one byte is buffered, we only @@ -480,7 +480,7 @@ endpos = self.pos + size data = ''.join(self.buffer[self.pos:endpos]) self.pos = endpos - return space.wrap(data) + return space.newbytes(data) def _read_all(self, space): "Read all the file, don't update the cache" @@ -505,7 +505,7 @@ if current_size == 0: return w_data break - data = space.str_w(w_data) + data = space.bytes_w(w_data) size = len(data) if size == 0: break @@ -513,7 +513,7 @@ current_size += size if self.abs_pos != -1: self.abs_pos += size - return space.wrap(builder.build()) + return space.newbytes(builder.build()) def _raw_read(self, space, buffer, start, length): length = intmask(length) @@ -644,11 +644,11 @@ else: pos = -1 if pos >= 0: - w_res = space.wrap(''.join(self.buffer[self.pos:pos+1])) + w_res = space.newbytes(''.join(self.buffer[self.pos:pos+1])) self.pos = pos + 1 return w_res if have == limit: - w_res = space.wrap(''.join(self.buffer[self.pos:self.pos+have])) + w_res = space.newbytes(''.join(self.buffer[self.pos:self.pos+have])) self.pos += have return w_res @@ -690,7 +690,7 @@ written += have if limit >= 0: limit -= have - return space.wrap(''.join(chunks)) + return space.newbytes(''.join(chunks)) # ____________________________________________________ # Write methods @@ -1024,7 +1024,6 @@ self._deprecated_max_buffer_size(space) self.state = STATE_ZERO - check_readable_w(space, w_raw) check_writable_w(space, w_raw) check_seekable_w(space, w_raw) diff --git a/pypy/module/_io/interp_bytesio.py b/pypy/module/_io/interp_bytesio.py --- a/pypy/module/_io/interp_bytesio.py +++ b/pypy/module/_io/interp_bytesio.py @@ -32,12 +32,12 @@ def read_w(self, space, w_size=None): self._check_closed(space) size = convert_size(space, w_size) - return space.wrap(self.read(size)) + return space.newbytes(self.read(size)) def readline_w(self, space, w_limit=None): self._check_closed(space) limit = convert_size(space, w_limit) - return space.wrap(self.readline(limit)) + return space.newbytes(self.readline(limit)) def read1_w(self, space, w_size): return self.read_w(space, w_size) @@ -81,7 +81,7 @@ def getvalue_w(self, space): self._check_closed(space) - return space.wrap(self.getvalue()) + return space.newbytes(self.getvalue()) def tell_w(self, space): self._check_closed(space) @@ -128,7 +128,7 @@ def getstate_w(self, space): self._check_closed(space) return space.newtuple([ - space.wrap(self.getvalue()), + space.newbytes(self.getvalue()), space.wrap(self.tell()), self.getdict(space)]) diff --git a/pypy/module/_io/interp_fileio.py b/pypy/module/_io/interp_fileio.py --- a/pypy/module/_io/interp_fileio.py +++ b/pypy/module/_io/interp_fileio.py @@ -361,7 +361,7 @@ raise wrap_oserror(space, e, exception_name='w_IOError') - return space.wrap(s) + return space.newbytes(s) def readinto_w(self, space, w_buffer): self._check_closed(space) @@ -405,7 +405,7 @@ break builder.append(chunk) total += len(chunk) - return space.wrap(builder.build()) + return space.newbytes(builder.build()) if sys.platform == "win32": def _truncate(self, size): diff --git a/pypy/module/_io/interp_iobase.py b/pypy/module/_io/interp_iobase.py --- a/pypy/module/_io/interp_iobase.py +++ b/pypy/module/_io/interp_iobase.py @@ -192,7 +192,7 @@ length = space.len_w(w_readahead) if length > 0: n = 0 - buf = space.str_w(w_readahead) + buf = space.bytes_w(w_readahead) if limit >= 0: while True: if n >= length or n >= limit: @@ -219,7 +219,7 @@ raise oefmt(space.w_IOError, "peek() should have returned a bytes object, not " "'%T'", w_read) - read = space.str_w(w_read) + read = space.bytes_w(w_read) if not read: break @@ -229,7 +229,7 @@ if read[-1] == '\n': break - return space.wrap(builder.build()) + return space.newbytes(builder.build()) def readlines_w(self, space, w_hint=None): hint = convert_size(space, w_hint) @@ -339,11 +339,11 @@ if not space.isinstance_w(w_data, space.w_str): raise oefmt(space.w_TypeError, "read() should return bytes") - data = space.str_w(w_data) + data = space.bytes_w(w_data) if not data: break builder.append(data) - return space.wrap(builder.build()) + return space.newbytes(builder.build()) W_RawIOBase.typedef = TypeDef( '_io._RawIOBase', W_IOBase.typedef, diff --git a/pypy/module/_io/interp_textio.py b/pypy/module/_io/interp_textio.py --- a/pypy/module/_io/interp_textio.py +++ b/pypy/module/_io/interp_textio.py @@ -160,7 +160,7 @@ w_buffer, w_flag = space.unpackiterable(w_state, 2) flag = space.r_longlong_w(w_flag) else: - w_buffer = space.wrap("") + w_buffer = space.newbytes("") flag = 0 flag <<= 1 if self.pendingcr: @@ -556,7 +556,7 @@ # Given this, we know there was a valid snapshot point # len(dec_buffer) bytes ago with decoder state (b'', dec_flags). w_dec_buffer, w_dec_flags = space.unpackiterable(w_state, 2) - dec_buffer = space.str_w(w_dec_buffer) + dec_buffer = space.bytes_w(w_dec_buffer) dec_flags = space.int_w(w_dec_flags) else: dec_buffer = None @@ -582,7 +582,7 @@ if self.telling: # At the snapshot point, len(dec_buffer) bytes before the read, # the next input to be decoded is dec_buffer + input_chunk. - next_input = dec_buffer + space.str_w(w_input) + next_input = dec_buffer + space.bytes_w(w_input) self.snapshot = PositionSnapshot(dec_flags, next_input) return not eof @@ -769,7 +769,7 @@ else: w_bytes = space.call_method(self.w_encoder, "encode", w_text) - b = space.str_w(w_bytes) + b = space.bytes_w(w_bytes) if not self.pending_bytes: self.pending_bytes = [] self.pending_bytes_count = 0 @@ -799,7 +799,8 @@ while True: try: - space.call_method(self.w_buffer, "write", space.wrap(pending_bytes)) + space.call_method(self.w_buffer, "write", + space.newbytes(pending_bytes)) except OperationError as e: if trap_eintr(space, e): continue @@ -828,7 +829,7 @@ space.call_method(self.w_decoder, "reset") else: space.call_method(self.w_decoder, "setstate", - space.newtuple([space.wrap(""), + space.newtuple([space.newbytes(""), space.wrap(cookie.dec_flags)])) def _encoder_setstate(self, space, cookie): @@ -904,7 +905,7 @@ raise oefmt(space.w_TypeError, msg, w_chunk) self.snapshot = PositionSnapshot(cookie.dec_flags, - space.str_w(w_chunk)) + space.bytes_w(w_chunk)) w_decoded = space.call_method(self.w_decoder, "decode", w_chunk, space.wrap(cookie.need_eof)) @@ -975,7 +976,7 @@ i = 0 while i < len(input): w_decoded = space.call_method(self.w_decoder, "decode", - space.wrap(input[i])) + space.newbytes(input[i])) check_decoded(space, w_decoded) chars_decoded += len(space.unicode_w(w_decoded)) diff --git a/pypy/module/_md5/interp_md5.py b/pypy/module/_md5/interp_md5.py --- a/pypy/module/_md5/interp_md5.py +++ b/pypy/module/_md5/interp_md5.py @@ -20,7 +20,7 @@ self.update(string) def digest_w(self): - return self.space.wrap(self.digest()) + return self.space.newbytes(self.digest()) def hexdigest_w(self): return self.space.wrap(self.hexdigest()) diff --git a/pypy/module/_minimal_curses/interp_curses.py b/pypy/module/_minimal_curses/interp_curses.py --- a/pypy/module/_minimal_curses/interp_curses.py +++ b/pypy/module/_minimal_curses/interp_curses.py @@ -83,12 +83,12 @@ return space.w_None except curses_error as e: raise convert_error(space, e) - return space.wrap(result) + return space.newbytes(result) @unwrap_spec(s=str) def tparm(space, s, args_w): args = [space.int_w(a) for a in args_w] try: - return space.wrap(_curses_tparm(s, args)) + return space.newbytes(_curses_tparm(s, args)) except curses_error as e: raise convert_error(space, e) diff --git a/pypy/module/_multibytecodec/interp_incremental.py b/pypy/module/_multibytecodec/interp_incremental.py --- a/pypy/module/_multibytecodec/interp_incremental.py +++ b/pypy/module/_multibytecodec/interp_incremental.py @@ -113,7 +113,7 @@ pos = c_codecs.pypy_cjk_enc_inbuf_consumed(self.encodebuf) assert 0 <= pos <= len(object) self.pending = object[pos:] - return space.wrap(output) + return space.newbytes(output) @unwrap_spec(errors="str_or_None") diff --git a/pypy/module/_multibytecodec/interp_multibytecodec.py b/pypy/module/_multibytecodec/interp_multibytecodec.py --- a/pypy/module/_multibytecodec/interp_multibytecodec.py +++ b/pypy/module/_multibytecodec/interp_multibytecodec.py @@ -40,7 +40,7 @@ raise wrap_unicodeencodeerror(space, e, input, self.name) except RuntimeError: raise wrap_runtimeerror(space) - return space.newtuple([space.wrap(output), + return space.newtuple([space.newbytes(output), space.wrap(len(input))]) @@ -66,7 +66,7 @@ space.w_UnicodeDecodeError, space.newtuple([ space.wrap(name), - space.wrap(input), + space.newbytes(input), space.wrap(e.start), space.wrap(e.end), space.wrap(e.reason)])) diff --git a/pypy/module/_multiprocessing/interp_connection.py b/pypy/module/_multiprocessing/interp_connection.py --- a/pypy/module/_multiprocessing/interp_connection.py +++ b/pypy/module/_multiprocessing/interp_connection.py @@ -122,9 +122,9 @@ space, self.BUFFER_SIZE, maxlength) try: if newbuf: - return space.wrap(rffi.charpsize2str(newbuf, res)) + return space.newbytes(rffi.charpsize2str(newbuf, res)) else: - return space.wrap(rffi.charpsize2str(self.buffer, res)) + return space.newbytes(rffi.charpsize2str(self.buffer, res)) finally: if newbuf: rffi.free_charp(newbuf) @@ -138,7 +138,7 @@ space, length - offset, PY_SSIZE_T_MAX) try: if newbuf: - raise BufferTooShort(space, space.wrap( + raise BufferTooShort(space, space.newbytes( rffi.charpsize2str(newbuf, res))) rwbuffer.setslice(offset, rffi.charpsize2str(self.buffer, res)) finally: @@ -166,9 +166,9 @@ space, self.BUFFER_SIZE, PY_SSIZE_T_MAX) try: if newbuf: - w_received = space.wrap(rffi.charpsize2str(newbuf, res)) + w_received = space.newbytes(rffi.charpsize2str(newbuf, res)) else: - w_received = space.wrap(rffi.charpsize2str(self.buffer, res)) + w_received = space.newbytes(rffi.charpsize2str(self.buffer, res)) finally: if newbuf: rffi.free_charp(newbuf) diff --git a/pypy/module/_rawffi/alt/test/test_type_converter.py b/pypy/module/_rawffi/alt/test/test_type_converter.py --- a/pypy/module/_rawffi/alt/test/test_type_converter.py +++ b/pypy/module/_rawffi/alt/test/test_type_converter.py @@ -12,14 +12,14 @@ handle_signed = handle_all handle_unsigned = handle_all handle_pointer = handle_all - handle_char = handle_all + handle_char = handle_all handle_unichar = handle_all handle_longlong = handle_all handle_char_p = handle_all handle_unichar_p = handle_all handle_float = handle_all handle_singlefloat = handle_all - + def handle_struct(self, w_ffitype, w_structinstance): self.lastval = w_structinstance @@ -119,12 +119,12 @@ def test_strings(self): # first, try automatic conversion from applevel - self.check(app_types.char_p, self.space.wrap('foo'), 'foo') - self.check(app_types.unichar_p, self.space.wrap(u'foo\u1234'), u'foo\u1234') - self.check(app_types.unichar_p, self.space.wrap('foo'), u'foo') + self.check(app_types.char_p, self.space.newbytes('foo'), 'foo') + self.check(app_types.unichar_p, self.space.wrap(u'foo\u1234'), u'foo\u1234') + self.check(app_types.unichar_p, self.space.wrap('foo'), u'foo') # then, try to pass explicit pointers self.check(app_types.char_p, self.space.wrap(42), 42) - self.check(app_types.unichar_p, self.space.wrap(42), 42) + self.check(app_types.unichar_p, self.space.wrap(42), 42) @@ -136,7 +136,7 @@ get_signed = get_all get_unsigned = get_all get_pointer = get_all - get_char = get_all + get_char = get_all get_unichar = get_all get_longlong = get_all get_char_p = get_all @@ -144,7 +144,7 @@ get_float = get_all get_singlefloat = get_all get_unsigned_which_fits_into_a_signed = get_all - + def convert(self, w_ffitype, val): self.val = val return self.do_and_wrap(w_ffitype) diff --git a/pypy/module/_rawffi/array.py b/pypy/module/_rawffi/array.py --- a/pypy/module/_rawffi/array.py +++ b/pypy/module/_rawffi/array.py @@ -181,7 +181,7 @@ start, stop = self.decodeslice(space, w_slice) ll_buffer = self.ll_buffer result = [ll_buffer[i] for i in range(start, stop)] - return space.wrap(''.join(result)) + return space.newbytes(''.join(result)) def setslice(self, space, w_slice, w_value): start, stop = self.decodeslice(space, w_slice) diff --git a/pypy/module/_rawffi/interp_rawffi.py b/pypy/module/_rawffi/interp_rawffi.py --- a/pypy/module/_rawffi/interp_rawffi.py +++ b/pypy/module/_rawffi/interp_rawffi.py @@ -570,7 +570,7 @@ s = rffi.charp2str(charp_addr) else: s = rffi.charp2strn(charp_addr, maxlength) - return space.wrap(s) + return space.newbytes(s) @unwrap_spec(address=r_uint, maxlength=int) def wcharp2unicode(space, address, maxlength=-1): @@ -588,7 +588,7 @@ if maxlength == -1: return charp2string(space, address) s = rffi.charpsize2str(rffi.cast(rffi.CCHARP, address), maxlength) - return space.wrap(s) + return space.newbytes(s) @unwrap_spec(address=r_uint, maxlength=int) def wcharp2rawunicode(space, address, maxlength=-1): diff --git a/pypy/module/_socket/interp_func.py b/pypy/module/_socket/interp_func.py --- a/pypy/module/_socket/interp_func.py +++ b/pypy/module/_socket/interp_func.py @@ -209,7 +209,7 @@ buf = rsocket.inet_aton(ip) except SocketError as e: raise converted_error(space, e) - return space.wrap(buf) + return space.newbytes(buf) @unwrap_spec(packed=str) def inet_ntoa(space, packed): @@ -234,7 +234,7 @@ buf = rsocket.inet_pton(family, ip) except SocketError as e: raise converted_error(space, e) - return space.wrap(buf) + return space.newbytes(buf) @unwrap_spec(family=int, packed=str) def inet_ntop(space, family, packed): @@ -263,10 +263,10 @@ if space.is_w(w_host, space.w_None): host = None elif space.isinstance_w(w_host, space.w_str): - host = space.str_w(w_host) + host = space.bytes_w(w_host) elif space.isinstance_w(w_host, space.w_unicode): w_shost = space.call_method(w_host, "encode", space.wrap("idna")) - host = space.str_w(w_shost) + host = space.bytes_w(w_shost) else: raise oefmt(space.w_TypeError, "getaddrinfo() argument 1 must be string or None") @@ -277,7 +277,7 @@ elif space.isinstance_w(w_port, space.w_int) or space.isinstance_w(w_port, space.w_long): port = str(space.int_w(w_port)) elif space.isinstance_w(w_port, space.w_str): - port = space.str_w(w_port) + port = space.bytes_w(w_port) else: raise oefmt(space.w_TypeError, "getaddrinfo() argument 2 must be integer or string") diff --git a/pypy/module/_socket/interp_socket.py b/pypy/module/_socket/interp_socket.py --- a/pypy/module/_socket/interp_socket.py +++ b/pypy/module/_socket/interp_socket.py @@ -296,7 +296,7 @@ except SocketError as e: raise converted_error(space, e) buflen = space.int_w(w_buflen) - return space.wrap(self.sock.getsockopt(level, optname, buflen)) + return space.newbytes(self.sock.getsockopt(level, optname, buflen)) def gettimeout_w(self, space): """gettimeout() -> timeout @@ -345,7 +345,7 @@ data = self.sock.recv(buffersize, flags) except SocketError as e: raise converted_error(space, e) - return space.wrap(data) + return space.newbytes(data) @unwrap_spec(buffersize='nonnegint', flags=int) def recvfrom_w(self, space, buffersize, flags=0): @@ -359,7 +359,7 @@ w_addr = addr_as_object(addr, self.sock.fd, space) else: w_addr = space.w_None - return space.newtuple([space.wrap(data), w_addr]) + return space.newtuple([space.newbytes(data), w_addr]) except SocketError as e: raise converted_error(space, e) @@ -436,7 +436,7 @@ except OperationError as e: if e.async(space): raise - optval = space.str_w(w_optval) + optval = space.bytes_w(w_optval) try: self.sock.setsockopt(level, optname, optval) except SocketError as e: diff --git a/pypy/module/_sre/interp_sre.py b/pypy/module/_sre/interp_sre.py --- a/pypy/module/_sre/interp_sre.py +++ b/pypy/module/_sre/interp_sre.py @@ -36,11 +36,12 @@ def slice_w(space, ctx, start, end, w_default): if 0 <= start <= end: if isinstance(ctx, rsre_core.BufMatchContext): - return space.wrap(ctx._buffer.getslice(start, end, 1, end-start)) + return space.newbytes(ctx._buffer.getslice(start, end, 1, + end-start)) if isinstance(ctx, rsre_core.StrMatchContext): - return space.wrap(ctx._string[start:end]) + return space.newbytes(ctx._string[start:end]) elif isinstance(ctx, rsre_core.UnicodeMatchContext): - return space.wrap(ctx._unicodestr[start:end]) + return space.newunicode(ctx._unicodestr[start:end]) else: # unreachable raise SystemError @@ -242,7 +243,7 @@ space.isinstance_w(w_string, space.w_unicode) and literal) else: try: - filter_as_string = space.str_w(w_ptemplate) + filter_as_string = space.bytes_w(w_ptemplate) except OperationError as e: if e.async(space): raise @@ -331,15 +332,15 @@ strbuilder, unicodebuilder, last_pos, ctx.end) if use_builder: if strbuilder is not None: - return space.wrap(strbuilder.build()), n + return space.newbytes(strbuilder.build()), n else: assert unicodebuilder is not None - return space.wrap(unicodebuilder.build()), n + return space.newunicode(unicodebuilder.build()), n else: if space.isinstance_w(w_string, space.w_unicode): - w_emptystr = space.wrap(u'') + w_emptystr = space.newunicode(u'') else: - w_emptystr = space.wrap('') + w_emptystr = space.newbytes('') w_item = space.call_method(w_emptystr, 'join', space.newlist(sublist_w)) return w_item, n @@ -565,11 +566,11 @@ def fget_string(self, space): ctx = self.ctx if isinstance(ctx, rsre_core.BufMatchContext): - return space.wrap(ctx._buffer.as_str()) + return space.newbytes(ctx._buffer.as_str()) elif isinstance(ctx, rsre_core.StrMatchContext): - return space.wrap(ctx._string) + return space.newbytes(ctx._string) elif isinstance(ctx, rsre_core.UnicodeMatchContext): - return space.wrap(ctx._unicodestr) + return space.newunicode(ctx._unicodestr) else: raise SystemError diff --git a/pypy/module/_ssl/interp_ssl.py b/pypy/module/_ssl/interp_ssl.py --- a/pypy/module/_ssl/interp_ssl.py +++ b/pypy/module/_ssl/interp_ssl.py @@ -666,7 +666,7 @@ length = libssl_SSL_get_peer_finished(self.ssl, buf, CB_MAXLEN) if length > 0: - return space.wrap(rffi.charpsize2str(buf, intmask(length))) + return space.newbytes(rffi.charpsize2str(buf, intmask(length))) def descr_get_context(self, space): return self.w_ctx @@ -707,7 +707,7 @@ if length < 0: raise _ssl_seterror(space, None, 0) try: - return space.wrap(rffi.charpsize2str(buf_ptr[0], length)) + return space.newbytes(rffi.charpsize2str(buf_ptr[0], length)) finally: libssl_OPENSSL_free(buf_ptr[0]) @@ -926,7 +926,7 @@ if length < 0: raise _ssl_seterror(space, None, 0) try: - w_value = space.wrap(rffi.charpsize2str(buf_ptr[0], length)) + w_value = space.newbytes(rffi.charpsize2str(buf_ptr[0], length)) w_value = space.call_method(w_value, "decode", space.wrap("utf-8")) finally: libssl_OPENSSL_free(buf_ptr[0]) @@ -1232,7 +1232,7 @@ w_ssl_socket, space.w_None, w_ctx) else: - w_servername = space.wrapbytes(rffi.charp2str(servername)) + w_servername = space.newbytes(rffi.charp2str(servername)) try: w_servername_idna = space.call_method( w_servername, 'decode', space.wrap('idna')) @@ -1778,7 +1778,7 @@ if not path: return space.w_None else: - return space.wrapbytes(rffi.charp2str(path)) + return space.newbytes(rffi.charp2str(path)) def get_default_verify_paths(space): return space.newtuple([ diff --git a/pypy/module/_ssl/interp_win32.py b/pypy/module/_ssl/interp_win32.py --- a/pypy/module/_ssl/interp_win32.py +++ b/pypy/module/_ssl/interp_win32.py @@ -74,7 +74,7 @@ def w_parseKeyUsage(space, pCertCtx, flags): with lltype.scoped_alloc(rwin32.LPDWORD.TO, 1) as size_ptr: - if not CertGetEnhancedKeyUsage(pCertCtx, flags, + if not CertGetEnhancedKeyUsage(pCertCtx, flags, lltype.nullptr(CERT_ENHKEY_USAGE), size_ptr): last_error = rwin32.lastSavedWindowsError() if last_error.winerror == CRYPT_E_NOT_FOUND: @@ -120,7 +120,7 @@ pCertCtx = CertEnumCertificatesInStore(hStore, pCertCtx) if not pCertCtx: break - w_cert = space.wrapbytes( + w_cert = space.newbytes( rffi.charpsize2str(pCertCtx.c_pbCertEncoded, intmask(pCertCtx.c_cbCertEncoded))) w_enc = w_certEncodingType(space, pCertCtx.c_dwCertEncodingType) @@ -162,7 +162,7 @@ pCrlCtx = CertEnumCRLsInStore(hStore, pCrlCtx) if not pCrlCtx: break - w_crl = space.wrapbytes( + w_crl = space.newbytes( rffi.charpsize2str(pCrlCtx.c_pbCrlEncoded, intmask(pCrlCtx.c_cbCrlEncoded))) w_enc = w_certEncodingType(space, pCrlCtx.c_dwCertEncodingType) diff --git a/pypy/module/_winreg/interp_winreg.py b/pypy/module/_winreg/interp_winreg.py --- a/pypy/module/_winreg/interp_winreg.py +++ b/pypy/module/_winreg/interp_winreg.py @@ -380,7 +380,7 @@ return space.newlist(l) else: # REG_BINARY and all other types - return space.wrap(rffi.charpsize2str(buf, buflen)) + return space.newbytes(rffi.charpsize2str(buf, buflen)) @unwrap_spec(value_name=str, typ=int) def SetValueEx(space, w_hkey, value_name, w_reserved, typ, w_value): diff --git a/pypy/module/array/interp_array.py b/pypy/module/array/interp_array.py --- a/pypy/module/array/interp_array.py +++ b/pypy/module/array/interp_array.py @@ -225,11 +225,11 @@ """ size = self.len if size == 0: - return space.wrap('') + return space.newbytes('') cbuf = self._charbuf_start() s = rffi.charpsize2str(cbuf, size * self.itemsize) self._charbuf_stop() - return self.space.wrap(s) + return self.space.newbytes(s) def descr_fromstring(self, space, w_s): """ fromstring(string) @@ -263,7 +263,7 @@ except OverflowError: raise MemoryError w_item = space.call_method(w_f, 'read', space.wrap(size)) - item = space.str_w(w_item) + item = space.bytes_w(w_item) if len(item) < size: n = len(item) % self.itemsize elems = max(0, len(item) - (len(item) % self.itemsize)) @@ -338,10 +338,10 @@ else: args = [space.wrap(self.typecode)] try: - dct = space.getattr(self, space.wrap('__dict__')) + w_dict = space.getattr(self, space.wrap('__dict__')) except OperationError: - dct = space.w_None - return space.newtuple([space.type(self), space.newtuple(args), dct]) + w_dict = space.w_None + return space.newtuple([space.type(self), space.newtuple(args), w_dict]) def descr_copy(self, space): """ copy(array) diff --git a/pypy/module/binascii/interp_base64.py b/pypy/module/binascii/interp_base64.py --- a/pypy/module/binascii/interp_base64.py +++ b/pypy/module/binascii/interp_base64.py @@ -71,7 +71,7 @@ if leftbits != 0: raise_Error(space, "Incorrect padding") - return space.wrap(res.build()) + return space.newbytes(res.build()) # ____________________________________________________________ @@ -110,4 +110,4 @@ res.append(table_b2a_base64[(leftchar & 0xf) << 2]) res.append(PAD) res.append('\n') - return space.wrap(res.build()) + return space.newbytes(res.build()) diff --git a/pypy/module/binascii/interp_hexlify.py b/pypy/module/binascii/interp_hexlify.py --- a/pypy/module/binascii/interp_hexlify.py +++ b/pypy/module/binascii/interp_hexlify.py @@ -24,7 +24,7 @@ for c in data: res.append(_value2char(ord(c) >> 4)) res.append(_value2char(ord(c) & 0xf)) - return space.wrap(res.build()) + return space.newbytes(res.build()) # ____________________________________________________________ @@ -53,4 +53,4 @@ a = _char2value(space, hexstr[i]) b = _char2value(space, hexstr[i+1]) res.append(chr((a << 4) | b)) - return space.wrap(res.build()) + return space.newbytes(res.build()) diff --git a/pypy/module/binascii/interp_hqx.py b/pypy/module/binascii/interp_hqx.py --- a/pypy/module/binascii/interp_hqx.py +++ b/pypy/module/binascii/interp_hqx.py @@ -11,37 +11,37 @@ FAIL = 0x7d table_a2b_hqx = [ - #^@ ^A ^B ^C ^D ^E ^F ^G + #^@ ^A ^B ^C ^D ^E ^F ^G FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, - #\b \t \n ^K ^L \r ^N ^O + #\b \t \n ^K ^L \r ^N ^O FAIL, FAIL, SKIP, FAIL, FAIL, SKIP, FAIL, FAIL, - #^P ^Q ^R ^S ^T ^U ^V ^W + #^P ^Q ^R ^S ^T ^U ^V ^W FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, - #^X ^Y ^Z ^[ ^\ ^] ^^ ^_ + #^X ^Y ^Z ^[ ^\ ^] ^^ ^_ FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, - # ! " # $ % & ' + # ! " # $ % & ' FAIL, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, - #( ) * + , - . / + #( ) * + , - . / 0x07, 0x08, 0x09, 0x0A, 0x0B, 0x0C, FAIL, FAIL, - #0 1 2 3 4 5 6 7 + #0 1 2 3 4 5 6 7 0x0D, 0x0E, 0x0F, 0x10, 0x11, 0x12, 0x13, FAIL, - #8 9 : ; < = > ? + #8 9 : ; < = > ? 0x14, 0x15, DONE, FAIL, FAIL, FAIL, FAIL, FAIL, - #@ A B C D E F G + #@ A B C D E F G 0x16, 0x17, 0x18, 0x19, 0x1A, 0x1B, 0x1C, 0x1D, - #H I J K L M N O + #H I J K L M N O 0x1E, 0x1F, 0x20, 0x21, 0x22, 0x23, 0x24, FAIL, - #P Q R S T U V W + #P Q R S T U V W 0x25, 0x26, 0x27, 0x28, 0x29, 0x2A, 0x2B, FAIL, - #X Y Z [ \ ] ^ _ + #X Y Z [ \ ] ^ _ 0x2C, 0x2D, 0x2E, 0x2F, FAIL, FAIL, FAIL, FAIL, - #` a b c d e f g + #` a b c d e f g 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, FAIL, - #h i j k l m n o + #h i j k l m n o 0x37, 0x38, 0x39, 0x3A, 0x3B, 0x3C, FAIL, FAIL, - #p q r s t u v w + #p q r s t u v w 0x3D, 0x3E, 0x3F, FAIL, FAIL, FAIL, FAIL, FAIL, - #x y z { | } ~ ^? + #x y z { | } ~ ^? FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, @@ -97,7 +97,7 @@ else: if pending_bits > 0: raise_Incomplete(space, 'String has incomplete number of bytes') - return space.newtuple([space.wrap(res.build()), space.wrap(done)]) + return space.newtuple([space.newbytes(res.build()), space.wrap(done)]) # ____________________________________________________________ @@ -128,7 +128,7 @@ if leftbits > 0: leftchar <<= (6 - leftbits) res.append(hqx_encoding[leftchar & 0x3f]) - return space.wrap(res.build()) + return space.newbytes(res.build()) # ____________________________________________________________ @@ -150,7 +150,7 @@ lastpushed = ord(c) else: if i == end: - raise_Incomplete(space, 'String ends with the RLE code \x90') + raise_Incomplete(space, 'String ends with the RLE code \\x90') count = ord(hexbin[i]) - 1 i += 1 if count < 0: @@ -158,9 +158,9 @@ lastpushed = 0x90 else: if lastpushed < 0: - raise_Error(space, 'String starts with the RLE code \x90') + raise_Error(space, 'String starts with the RLE code \\x90') res.append_multiple_char(chr(lastpushed), count) - return space.wrap(res.build()) + return space.newbytes(res.build()) # ____________________________________________________________ @@ -197,7 +197,7 @@ # string that rledecode_hqx() would expand back to 'data', there are # some programs somewhere that would start failing obscurely in rare # cases. - return space.wrap(res.build()) + return space.newbytes(res.build()) # ____________________________________________________________ diff --git a/pypy/module/binascii/interp_qp.py b/pypy/module/binascii/interp_qp.py --- a/pypy/module/binascii/interp_qp.py +++ b/pypy/module/binascii/interp_qp.py @@ -56,7 +56,7 @@ if header and c == '_': c = ' ' odata.append(c) - return space.wrap(odata.build()) + return space.newbytes(odata.build()) # ____________________________________________________________ @@ -159,4 +159,4 @@ odata.append(c) inp += 1 - return space.wrap(odata.build()) + return space.newbytes(odata.build()) diff --git a/pypy/module/binascii/interp_uu.py b/pypy/module/binascii/interp_uu.py --- a/pypy/module/binascii/interp_uu.py +++ b/pypy/module/binascii/interp_uu.py @@ -54,7 +54,7 @@ remaining = length - res.getlength() if remaining > 0: res.append_multiple_char('\x00', remaining) - return space.wrap(res.build()) + return space.newbytes(res.build()) # ____________________________________________________________ @@ -86,4 +86,4 @@ res.append(chr(0x20 + (C & 0x3F))) res.append('\n') - return space.wrap(res.build()) + return space.newbytes(res.build()) diff --git a/pypy/module/bz2/interp_bz2.py b/pypy/module/bz2/interp_bz2.py --- a/pypy/module/bz2/interp_bz2.py +++ b/pypy/module/bz2/interp_bz2.py @@ -557,7 +557,7 @@ datasize = len(data) if datasize == 0: - return self.space.wrap("") + return self.space.newbytes("") if not self.running: raise oefmt(self.space.w_ValueError, @@ -582,7 +582,7 @@ out.prepare_next_chunk() res = out.make_result_string() - return self.space.wrap(res) + return self.space.newbytes(res) def flush(self): if not self.running: @@ -602,7 +602,7 @@ out.prepare_next_chunk() res = out.make_result_string() - return self.space.wrap(res) + return self.space.newbytes(res) W_BZ2Compressor.typedef = TypeDef("BZ2Compressor", __doc__ = W_BZ2Compressor.__doc__, @@ -669,7 +669,7 @@ raise oefmt(self.space.w_EOFError, "end of stream was already found") if data == '': - return self.space.wrap('') + return self.space.newbytes('') in_bufsize = len(data) @@ -698,7 +698,7 @@ out.prepare_next_chunk() res = out.make_result_string() - return self.space.wrap(res) + return self.space.newbytes(res) W_BZ2Decompressor.typedef = TypeDef("BZ2Decompressor", diff --git a/pypy/module/cpyext/bufferobject.py b/pypy/module/cpyext/bufferobject.py --- a/pypy/module/cpyext/bufferobject.py +++ b/pypy/module/cpyext/bufferobject.py @@ -79,5 +79,5 @@ Py_DecRef(space, py_buf.c_b_base) else: rffi.free_charp(rffi.cast(rffi.CCHARP, py_buf.c_b_ptr)) - from pypy.module.cpyext.object import PyObject_dealloc - PyObject_dealloc(space, py_obj) + from pypy.module.cpyext.object import _dealloc + _dealloc(space, py_obj) diff --git a/pypy/module/cpyext/bytearrayobject.py b/pypy/module/cpyext/bytearrayobject.py --- a/pypy/module/cpyext/bytearrayobject.py +++ b/pypy/module/cpyext/bytearrayobject.py @@ -25,55 +25,11 @@ PyByteArrayObjectStruct = lltype.ForwardReference() PyByteArrayObject = lltype.Ptr(PyByteArrayObjectStruct) -PyByteArrayObjectFields = PyVarObjectFields -# (("ob_exports", rffi.INT), ("ob_alloc", rffi.LONG), ("ob_bytes", rffi.CCHARP)) +PyByteArrayObjectFields = PyVarObjectFields cpython_struct("PyByteArrayObject", PyByteArrayObjectFields, PyByteArrayObjectStruct) - at bootstrap_function -def init_bytearrayobject(space): - "Type description of PyByteArrayObject" - #make_typedescr(space.w_bytearray.layout.typedef, - # basestruct=PyByteArrayObject.TO, - # attach=bytearray_attach, - # dealloc=bytearray_dealloc, - # realize=bytearray_realize) - PyByteArray_Check, PyByteArray_CheckExact = build_type_checkers("ByteArray", "w_bytearray") -# XXX dead code to be removed -#def bytearray_attach(space, py_obj, w_obj): -# """ -# Fills a newly allocated PyByteArrayObject with the given bytearray object -# """ -# py_ba = rffi.cast(PyByteArrayObject, py_obj) -# py_ba.c_ob_size = len(space.str_w(w_obj)) -# py_ba.c_ob_bytes = lltype.nullptr(rffi.CCHARP.TO) -# py_ba.c_ob_exports = rffi.cast(rffi.INT, 0) - -#def bytearray_realize(space, py_obj): -# """ -# Creates the bytearray in the interpreter. -# """ -# py_ba = rffi.cast(PyByteArrayObject, py_obj) -# if not py_ba.c_ob_bytes: -# py_ba.c_buffer = lltype.malloc(rffi.CCHARP.TO, py_ba.c_ob_size + 1, -# flavor='raw', zero=True) -# s = rffi.charpsize2str(py_ba.c_ob_bytes, py_ba.c_ob_size) -# w_obj = space.wrap(s) -# py_ba.c_ob_exports = rffi.cast(rffi.INT, 0) -# track_reference(space, py_obj, w_obj) -# return w_obj - -#@cpython_api([PyObject], lltype.Void, header=None) -#def bytearray_dealloc(space, py_obj): -# """Frees allocated PyByteArrayObject resources. -# """ -# py_ba = rffi.cast(PyByteArrayObject, py_obj) -# if py_ba.c_ob_bytes: -# lltype.free(py_ba.c_ob_bytes, flavor="raw") -# from pypy.module.cpyext.object import PyObject_dealloc -# PyObject_dealloc(space, py_obj) - #_______________________________________________________________________ @cpython_api([PyObject], PyObject, result_is_ll=True) @@ -90,9 +46,9 @@ """Create a new bytearray object from string and its length, len. On failure, NULL is returned.""" if char_p: - w_s = space.wrap(rffi.charpsize2str(char_p, length)) + w_s = space.newbytes(rffi.charpsize2str(char_p, length)) else: - w_s = space.wrap(length) + w_s = space.newint(length) w_buffer = space.call_function(space.w_bytearray, w_s) return make_ref(space, w_buffer) @@ -124,7 +80,7 @@ if space.isinstance_w(w_obj, space.w_bytearray): oldlen = space.len_w(w_obj) if newlen > oldlen: - space.call_method(w_obj, 'extend', space.wrap('\x00' * (newlen - oldlen))) + space.call_method(w_obj, 'extend', space.newbytes('\x00' * (newlen - oldlen))) elif oldlen > newlen: assert newlen >= 0 space.delslice(w_obj, space.wrap(newlen), space.wrap(oldlen)) diff --git a/pypy/module/cpyext/bytesobject.py b/pypy/module/cpyext/bytesobject.py --- a/pypy/module/cpyext/bytesobject.py +++ b/pypy/module/cpyext/bytesobject.py @@ -11,7 +11,7 @@ from pypy.objspace.std.bytesobject import W_BytesObject ## -## Implementation of PyStringObject +## Implementation of PyBytesObject ## ================================ ## ## The problem @@ -29,15 +29,15 @@ ## Solution ## -------- ## -## PyStringObject contains two additional members: the ob_size and a pointer to a +## PyBytesObject contains two additional members: the ob_size and a pointer to a ## char ob_sval; it may be NULL. ## -## - A string allocated by pypy will be converted into a PyStringObject with a +## - A string allocated by pypy will be converted into a PyBytesObject with a ## NULL buffer. The first time PyString_AsString() is called, memory is ## allocated (with flavor='raw') and content is copied. ## ## - A string allocated with PyString_FromStringAndSize(NULL, size) will -## allocate a PyStringObject structure, and a buffer with the specified +## allocate a PyBytesObject structure, and a buffer with the specified ## size+1, but the reference won't be stored in the global map; there is no ## corresponding object in pypy. When from_ref() or Py_INCREF() is called, ## the pypy string is created, and added to the global map of tracked @@ -55,58 +55,58 @@ ## corresponds to the pypy gc-managed string. ## -PyStringObjectStruct = lltype.ForwardReference() -PyStringObject = lltype.Ptr(PyStringObjectStruct) -PyStringObjectFields = PyVarObjectFields + \ +PyBytesObjectStruct = lltype.ForwardReference() +PyBytesObject = lltype.Ptr(PyBytesObjectStruct) +PyBytesObjectFields = PyVarObjectFields + \ (("ob_shash", rffi.LONG), ("ob_sstate", rffi.INT), ("ob_sval", rffi.CArray(lltype.Char))) -cpython_struct("PyStringObject", PyStringObjectFields, PyStringObjectStruct) +cpython_struct("PyStringObject", PyBytesObjectFields, PyBytesObjectStruct) @bootstrap_function -def init_stringobject(space): - "Type description of PyStringObject" +def init_bytesobject(space): + "Type description of PyBytesObject" make_typedescr(space.w_str.layout.typedef, - basestruct=PyStringObject.TO, - attach=string_attach, - dealloc=string_dealloc, - realize=string_realize) + basestruct=PyBytesObject.TO, + attach=bytes_attach, + dealloc=bytes_dealloc, + realize=bytes_realize) PyString_Check, PyString_CheckExact = build_type_checkers("String", "w_str") def new_empty_str(space, length): """ - Allocate a PyStringObject and its ob_sval, but without a corresponding - interpreter object. The ob_sval may be mutated, until string_realize() is + Allocate a PyBytesObject and its ob_sval, but without a corresponding + interpreter object. The ob_sval may be mutated, until bytes_realize() is called. Refcount of the result is 1. """ typedescr = get_typedescr(space.w_str.layout.typedef) py_obj = typedescr.allocate(space, space.w_str, length) - py_str = rffi.cast(PyStringObject, py_obj) + py_str = rffi.cast(PyBytesObject, py_obj) py_str.c_ob_shash = -1 py_str.c_ob_sstate = rffi.cast(rffi.INT, 0) # SSTATE_NOT_INTERNED return py_str -def string_attach(space, py_obj, w_obj): +def bytes_attach(space, py_obj, w_obj): """ - Copy RPython string object contents to a PyStringObject. The + Copy RPython string object contents to a PyBytesObject. The c_ob_sval must not be modified. """ - py_str = rffi.cast(PyStringObject, py_obj) + py_str = rffi.cast(PyBytesObject, py_obj) s = space.str_w(w_obj) if py_str.c_ob_size < len(s): raise oefmt(space.w_ValueError, - "string_attach called on object with ob_size %d but trying to store %d", - py_str.c_ob_size, len(s)) + "bytes_attach called on object with ob_size %d but trying to store %d", + py_str.c_ob_size, len(s)) rffi.c_memcpy(py_str.c_ob_sval, rffi.str2charp(s), len(s)) py_str.c_ob_sval[len(s)] = '\0' py_str.c_ob_shash = space.hash_w(w_obj) py_str.c_ob_sstate = rffi.cast(rffi.INT, 1) # SSTATE_INTERNED_MORTAL -def string_realize(space, py_obj): +def bytes_realize(space, py_obj): """ - Creates the string in the interpreter. The PyStringObject ob_sval must not + Creates the string in the interpreter. The PyBytesObject ob_sval must not be modified after this call. """ - py_str = rffi.cast(PyStringObject, py_obj) + py_str = rffi.cast(PyBytesObject, py_obj) s = rffi.charpsize2str(py_str.c_ob_sval, py_str.c_ob_size) w_type = from_ref(space, rffi.cast(PyObject, py_obj.c_ob_type)) w_obj = space.allocate_instance(W_BytesObject, w_type) @@ -117,11 +117,11 @@ return w_obj @cpython_api([PyObject], lltype.Void, header=None) -def string_dealloc(space, py_obj): - """Frees allocated PyStringObject resources. +def bytes_dealloc(space, py_obj): + """Frees allocated PyBytesObject resources. """ - from pypy.module.cpyext.object import PyObject_dealloc - PyObject_dealloc(space, py_obj) + from pypy.module.cpyext.object import _dealloc + _dealloc(space, py_obj) #_______________________________________________________________________ @@ -154,10 +154,10 @@ raise oefmt(space.w_TypeError, "expected string or Unicode object, %T found", from_ref(space, ref)) - ref_str = rffi.cast(PyStringObject, ref) + ref_str = rffi.cast(PyBytesObject, ref) if not pyobj_has_w_obj(ref): # XXX Force the ref? - string_realize(space, ref) + bytes_realize(space, ref) return ref_str.c_ob_sval @cpython_api([rffi.VOIDP], rffi.CCHARP, error=0) @@ -166,7 +166,7 @@ # if no w_str is associated with this ref, # return the c-level ptr as RW if not pyobj_has_w_obj(ref): - py_str = rffi.cast(PyStringObject, ref) + py_str = rffi.cast(PyBytesObject, ref) return py_str.c_ob_sval return _PyString_AsString(space, ref) @@ -183,8 +183,8 @@ from_ref(space, ref)) if not pyobj_has_w_obj(ref): # force the ref - string_realize(space, ref) - ref_str = rffi.cast(PyStringObject, ref) + bytes_realize(space, ref) + ref_str = rffi.cast(PyBytesObject, ref) data[0] = ref_str.c_ob_sval if length: length[0] = ref_str.c_ob_size @@ -200,7 +200,7 @@ @cpython_api([PyObject], Py_ssize_t, error=-1) def PyString_Size(space, ref): if from_ref(space, rffi.cast(PyObject, ref.c_ob_type)) is space.w_str: - ref = rffi.cast(PyStringObject, ref) + ref = rffi.cast(PyBytesObject, ref) return ref.c_ob_size else: w_obj = from_ref(space, ref) @@ -222,7 +222,7 @@ if pyobj_has_w_obj(ref[0]): raise oefmt(space.w_SystemError, "_PyString_Resize called on already created string") - py_str = rffi.cast(PyStringObject, ref[0]) + py_str = rffi.cast(PyBytesObject, ref[0]) try: py_newstr = new_empty_str(space, newsize) except MemoryError: diff --git a/pypy/module/cpyext/eval.py b/pypy/module/cpyext/eval.py --- a/pypy/module/cpyext/eval.py +++ b/pypy/module/cpyext/eval.py @@ -13,7 +13,7 @@ "PyCompilerFlags", (("cf_flags", rffi.INT),)) PyCompilerFlagsPtr = lltype.Ptr(PyCompilerFlags) -PyCF_MASK = (consts.CO_FUTURE_DIVISION | +PyCF_MASK = (consts.CO_FUTURE_DIVISION | consts.CO_FUTURE_ABSOLUTE_IMPORT | consts.CO_FUTURE_WITH_STATEMENT | consts.CO_FUTURE_PRINT_FUNCTION | @@ -94,7 +94,7 @@ Py_eval_input = 258 def compile_string(space, source, filename, start, flags=0): - w_source = space.wrap(source) + w_source = space.newbytes(source) start = rffi.cast(lltype.Signed, start) if start == Py_file_input: mode = 'exec' @@ -227,4 +227,4 @@ cf.c_cf_flags = rffi.cast(rffi.INT, flags) return result - + diff --git a/pypy/module/cpyext/frameobject.py b/pypy/module/cpyext/frameobject.py --- a/pypy/module/cpyext/frameobject.py +++ b/pypy/module/cpyext/frameobject.py @@ -46,8 +46,8 @@ Py_DecRef(space, py_code) Py_DecRef(space, py_frame.c_f_globals) Py_DecRef(space, py_frame.c_f_locals) - from pypy.module.cpyext.object import PyObject_dealloc - PyObject_dealloc(space, py_obj) + from pypy.module.cpyext.object import _dealloc + _dealloc(space, py_obj) def frame_realize(space, py_obj): """ diff --git a/pypy/module/cpyext/funcobject.py b/pypy/module/cpyext/funcobject.py --- a/pypy/module/cpyext/funcobject.py +++ b/pypy/module/cpyext/funcobject.py @@ -60,8 +60,8 @@ def function_dealloc(space, py_obj): py_func = rffi.cast(PyFunctionObject, py_obj) Py_DecRef(space, py_func.c_func_name) - from pypy.module.cpyext.object import PyObject_dealloc - PyObject_dealloc(space, py_obj) + from pypy.module.cpyext.object import _dealloc + _dealloc(space, py_obj) def code_attach(space, py_obj, w_obj): py_code = rffi.cast(PyCodeObject, py_obj) @@ -80,8 +80,8 @@ py_code = rffi.cast(PyCodeObject, py_obj) Py_DecRef(space, py_code.c_co_name) Py_DecRef(space, py_code.c_co_filename) - from pypy.module.cpyext.object import PyObject_dealloc - PyObject_dealloc(space, py_obj) + from pypy.module.cpyext.object import _dealloc + _dealloc(space, py_obj) @cpython_api([PyObject], PyObject, result_borrowed=True) def PyFunction_GetCode(space, w_func): diff --git a/pypy/module/cpyext/include/_numpypy/numpy/__multiarray_api.h b/pypy/module/cpyext/include/_numpypy/numpy/__multiarray_api.h --- a/pypy/module/cpyext/include/_numpypy/numpy/__multiarray_api.h +++ b/pypy/module/cpyext/include/_numpypy/numpy/__multiarray_api.h @@ -5,7 +5,12 @@ npy_bool obval; } PyBoolScalarObject; -static int import_array(){return 0;}; -static int _import_array(){return 0;}; -static int _import_math(){return 0;}; +#if PY_VERSION_HEX >= 0x03000000 +#define NUMPY_IMPORT_ARRAY_RETVAL NULL +#else +#define NUMPY_IMPORT_ARRAY_RETVAL +#endif +#define import_array() {return NUMPY_IMPORT_ARRAY_RETVAL;} + + diff --git a/pypy/module/cpyext/include/tupleobject.h b/pypy/module/cpyext/include/tupleobject.h --- a/pypy/module/cpyext/include/tupleobject.h +++ b/pypy/module/cpyext/include/tupleobject.h @@ -8,9 +8,12 @@ #endif typedef struct { - PyObject_HEAD - Py_ssize_t ob_size; - PyObject **ob_item; /* XXX optimize to ob_item[] */ + PyObject_VAR_HEAD + PyObject *ob_item[1]; + /* ob_item contains space for 'ob_size' elements. + * Items must normally not be NULL, except during construction when + * the tuple is not yet visible outside the function that builds it. + */ } PyTupleObject; /* defined in varargswrapper.c */ diff --git a/pypy/module/cpyext/methodobject.py b/pypy/module/cpyext/methodobject.py --- a/pypy/module/cpyext/methodobject.py +++ b/pypy/module/cpyext/methodobject.py @@ -55,8 +55,8 @@ py_func = rffi.cast(PyCFunctionObject, py_obj) Py_DecRef(space, py_func.c_m_self) Py_DecRef(space, py_func.c_m_module) - from pypy.module.cpyext.object import PyObject_dealloc - PyObject_dealloc(space, py_obj) + from pypy.module.cpyext.object import _dealloc + _dealloc(space, py_obj) class W_PyCFunctionObject(W_Root): diff --git a/pypy/module/cpyext/object.py b/pypy/module/cpyext/object.py --- a/pypy/module/cpyext/object.py +++ b/pypy/module/cpyext/object.py @@ -54,6 +54,9 @@ @cpython_api([PyObject], lltype.Void) def PyObject_dealloc(space, obj): + return _dealloc(space, obj) + +def _dealloc(space, obj): # This frees an object after its refcount dropped to zero, so we # assert that it is really zero here. assert obj.c_ob_refcnt == 0 diff --git a/pypy/module/cpyext/pyfile.py b/pypy/module/cpyext/pyfile.py --- a/pypy/module/cpyext/pyfile.py +++ b/pypy/module/cpyext/pyfile.py @@ -23,7 +23,7 @@ try: w_readline = space.getattr(w_obj, space.wrap('readline')) except OperationError: - raise oefmt(space.w_TypeError, + raise oefmt(space.w_TypeError, "argument must be a file, or have a readline() method.") n = rffi.cast(lltype.Signed, n) @@ -41,7 +41,7 @@ On success, return a new file object that is opened on the file given by filename, with a file mode given by mode, where mode has the same semantics as the standard C routine fopen(). On failure, return NULL.""" - w_filename = space.wrap(rffi.charp2str(filename)) + w_filename = space.newbytes(rffi.charp2str(filename)) w_mode = space.wrap(rffi.charp2str(mode)) return space.call_method(space.builtin, 'file', w_filename, w_mode) diff --git a/pypy/module/cpyext/pyobject.py b/pypy/module/cpyext/pyobject.py --- a/pypy/module/cpyext/pyobject.py +++ b/pypy/module/cpyext/pyobject.py @@ -164,7 +164,7 @@ pytype = rffi.cast(PyTypeObjectPtr, as_pyobj(space, w_type)) typedescr = get_typedescr(w_obj.typedef) if pytype.c_tp_itemsize != 0: - itemcount = space.len_w(w_obj) # PyStringObject and subclasses + itemcount = space.len_w(w_obj) # PyBytesObject and subclasses else: itemcount = 0 py_obj = typedescr.allocate(space, w_type, itemcount=itemcount) diff --git a/pypy/module/cpyext/pytraceback.py b/pypy/module/cpyext/pytraceback.py --- a/pypy/module/cpyext/pytraceback.py +++ b/pypy/module/cpyext/pytraceback.py @@ -46,5 +46,5 @@ py_traceback = rffi.cast(PyTracebackObject, py_obj) Py_DecRef(space, rffi.cast(PyObject, py_traceback.c_tb_next)) Py_DecRef(space, rffi.cast(PyObject, py_traceback.c_tb_frame)) - from pypy.module.cpyext.object import PyObject_dealloc - PyObject_dealloc(space, py_obj) + from pypy.module.cpyext.object import _dealloc + _dealloc(space, py_obj) diff --git a/pypy/module/cpyext/sliceobject.py b/pypy/module/cpyext/sliceobject.py --- a/pypy/module/cpyext/sliceobject.py +++ b/pypy/module/cpyext/sliceobject.py @@ -38,14 +38,14 @@ @cpython_api([PyObject], lltype.Void, header=None) def slice_dealloc(space, py_obj): - """Frees allocated PyStringObject resources. + """Frees allocated PyBytesObject resources. """ py_slice = rffi.cast(PySliceObject, py_obj) Py_DecRef(space, py_slice.c_start) Py_DecRef(space, py_slice.c_stop) Py_DecRef(space, py_slice.c_step) - from pypy.module.cpyext.object import PyObject_dealloc - PyObject_dealloc(space, py_obj) + from pypy.module.cpyext.object import _dealloc + _dealloc(space, py_obj) PySlice_Check, PySlice_CheckExact = build_type_checkers("Slice") @@ -73,7 +73,7 @@ length length, and store the length of the slice in slicelength. Out of bounds indices are clipped in a manner consistent with the handling of normal slices. - + Returns 0 on success and -1 on error with exception set.""" if not PySlice_Check(space, w_slice): PyErr_BadInternalCall(space) @@ -88,11 +88,11 @@ """Retrieve the start, stop and step indices from the slice object slice, assuming a sequence of length length. Treats indices greater than length as errors. - + Returns 0 on success and -1 on error with no exception set (unless one of the indices was not None and failed to be converted to an integer, in which case -1 is returned with an exception set). - + You probably do not want to use this function. If you want to use slice objects in versions of Python prior to 2.3, you would probably do well to incorporate the source of PySlice_GetIndicesEx(), suitably renamed, diff --git a/pypy/module/cpyext/test/array.c b/pypy/module/cpyext/test/array.c --- a/pypy/module/cpyext/test/array.c +++ b/pypy/module/cpyext/test/array.c @@ -1858,6 +1858,110 @@ } } +static PyObject* +array_multiply(PyObject* obj1, PyObject* obj2) +{ + if (PyList_Check(obj1) && ((arrayobject*)obj2)->ob_descr->typecode == 'i' && Py_SIZE(obj2) == 1) + { + int ii, nn; + int n = PyList_Size(obj1); + PyObject *v = getarrayitem(obj2, 0); + int i = ((PyIntObject*)v)->ob_ival; + PyObject * ret = PyList_New(n*i); + for (ii = 0; ii < i; ii++) + for (nn = 0; nn < n; nn++) + { + v = PyList_GetItem(obj1, nn); + PyList_SetItem(ret, nn+ii*n, v); + } + return ret; + } + else if (PyList_Check(obj2) && ((arrayobject*)obj1)->ob_descr->typecode == 'i' && Py_SIZE(obj1) == 1) + { + int ii, nn; + int n = PyList_Size(obj2); + PyObject *v = getarrayitem(obj1, 0); + int i = ((PyIntObject*)v)->ob_ival; + PyObject * ret = PyList_New(n*i); + for (ii = 0; ii < i; ii++) + for (nn = 0; nn < n; nn++) + { + v = PyList_GetItem(obj2, nn); + PyList_SetItem(ret, nn+ii*n, v); + } + return ret; + } + else if(obj1->ob_type == &Arraytype) + fprintf(stderr, "\nCannot multiply array of type %c and %s\n", + ((arrayobject*)obj1)->ob_descr->typecode, obj2->ob_type->tp_name); + else if(obj2->ob_type == &Arraytype) + fprintf(stderr, "\nCannot multiply array of type %c and %s\n", + ((arrayobject*)obj2)->ob_descr->typecode, obj1->ob_type->tp_name); + Py_INCREF(Py_NotImplemented); + return Py_NotImplemented; +} + +static PyNumberMethods array_as_number = { + (binaryfunc)NULL, /* nb_add*/ + (binaryfunc)NULL, /* nb_subtract */ + (binaryfunc)array_multiply, /* nb_multiply */ + (binaryfunc)NULL, /* nb_divide */ +}; + +static PyObject* +array_base_multiply(PyObject* obj1, PyObject* obj2) +{ + if (PyList_Check(obj1) && ((arrayobject*)obj2)->ob_descr->typecode == 'i' && Py_SIZE(obj2) == 1) + { + int nn; + int n = PyList_Size(obj1); + PyObject *v = getarrayitem(obj2, 0); + int i = ((PyIntObject*)v)->ob_ival; + PyObject * ret = PyList_New(n); + for (nn = 0; nn < n; nn++) + { + v = PyList_GetItem(obj1, nn); + if (PyInt_Check(v)) + PyList_SetItem(ret, nn, PyLong_FromLong(i * ((PyIntObject*)v)->ob_ival)); + else + PyList_SetItem(ret, nn, v); + } + return ret; + } + else if (PyList_Check(obj2) && ((arrayobject*)obj1)->ob_descr->typecode == 'i' && Py_SIZE(obj1) == 1) + { + int nn; + int n = PyList_Size(obj2); + PyObject *v = getarrayitem(obj1, 0); + int i = ((PyIntObject*)v)->ob_ival; + PyObject * ret = PyList_New(n); + for (nn = 0; nn < n; nn++) + { + v = PyList_GetItem(obj2, nn); + if (PyInt_Check(v)) + PyList_SetItem(ret, nn, PyLong_FromLong(i * ((PyIntObject*)v)->ob_ival)); + else + PyList_SetItem(ret, nn, v); + } + return ret; + } + else if(obj1->ob_type == &Arraytype) + fprintf(stderr, "\nCannot multiply array of type %c and %s\n", + ((arrayobject*)obj1)->ob_descr->typecode, obj2->ob_type->tp_name); + else if(obj2->ob_type == &Arraytype) + fprintf(stderr, "\nCannot multiply array of type %c and %s\n", + ((arrayobject*)obj2)->ob_descr->typecode, obj1->ob_type->tp_name); + Py_INCREF(Py_NotImplemented); + return Py_NotImplemented; +} + +static PyNumberMethods array_base_as_number = { + (binaryfunc)NULL, /* nb_add*/ + (binaryfunc)NULL, /* nb_subtract */ + (binaryfunc)array_base_multiply, /* nb_multiply */ + (binaryfunc)NULL, /* nb_divide */ +}; + static PyMappingMethods array_as_mapping = { (lenfunc)array_length, (binaryfunc)array_subscr, @@ -2106,6 +2210,49 @@ static PyObject *array_iter(arrayobject *ao); +static PyTypeObject ArrayBasetype = { + PyVarObject_HEAD_INIT(NULL, 0) + "array.basearray", + sizeof(arrayobject), + 0, + (destructor)array_dealloc, /* tp_dealloc */ + 0, /* tp_print */ + 0, /* tp_getattr */ + 0, /* tp_setattr */ + 0, /* tp_compare */ + (reprfunc)array_repr, /* tp_repr */ From pypy.commits at gmail.com Wed Jul 6 04:31:32 2016 From: pypy.commits at gmail.com (plan_rich) Date: Wed, 06 Jul 2016 01:31:32 -0700 (PDT) Subject: [pypy-commit] pypy ppc-vsx-support: removes mixin class and add functions directly to BasicPPCAssembler Message-ID: <577cc1e4.571a1c0a.7d7b4.4441@mx.google.com> Author: Richard Plangger Branch: ppc-vsx-support Changeset: r85571:7da69acc5ef4 Date: 2016-07-06 10:30 +0200 http://bitbucket.org/pypy/pypy/changeset/7da69acc5ef4/ Log: removes mixin class and add functions directly to BasicPPCAssembler diff --git a/rpython/jit/backend/ppc/codebuilder.py b/rpython/jit/backend/ppc/codebuilder.py --- a/rpython/jit/backend/ppc/codebuilder.py +++ b/rpython/jit/backend/ppc/codebuilder.py @@ -580,10 +580,7 @@ xor = XS(31, XO1=316, Rc=0) xorx = XS(31, XO1=316, Rc=1) - # - -class PPCVSXAssembler(object): - _mixin_ = True + # Vector Ext # floating point operations (ppc got it's own vector # unit for double/single precision floating points @@ -722,13 +719,7 @@ vsplth = VX_splat(4, XO8=588) vspltw = VX_splat(4, XO8=652) - - - - - - -class PPCAssembler(BasicPPCAssembler, PPCVSXAssembler): +class PPCAssembler(BasicPPCAssembler): BA = BasicPPCAssembler # awkward mnemonics: From pypy.commits at gmail.com Wed Jul 6 05:25:02 2016 From: pypy.commits at gmail.com (plan_rich) Date: Wed, 06 Jul 2016 02:25:02 -0700 (PDT) Subject: [pypy-commit] pypy default: apply further changes to correctly push/pop gcmap (started in a09c99e17a27) Message-ID: <577cce6e.4d9a1c0a.57aff.002d@mx.google.com> Author: Richard Plangger Branch: Changeset: r85572:35001d8b5219 Date: 2016-07-06 11:19 +0200 http://bitbucket.org/pypy/pypy/changeset/35001d8b5219/ Log: apply further changes to correctly push/pop gcmap (started in a09c99e17a27) diff --git a/rpython/jit/backend/zarch/assembler.py b/rpython/jit/backend/zarch/assembler.py --- a/rpython/jit/backend/zarch/assembler.py +++ b/rpython/jit/backend/zarch/assembler.py @@ -317,7 +317,7 @@ self._push_fp_regs_to_jitframe(mc) - # First argument is SPP (= r31), which is the jitframe + # First argument is SPP, which is the jitframe mc.LGR(r.r2, r.SPP) # no need to move second argument (frame_depth), @@ -385,14 +385,12 @@ # signature of these cond_call_slowpath functions: # * on entry, r11 contains the function to call # * r2, r3, r4, r5 contain arguments for the call - # * r0 is the gcmap + # * gcmap is pushed # * the old value of these regs must already be stored in the jitframe # * on exit, all registers are restored from the jitframe mc = InstrBuilder() self.mc = mc - ofs2 = self.cpu.get_ofs_of_frame_field('jf_gcmap') - mc.STG(r.SCRATCH2, l.addr(ofs2,r.SPP)) mc.store_link() mc.push_std_frame() diff --git a/rpython/jit/backend/zarch/opassembler.py b/rpython/jit/backend/zarch/opassembler.py --- a/rpython/jit/backend/zarch/opassembler.py +++ b/rpython/jit/backend/zarch/opassembler.py @@ -387,8 +387,7 @@ if reg in self._COND_CALL_SAVE_REGS] self._push_core_regs_to_jitframe(self.mc, should_be_saved) - # load gc map into unusual location: r0 - self.load_gcmap(self.mc, r.SCRATCH2, regalloc.get_gcmap()) + self.push_gcmap(self.mc, regalloc.get_gcmap()) # # load the 0-to-4 arguments into these registers, with the address of # the function to call into r11 From pypy.commits at gmail.com Wed Jul 6 05:25:04 2016 From: pypy.commits at gmail.com (plan_rich) Date: Wed, 06 Jul 2016 02:25:04 -0700 (PDT) Subject: [pypy-commit] pypy ppc-vsx-support: invoked missing function (typo) Message-ID: <577cce70.4aa6c20a.cb609.ffffa1d8@mx.google.com> Author: Richard Plangger Branch: ppc-vsx-support Changeset: r85573:2f5745203fcb Date: 2016-07-06 11:24 +0200 http://bitbucket.org/pypy/pypy/changeset/2f5745203fcb/ Log: invoked missing function (typo) diff --git a/rpython/jit/backend/ppc/vector_ext.py b/rpython/jit/backend/ppc/vector_ext.py --- a/rpython/jit/backend/ppc/vector_ext.py +++ b/rpython/jit/backend/ppc/vector_ext.py @@ -253,7 +253,7 @@ elif size == 8: self.mc.xvabsdp(resloc.value, argloc.value) else: - notimplemented("[ppc/assembler] float abs for size %d" % size) + not_implemented("float abs for size %d" % size) def emit_vec_float_neg(self, op, arglocs, regalloc): resloc, argloc, sizeloc = arglocs @@ -263,7 +263,7 @@ elif size == 8: self.mc.xvnegdp(resloc.value, argloc.value) else: - notimplemented("[ppc/assembler] float neg for size %d" % size) + not_implemented("float neg for size %d" % size) def emit_vec_guard_true(self, guard_op, arglocs, regalloc): self._emit_guard(guard_op, arglocs) @@ -401,7 +401,7 @@ self.mc.xvcmpeqdpx(tmp, loc1.value, loc2.value) self.mc.stxvd2x(tmp, off, r.SP.value) else: - notimplemented("[ppc/assembler] float == for size %d" % size) + not_implemented("[ppc/assembler] float == for size %d" % size) self.mc.lvx(resloc.value, off, r.SP.value) flush_vec_cc(self, regalloc, c.VEQI, op.bytesize, resloc) @@ -427,7 +427,7 @@ self.mc.xvcmpeqdpx(tmp, loc1.value, loc2.value) self.mc.stxvd2x(tmp, off, r.SP.value) else: - notimplemented("[ppc/assembler] float == for size %d" % size) + not_implemented("float == for size %d" % size) res = resloc.value self.mc.lvx(res, off, r.SP.value) self.mc.vnor(res, res, res) # complement @@ -487,7 +487,7 @@ src = srcloc.value self.mc.xxspltdl(res, src, src) else: - notimplemented("[ppc/assembler] vec expand in this combination not supported") + not_implemented("vec expand in this combination not supported") def emit_vec_expand_i(self, op, arglocs, regalloc): res, l0, off = arglocs @@ -513,7 +513,7 @@ elif size == 8: pass else: - notimplemented("[expand int size not impl]") + not_implemented("expand int size not impl") def emit_vec_pack_i(self, op, arglocs, regalloc): resultloc, vloc, sourceloc, residxloc, srcidxloc, countloc = arglocs @@ -531,13 +531,13 @@ self.mc.store(src, r.SP.value, PARAM_SAVE_AREA_OFFSET+8*residx) self.mc.lvx(res, r.SCRATCH2.value, r.SP.value) else: - notimplemented("[ppc/vec_pack_i] 64 bit float") + not_implemented("64 bit float") elif size == 4: - notimplemented("[ppc/vec_pack_i]") + not_implemented("vec_pack_i") elif size == 2: - notimplemented("[ppc/vec_pack_i]") + not_implemented("vec_pack_i") elif size == 1: - notimplemented("[ppc/vec_pack_i]") + not_implemented("vec_pack_i") def emit_vec_unpack_i(self, op, arglocs, regalloc): resloc, srcloc, idxloc, countloc = arglocs @@ -564,7 +564,7 @@ self.mc.extsb(res, res) return - notimplemented("[ppc/vec_unpack_i] %d bit integer, count %d" % \ + not_implemented("%d bit integer, count %d" % \ (size*8, count)) def emit_vec_pack_f(self, op, arglocs, regalloc): From pypy.commits at gmail.com Wed Jul 6 05:49:14 2016 From: pypy.commits at gmail.com (arigo) Date: Wed, 06 Jul 2016 02:49:14 -0700 (PDT) Subject: [pypy-commit] extradoc extradoc: in-progress: reverse debugger Message-ID: <577cd41a.4aa71c0a.38505.ffffbd8c@mx.google.com> Author: Armin Rigo Branch: extradoc Changeset: r5645:09dd2764135d Date: 2016-07-06 11:50 +0200 http://bitbucket.org/pypy/extradoc/changeset/09dd2764135d/ Log: in-progress: reverse debugger diff --git a/blog/draft/revdb.rst b/blog/draft/revdb.rst new file mode 100644 --- /dev/null +++ b/blog/draft/revdb.rst @@ -0,0 +1,143 @@ +Hi all, + +If I had to pick the main advantage of PyPy over CPython, it is that +we have got with the RPython translation toolchain a real place for +experimentation. Every now and then, we build inside RPython some +feature that gives us an optionally tweaked version of the PyPy +interpreter---tweaked in a way that would be hard to do with CPython, +because it would require systematic changes everywhere. The most +obvious and successful examples are the GC and the JIT. But there +have been many other experiments along the same lines, from the +so-called "stackless transformation" in the early days, to the STM +version of PyPy. + +Today I would like to present you with last month's work (still very +much in alpha state). It is a RPython transformation that gives +support for a *reverse debugger* in PyPy or in any other interpreter +written in RPython. + + +Reverse debugging +----------------- + +What is `reverse debugging`__? It is a debugger where you can go +forward and backward in time. It is still a not commonly used +feature, and I have no idea why not. I have used UndoDB's reverse +debugger for C code, and I can only say that it saved me many, many +days of poking around blindly in gdb. + +.. __: https://en.wikipedia.org/wiki/Debugger#Reverse_debugging + +There are already some Python experiments about reverse debugging. +However, I claim that they are not very useful. How they work is +typically by recording changes to some objects, like lists and +dictionaries, in addition to recording the history of where your +program passed through. However, the problem of Python is, again, +that lists and dictionaries are not the end of the story. There are +many, many, many types of objects written in C which are mutable---in +fact, the immutable ones are the exception. You can try to +systematically record all changes, but it is a huge task and easy to +forget a detail. + +In other words it is a typical use case for tweaking the RPython +translation toolchain rather than the CPython or PyPy interpreter +directly. + + +RevDB in PyPy +------------- + +Right now, RevDB works barely enough to start being useful. I have +used it to track one real bug (for the interested people, see +bd220c268bc9). So here is what it is, what it is not, and how to use +it. + +RevDB is a Python debugger. It will not help track issues like +segfaults or crashes of the interpreter, but it will help track any +Python-level bugs. Think about bugs that end up as a Python traceback +or another wrong answer, but where the problem is really caused by +something earlier going wrong in your Python logic. + +RevDB is a logging system, similar to http://rr-project.org/ . You +first run your Python program by using a special version of PyPy. It +creates a log file which records the I/O actions. Sometimes you are +tracking a rare bug: you may need to run your program many times until +it shows the crash. That should still be reasonable: the special +version of PyPy is very slow (it does not contain any JIT nor one of +our high-performance GCs), but still not incredibly so---it is a few +times slower than running the same program on CPython. The point is +also that normally, what you need is just one recorded run of the +program showing the bug. You may struggle a bit to get that, but once +you have it, this part is done. + +Then you use the debugger on the log file. The debugger will also +internally re-run the special version of PyPy in a different mode. +This feels like a debugger, though it is really a program that +inspects any point of the history. Like in a normal pdb, you can use +commands like "next" and "p foo.bar" and even run more complicated +bits of Python code. You also have new commands like "bnext" to go +backwards. Most importantly, you can set *watchpoints*. More about +that later. + +What you cannot do is do any input/output from the debugger. Indeed, +the log file records all imports that were done and what the imported +modules contained. Running the debugger on the log file gives an +exact replay of what was recorded. + + + + + + + + + + +- no thread module for now. And, no cpyext module for now (the + CPython C API compatibility layer), because it depends on threads. + No micronumpy either. + These missing modules are probably blockers for large programs. + +- does not contain a JIT, and does not use our fast garbage collector. + +- for now, the process needs to get the same addresses (of C functions + and static data) when recording and when replaying. On the Linux I + tried it with, you get this result by disabling Address Space Layout + Randomization (ASLR):: + + echo 0 | sudo tee /proc/sys/kernel/randomize_va_space + +- OS/X and other Posix platforms are probably just a few fixes away. + Windows support will require some custom logic to replace the + forking done when replaying. This is more involved but should still + be possible. + +- maybe 15x memory usage on replaying (adjust number of forks in + process.py, MAX_SUBPROCESSES). + +- replaying issues: + + - Attempted to do I/O or access raw memory: we get this often, and + then we need "bstep+step" before we can print anything else + + - id() is globally unique, returning a reproducible 64-bit number, + so sometimes using id(x) is a workaround for when using x doesn't + work because of "Attempt to do I/O" issues (e.g. + ``p [id(x) for x in somelist]``) + + - next/bnext/finish/bfinish might jump around a bit non-predictably. + + - similarly, breaks on watchpoints can stop at apparently unexpected + places (when going backward, try to do "step" once). The issue is + that it can only stop at the beginning of every line. In the + extreme example, if a line is ``foo(somelist.pop(getindex()))``, + then ``somelist`` is modified in the middle. Immediately before + this modification occurs, we are in ``getindex()``, and + immediately afterwards we are in ``foo()``. The watchpoint will + stop the program at the end of ``getindex()`` if running backward, + and at the start of ``foo()`` if running forward, but never + actually on the line doing the change. + + - the first time you use $NUM to refer to an object, if it was + created long ago, then replaying might need to replay again from + that long-ago time From pypy.commits at gmail.com Wed Jul 6 05:49:46 2016 From: pypy.commits at gmail.com (arigo) Date: Wed, 06 Jul 2016 02:49:46 -0700 (PDT) Subject: [pypy-commit] pypy default: This should constant-fold ffi.offsetof("structname", "fieldname") Message-ID: <577cd43a.56311c0a.4117b.ffffddd5@mx.google.com> Author: Armin Rigo Branch: Changeset: r85574:11d652c7d64e Date: 2016-07-06 11:51 +0200 http://bitbucket.org/pypy/pypy/changeset/11d652c7d64e/ Log: This should constant-fold ffi.offsetof("structname", "fieldname") diff --git a/pypy/module/_cffi_backend/ctypestruct.py b/pypy/module/_cffi_backend/ctypestruct.py --- a/pypy/module/_cffi_backend/ctypestruct.py +++ b/pypy/module/_cffi_backend/ctypestruct.py @@ -90,7 +90,7 @@ self.force_lazy_struct() space = self.space try: - cfield = self._fields_dict[fieldname] + cfield = self._getcfield_const(fieldname) except KeyError: raise OperationError(space.w_KeyError, space.wrap(fieldname)) if cfield.bitshift >= 0: From pypy.commits at gmail.com Wed Jul 6 06:33:33 2016 From: pypy.commits at gmail.com (plan_rich) Date: Wed, 06 Jul 2016 03:33:33 -0700 (PDT) Subject: [pypy-commit] pypy default: remove pop_gcmap just before returning from malloc_slowpath, this will overwrite the nursery free ptr hold in r1 (s390x) Message-ID: <577cde7d.a459c20a.4dee5.ffffbf02@mx.google.com> Author: Richard Plangger Branch: Changeset: r85575:e472905e4fec Date: 2016-07-06 12:29 +0200 http://bitbucket.org/pypy/pypy/changeset/e472905e4fec/ Log: remove pop_gcmap just before returning from malloc_slowpath, this will overwrite the nursery free ptr hold in r1 (s390x) diff --git a/rpython/jit/backend/zarch/assembler.py b/rpython/jit/backend/zarch/assembler.py --- a/rpython/jit/backend/zarch/assembler.py +++ b/rpython/jit/backend/zarch/assembler.py @@ -500,7 +500,6 @@ # r.RSZ is loaded from [r1], to make the caller's store a no-op here mc.load(r.RSZ, r.r1, 0) # - self.pop_gcmap(mc) # push_gcmap(store=True) done by the caller mc.restore_link() mc.BCR(c.ANY, r.r14) self.mc = None From pypy.commits at gmail.com Wed Jul 6 09:33:55 2016 From: pypy.commits at gmail.com (cfbolz) Date: Wed, 06 Jul 2016 06:33:55 -0700 (PDT) Subject: [pypy-commit] extradoc extradoc: - linkify the commit Message-ID: <577d08c3.06a81c0a.47527.54c6@mx.google.com> Author: Carl Friedrich Bolz Branch: extradoc Changeset: r5646:0b3db1b1698e Date: 2016-07-06 15:33 +0200 http://bitbucket.org/pypy/extradoc/changeset/0b3db1b1698e/ Log: - linkify the commit - I didn't understand something diff --git a/blog/draft/revdb.rst b/blog/draft/revdb.rst --- a/blog/draft/revdb.rst +++ b/blog/draft/revdb.rst @@ -49,9 +49,11 @@ Right now, RevDB works barely enough to start being useful. I have used it to track one real bug (for the interested people, see -bd220c268bc9). So here is what it is, what it is not, and how to use +bd220c268bc9_). So here is what it is, what it is not, and how to use it. +.. _bd220c268bc9: https://bitbucket.org/pypy/pypy/commits/bd220c268bc9 + RevDB is a Python debugger. It will not help track issues like segfaults or crashes of the interpreter, but it will help track any Python-level bugs. Think about bugs that end up as a Python traceback @@ -79,6 +81,9 @@ backwards. Most importantly, you can set *watchpoints*. More about that later. + +XXX CF: it's not clear to me what "doing any input/output from the debugger" means + What you cannot do is do any input/output from the debugger. Indeed, the log file records all imports that were done and what the imported modules contained. Running the debugger on the log file gives an From pypy.commits at gmail.com Wed Jul 6 11:22:46 2016 From: pypy.commits at gmail.com (raffael_t) Date: Wed, 06 Jul 2016 08:22:46 -0700 (PDT) Subject: [pypy-commit] pypy py3.5-async: Include is_async check in second handle_with_stmt function, return async and await tokens in tokenizer Message-ID: <577d2246.c255c20a.cc0ee.1244@mx.google.com> Author: Raffael Tfirst Branch: py3.5-async Changeset: r85576:9a227fcddc2e Date: 2016-07-06 17:22 +0200 http://bitbucket.org/pypy/pypy/changeset/9a227fcddc2e/ Log: Include is_async check in second handle_with_stmt function, return async and await tokens in tokenizer diff --git a/pypy/interpreter/astcompiler/astbuilder.py b/pypy/interpreter/astcompiler/astbuilder.py --- a/pypy/interpreter/astcompiler/astbuilder.py +++ b/pypy/interpreter/astcompiler/astbuilder.py @@ -447,11 +447,16 @@ target = None return ast.withitem(test, target) - def handle_with_stmt(self, with_node): + def handle_with_stmt(self, with_node, is_async): body = self.handle_suite(with_node.get_child(-1)) items = [self.handle_with_item(with_node.get_child(i)) for i in range(1, with_node.num_children()-2, 2)] - return ast.With(items, body, with_node.get_lineno(), with_node.get_column()) + if is_async: + return ast.AsyncWith(items, body, with_node.get_lineno(), + with_node.get_column()) + else: + return ast.With(items, body, with_node.get_lineno(), + with_node.get_column()) def handle_classdef(self, classdef_node, decorators=None): name_node = classdef_node.get_child(1) @@ -502,10 +507,10 @@ funcdef_node.get_lineno(), funcdef_node.get_column()) def handle_async_funcdef(self, node, decorators=None): - return handle_funcdef_impl(node.get_child(1), 1, decorators) + return self.handle_funcdef_impl(node.get_child(1), 1, decorators) def handle_funcdef(self, node, decorators=None): - return handle_funcdef_impl(node, 0, decorators) + return self.handle_funcdef_impl(node, 0, decorators) def handle_async_stmt(self, node): ch = node.get_child(1) diff --git a/pypy/interpreter/pyparser/pytokenizer.py b/pypy/interpreter/pyparser/pytokenizer.py --- a/pypy/interpreter/pyparser/pytokenizer.py +++ b/pypy/interpreter/pyparser/pytokenizer.py @@ -253,7 +253,12 @@ if not verify_identifier(token): raise TokenError("invalid character in identifier", line, lnum, start + 1, token_list) - token_list.append((tokens.NAME, token, lnum, start, line)) + if token == "async": + token_list.append((tokens.ASYNC, token, lnum, start, line)) + elif token == "await": + token_list.append((tokens.AWAIT, token, lnum, start, line)) + else: + token_list.append((tokens.NAME, token, lnum, start, line)) last_comment = '' elif initial == '\\': # continued stmt continued = 1 From pypy.commits at gmail.com Wed Jul 6 11:44:06 2016 From: pypy.commits at gmail.com (plan_rich) Date: Wed, 06 Jul 2016 08:44:06 -0700 (PDT) Subject: [pypy-commit] pypy ppc-vsx-support: solve some translation issues Message-ID: <577d2746.d8b81c0a.c34c5.7061@mx.google.com> Author: Richard Plangger Branch: ppc-vsx-support Changeset: r85577:5576d80ffa0b Date: 2016-07-06 17:43 +0200 http://bitbucket.org/pypy/pypy/changeset/5576d80ffa0b/ Log: solve some translation issues diff --git a/rpython/jit/backend/llsupport/vector_ext.py b/rpython/jit/backend/llsupport/vector_ext.py --- a/rpython/jit/backend/llsupport/vector_ext.py +++ b/rpython/jit/backend/llsupport/vector_ext.py @@ -4,7 +4,8 @@ from rpython.rlib.objectmodel import specialize, always_inline from rpython.jit.metainterp.history import (VECTOR, FLOAT, INT) from rpython.jit.metainterp.resoperation import rop -from rpython.jit.metainterp.optimizeopt.schedule import forwarded_vecinfo +from rpython.jit.metainterp.optimizeopt.schedule import (forwarded_vecinfo, + failnbail_transformation) class TypeRestrict(object): ANY_TYPE = '\x00' diff --git a/rpython/jit/backend/ppc/runner.py b/rpython/jit/backend/ppc/runner.py --- a/rpython/jit/backend/ppc/runner.py +++ b/rpython/jit/backend/ppc/runner.py @@ -12,7 +12,7 @@ class PPC_CPU(AbstractLLCPU): - vector_ext = None + vector_ext = AltiVectorExt() vector_extension = False # may be set to true in setup vector_register_size = 16 vector_horizontal_operations = False @@ -49,7 +49,6 @@ def setup_once(self): self.assembler.setup_once() if detect_vsx(): - self.vector_ext = AltiVectorExt() self.vector_extension = True self.vector_horizontal_operations = True self.assembler.setup_once_vector() diff --git a/rpython/jit/backend/ppc/vector_ext.py b/rpython/jit/backend/ppc/vector_ext.py --- a/rpython/jit/backend/ppc/vector_ext.py +++ b/rpython/jit/backend/ppc/vector_ext.py @@ -272,49 +272,6 @@ self.guard_success_cc = c.negate(self.guard_success_cc) self._emit_guard(guard_op, arglocs) - #def guard_vector(self, guard_op, regalloc, true): - # assert isinstance(guard_op, VectorGuardOp) - # arg = guard_op.getarg(0) - # assert isinstance(arg, VectorOp) - # size = arg.bytesize - # temp = regalloc.get_scratch_reg().value - # load = arg.bytesize * arg.count - self.cpu.vector_register_size - # assert load == 0 - # if true: - # pass - # #self.mc.PXOR(temp, temp) - # # if the vector is not fully packed blend 1s - # #if load < 0: - # # self.mc.PCMPEQQ(temp, temp) # fill with ones - # # self._blend_unused_slots(loc, arg, temp) - # # # reset to zeros - # # self.mc.PXOR(temp, temp) - - # # cmp with zeros (in temp) creates ones at each slot where it is zero - # #self.mc.PCMPEQ(loc, temp, size) - # ## temp converted to ones - # #self.mc.PCMPEQQ(temp, temp) - # ## test if all slots are zero - # #self.mc.PTEST(loc, temp) - # #self.guard_success_cc = rx86.Conditions['Z'] - # else: - # # if the vector is not fully packed blend 1s - # #if load < 0: - # # temp = X86_64_XMM_SCRATCH_REG - # # self.mc.PXOR(temp, temp) - # # self._blend_unused_slots(loc, arg, temp) - # #self.mc.PTEST(loc, loc) - # self.guard_success_cc = rx86.Conditions['NZ'] - - #def _blend_unused_slots(self, loc, arg, temp): - # select = 0 - # bits_used = (arg.count * arg.bytesize * 8) - # index = bits_used // 16 - # while index < 8: - # select |= (1 << index) - # index += 1 - # self.mc.PBLENDW_xxi(loc.value, temp.value, select) - def _update_at_exit(self, fail_locs, fail_args, faildescr, regalloc): """ If accumulation is done in this loop, at the guard exit some vector registers must be adjusted to yield the correct value @@ -370,6 +327,7 @@ not_implemented("reduce sum for %s not impl." % arg) def emit_vec_int_is_true(self, op, arglocs, regalloc): + assert isinstance(op, VectorOp) resloc, argloc, sizeloc = arglocs size = sizeloc.value tmp = regalloc.vrm.get_scratch_reg(type=INT).value @@ -387,6 +345,7 @@ flush_vec_cc(self, regalloc, c.VNEI, op.bytesize, resloc) def emit_vec_float_eq(self, op, arglocs, regalloc): + assert isinstance(op, VectorOp) resloc, loc1, loc2, sizeloc = arglocs size = sizeloc.value tmp = regalloc.vrm.get_scratch_reg().value @@ -413,6 +372,7 @@ self.mc.xxlxor(res, r0, r1) def emit_vec_float_ne(self, op, arglocs, regalloc): + assert isinstance(op, VectorOp) resloc, loc1, loc2, sizeloc = arglocs size = sizeloc.value tmp = regalloc.vrm.get_scratch_reg().value @@ -444,6 +404,7 @@ self.mc.xvcvsxddp(res.value, res.value) def emit_vec_int_eq(self, op, arglocs, regalloc): + assert isinstance(op, VectorOp) res, l0, l1, sizeloc = arglocs size = sizeloc.value if size == 1: @@ -457,9 +418,10 @@ flush_vec_cc(self, regalloc, c.VEQI, op.bytesize, res) def emit_vec_int_ne(self, op, arglocs, regalloc): + assert isinstance(op, VectorOp) res, l0, l1, sizeloc = arglocs size = sizeloc.value - tmp = regalloc.get_scratch_reg().value + tmp = regalloc.vrm.get_scratch_reg(type=INT).value self.mc.vxor(tmp, tmp, tmp) if size == 1: self.mc.vcmpequbx(res.value, res.value, tmp) @@ -473,6 +435,7 @@ flush_vec_cc(self, regalloc, c.VEQI, op.bytesize, res) def emit_vec_expand_f(self, op, arglocs, regalloc): + assert isinstance(op, VectorOp) resloc, srcloc = arglocs size = op.bytesize res = resloc.value @@ -490,6 +453,7 @@ not_implemented("vec expand in this combination not supported") def emit_vec_expand_i(self, op, arglocs, regalloc): + assert isinstance(op, VectorOp) res, l0, off = arglocs size = op.bytesize @@ -516,6 +480,7 @@ not_implemented("expand int size not impl") def emit_vec_pack_i(self, op, arglocs, regalloc): + assert isinstance(op, VectorOp) resultloc, vloc, sourceloc, residxloc, srcidxloc, countloc = arglocs srcidx = srcidxloc.value residx = residxloc.value @@ -540,6 +505,7 @@ not_implemented("vec_pack_i") def emit_vec_unpack_i(self, op, arglocs, regalloc): + assert isinstance(op, VectorOp) resloc, srcloc, idxloc, countloc = arglocs idx = idxloc.value res = resloc.value @@ -568,6 +534,7 @@ (size*8, count)) def emit_vec_pack_f(self, op, arglocs, regalloc): + assert isinstance(op, VectorOp) resloc, vloc, srcloc, residxloc, srcidxloc, countloc = arglocs vec = vloc.value res = resloc.value @@ -692,9 +659,9 @@ prepare_vec_load_f = _prepare_load def prepare_vec_arith(self, op): + assert isinstance(op, VectorOp) a0 = op.getarg(0) a1 = op.getarg(1) - assert isinstance(op, VectorOp) size = op.bytesize args = op.getarglist() loc0 = self.ensure_vector_reg(a0) @@ -718,9 +685,9 @@ del prepare_vec_arith def prepare_vec_bool(self, op): + assert isinstance(op, VectorOp) a0 = op.getarg(0) a1 = op.getarg(1) - assert isinstance(op, VectorOp) size = op.bytesize args = op.getarglist() loc0 = self.ensure_vector_reg(a0) @@ -760,6 +727,7 @@ return [resloc, loc0] def prepare_vec_arith_unary(self, op): + assert isinstance(op, VectorOp) a0 = op.getarg(0) loc0 = self.ensure_vector_reg(a0) resloc = self.force_allocate_vector_reg(op) @@ -832,6 +800,7 @@ return l.ConstFloatLoc(adr) def prepare_vec_expand_f(self, op): + assert isinstance(op, VectorOp) arg = op.getarg(0) if arg.is_constant(): l0 = self.expand_float(op.bytesize, arg) @@ -842,9 +811,11 @@ return [res, l0] def prepare_vec_expand_i(self, op): + assert isinstance(op, VectorOp) arg = op.getarg(0) mc = self.assembler.mc if arg.is_constant(): + assert isinstance(arg, ConstInt) l0 = self.rm.get_scratch_reg() mc.load_imm(l0, arg.value) else: @@ -857,6 +828,7 @@ return [res, l0, imm(PARAM_SAVE_AREA_OFFSET)] def prepare_vec_int_is_true(self, op): + assert isinstance(op, VectorOp) arg = op.getarg(0) assert isinstance(arg, VectorOp) argloc = self.ensure_vector_reg(arg) From pypy.commits at gmail.com Wed Jul 6 11:57:10 2016 From: pypy.commits at gmail.com (arigo) Date: Wed, 06 Jul 2016 08:57:10 -0700 (PDT) Subject: [pypy-commit] pypy use-mmap: Trying to use mmap() for memory allocations done by the RPython GC Message-ID: <577d2a56.d11b1c0a.f09ba.12ec@mx.google.com> Author: Armin Rigo Branch: use-mmap Changeset: r85578:c645b576c5ad Date: 2016-07-06 15:14 +0200 http://bitbucket.org/pypy/pypy/changeset/c645b576c5ad/ Log: Trying to use mmap() for memory allocations done by the RPython GC diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py --- a/pypy/module/cpyext/api.py +++ b/pypy/module/cpyext/api.py @@ -613,7 +613,7 @@ }.items(): GLOBALS['%s#%s' % (cpyname, pypy_decl)] = ('PyTypeObject*', pypyexpr) - for cpyname in '''PyMethodObject PyListObject PyLongObject + for cpyname in '''PyMethodObject PyListObject PyDictObject PyClassObject'''.split(): FORWARD_DECLS.append('typedef struct { PyObject_HEAD } %s' % (cpyname, )) diff --git a/pypy/module/cpyext/include/longintrepr.h b/pypy/module/cpyext/include/longintrepr.h --- a/pypy/module/cpyext/include/longintrepr.h +++ b/pypy/module/cpyext/include/longintrepr.h @@ -1,1 +1,28 @@ -/* empty */ +#ifndef Py_LONGINTREPR_H +#define Py_LONGINTREPR_H + +/* + Should not be used, but is, by a few projects out there. +*/ +#include + + +#define PYLONG_BITS_IN_DIGIT 30 +typedef uint32_t digit; +typedef int32_t sdigit; +typedef uint64_t twodigits; +typedef int64_t stwodigits; +#define PyLong_SHIFT 30 +#define _PyLong_DECIMAL_SHIFT 9 /* max(e such that 10**e fits in a digit) */ +#define _PyLong_DECIMAL_BASE ((digit)1000000000) /* 10 ** DECIMAL_SHIFT */ + +#define PyLong_BASE ((digit)1 << PyLong_SHIFT) +#define PyLong_MASK ((digit)(PyLong_BASE - 1)) + +/* b/w compatibility with Python 2.5 */ +#define SHIFT PyLong_SHIFT +#define BASE PyLong_BASE +#define MASK PyLong_MASK + + +#endif diff --git a/pypy/module/cpyext/longobject.py b/pypy/module/cpyext/longobject.py --- a/pypy/module/cpyext/longobject.py +++ b/pypy/module/cpyext/longobject.py @@ -9,6 +9,14 @@ from rpython.rlib.rarithmetic import intmask +PyLongObjectStruct = lltype.ForwardReference() +PyLongObject = lltype.Ptr(PyLongObjectStruct) +Digits = rffi.CArray(PyObject) +PyTupleObjectFields = PyObjectFields + \ + (("ob_size", Py_ssize_t), ("ob_item", lltype.Ptr(ObjectItems))) +cpython_struct("PyTupleObject", PyTupleObjectFields, PyTupleObjectStruct) + + PyLong_Check, PyLong_CheckExact = build_type_checkers("Long") @cpython_api([lltype.Signed], PyObject) From pypy.commits at gmail.com Wed Jul 6 11:57:12 2016 From: pypy.commits at gmail.com (arigo) Date: Wed, 06 Jul 2016 08:57:12 -0700 (PDT) Subject: [pypy-commit] pypy use-mmap: Trying to use mmap() instead of malloc() to get arenas. Advantage: munmap() Message-ID: <577d2a58.67c0c20a.6d103.7b4a@mx.google.com> Author: Armin Rigo Branch: use-mmap Changeset: r85579:f63cad3be093 Date: 2016-07-06 16:43 +0200 http://bitbucket.org/pypy/pypy/changeset/f63cad3be093/ Log: Trying to use mmap() instead of malloc() to get arenas. Advantage: munmap() really returns the 256K/512K of memory to the system. diff --git a/rpython/memory/gc/incminimark.py b/rpython/memory/gc/incminimark.py --- a/rpython/memory/gc/incminimark.py +++ b/rpython/memory/gc/incminimark.py @@ -727,9 +727,10 @@ return llmemory.cast_adr_to_ptr(obj, llmemory.GCREF) - def collect(self, gen=2): + def collect(self, gen=3): """Do a minor (gen=0), start a major (gen=1), or do a full - major (gen>=2) collection.""" + major (gen>=2) collection. If gen>=3 then ask minimarkpage + to free now its cache of arenas.""" if gen < 0: self._minor_collection() # dangerous! no major GC cycle progress elif gen <= 1: @@ -738,6 +739,8 @@ self.major_collection_step() else: self.minor_and_major_collection() + if gen >= 3: + self.ac.kill_dying_arenas() self.rrc_invoke_callback() diff --git a/rpython/memory/gc/minimarkpage.py b/rpython/memory/gc/minimarkpage.py --- a/rpython/memory/gc/minimarkpage.py +++ b/rpython/memory/gc/minimarkpage.py @@ -124,6 +124,16 @@ flavor='raw', zero=True, immortal=True) # + # two lists of completely free arenas. These arenas are + # pending to be returned to the OS. They are first added to + # the 'dying_arenas' list when the major collection runs. At + # the end, they are moved to the 'dead_arenas' list, and all + # arenas that were already in the 'dead_arenas' list at that + # point (from the previous major collection) are really + # returned to the OS. + self.dying_arenas = ARENA_NULL + self.dead_arenas = ARENA_NULL + # # the arena currently consumed; it must have at least one page # available, or be NULL. The arena object that we point to is # not in any 'arenas_lists'. We will consume all its pages before @@ -244,7 +254,7 @@ def _all_arenas(self): - """For testing. Enumerates all arenas.""" + """For testing. Enumerates all not-completely-free arenas.""" if self.current_arena: yield self.current_arena for arena in self.arenas_lists: @@ -290,31 +300,58 @@ for a in self._all_arenas(): assert a.nfreepages == 0 # - # 'arena_base' points to the start of malloced memory; it might not - # be a page-aligned address - arena_base = llarena.arena_malloc(self.arena_size, False) - if not arena_base: - out_of_memory("out of memory: couldn't allocate the next arena") - arena_end = arena_base + self.arena_size + # Maybe we have a dying or a dead arena. + if self.dying_arenas: + arena = self.dying_arenas + self.dying_arenas = arena.nextarena + elif self.dead_arenas: + arena = self.dead_arenas + self.dead_arenas = arena.nextarena + else: + # + # 'arena_base' points to the start of malloced memory. It might + # not be a page-aligned address depending on the underlying call + # (although with mmap() it should be). + arena_base = llarena.arena_mmap(self.arena_size) + if not arena_base: + out_of_memory("out of memory: couldn't allocate the next arena") + arena_end = arena_base + self.arena_size + # + # 'firstpage' points to the first unused page + firstpage = start_of_page(arena_base + self.page_size - 1, + self.page_size) + # 'npages' is the number of full pages just allocated + npages = (arena_end - firstpage) // self.page_size + # + # Allocate an ARENA object and initialize it + arena = lltype.malloc(ARENA, flavor='raw', track_allocation=False) + arena.base = arena_base + arena.totalpages = npages # - # 'firstpage' points to the first unused page - firstpage = start_of_page(arena_base + self.page_size - 1, - self.page_size) - # 'npages' is the number of full pages just allocated - npages = (arena_end - firstpage) // self.page_size - # - # Allocate an ARENA object and initialize it - arena = lltype.malloc(ARENA, flavor='raw', track_allocation=False) - arena.base = arena_base arena.nfreepages = 0 # they are all uninitialized pages - arena.totalpages = npages - arena.freepages = firstpage - self.num_uninitialized_pages = npages + arena.freepages = start_of_page(arena.base + self.page_size - 1, + self.page_size) + arena.nextarena = ARENA_NULL + self.num_uninitialized_pages = arena.totalpages self.current_arena = arena # allocate_new_arena._dont_inline_ = True + def kill_dying_arenas(self): + """Return to the OS all dead arenas, and then move the 'dying' + arenas to the 'dead' arenas list. + """ + arena = self.dead_arenas + while arena: + nextarena = arena.nextarena + llarena.arena_munmap(arena.base, self.arena_size) + lltype.free(arena, flavor='raw', track_allocation=False) + arena = nextarena + self.dead_arenas = self.dying_arenas + self.dying_arenas = ARENA_NULL + + def mass_free_prepare(self): """Prepare calls to mass_free_incremental(): moves the chained lists into 'self.old_xxx'. @@ -360,6 +397,7 @@ if size_class >= 0: self._rehash_arenas_lists() self.size_class_with_old_pages = -1 + self.kill_dying_arenas() # return True @@ -394,9 +432,12 @@ # if arena.nfreepages == arena.totalpages: # - # The whole arena is empty. Free it. - llarena.arena_free(arena.base) - lltype.free(arena, flavor='raw', track_allocation=False) + # The whole arena is empty. Move it to the dying list. + arena.nextarena = self.dying_arenas + self.dying_arenas = arena + llarena.arena_reset(arena.base, + llarena.RESET_WHOLE_ARENA, + 0) # else: # Insert 'arena' in the correct arenas_lists[n] diff --git a/rpython/rlib/rmmap.py b/rpython/rlib/rmmap.py --- a/rpython/rlib/rmmap.py +++ b/rpython/rlib/rmmap.py @@ -691,6 +691,22 @@ m.setdata(res, map_size) return m + def allocate_memory_chunk(map_size): + # used by the memory allocator (in llarena.py, from minimarkpage.py) + flags = MAP_PRIVATE | MAP_ANONYMOUS + prot = PROT_READ | PROT_WRITE + if we_are_translated(): + flags = NonConstant(flags) + prot = NonConstant(prot) + res = c_mmap_safe(rffi.cast(PTR, 0), map_size, prot, flags, -1, 0) + if res == rffi.cast(PTR, -1): + res = rffi.cast(PTR, 0) + return res + + def free_memory_chunk(addr, map_size): + # used by the memory allocator (in llarena.py, from minimarkpage.py) + c_munmap_safe(addr, map_size) + def alloc_hinted(hintp, map_size): flags = MAP_PRIVATE | MAP_ANONYMOUS prot = PROT_EXEC | PROT_READ | PROT_WRITE diff --git a/rpython/rtyper/lltypesystem/llarena.py b/rpython/rtyper/lltypesystem/llarena.py --- a/rpython/rtyper/lltypesystem/llarena.py +++ b/rpython/rtyper/lltypesystem/llarena.py @@ -13,13 +13,15 @@ # it internally uses raw_malloc_usage() to estimate the number of bytes # it needs to reserve. +RESET_WHOLE_ARENA = -909 + class ArenaError(Exception): pass class Arena(object): _count_arenas = 0 - def __init__(self, nbytes, zero): + def __init__(self, nbytes, zero, mmaped=False): Arena._count_arenas += 1 self._arena_index = Arena._count_arenas self.nbytes = nbytes @@ -29,13 +31,14 @@ self.freed = False self.protect_inaccessible = None self.reset(zero) + self.mmaped = mmaped def __repr__(self): return '' % (self._arena_index, self.nbytes) - def reset(self, zero, start=0, size=None): + def reset(self, zero, start=0, size=RESET_WHOLE_ARENA): self.check() - if size is None: + if type(size) is int and size == RESET_WHOLE_ARENA: stop = self.nbytes else: stop = start + llmemory.raw_malloc_usage(size) @@ -315,18 +318,33 @@ # arena_new_view(ptr) is a no-op when translated, returns fresh view # on previous arena when run on top of llinterp +# arena_mmap() and arena_munmap() are similar to arena_malloc(zero=True) +# and arena_free(), but use the OS's mmap() function. + def arena_malloc(nbytes, zero): """Allocate and return a new arena, optionally zero-initialized.""" return Arena(nbytes, zero).getaddr(0) def arena_free(arena_addr): """Release an arena.""" + _arena_free(arena_addr, mmaped=False) + +def _arena_free(arena_addr, mmaped): assert isinstance(arena_addr, fakearenaaddress) assert arena_addr.offset == 0 + assert arena_addr.arena.mmaped == mmaped arena_addr.arena.reset(False) assert not arena_addr.arena.objectptrs arena_addr.arena.mark_freed() +def arena_mmap(nbytes): + """Allocate a new arena using mmap().""" + return Arena(nbytes, zero=True, mmaped=True).getaddr(0) + +def arena_munmap(arena_addr, original_nbytes): + """Release an arena allocated with arena_mmap().""" + _arena_free(arena_addr, mmaped=True) + def arena_reset(arena_addr, size, zero): """Free all objects in the arena, which can then be reused. This can also be used on a subrange of the arena. @@ -335,6 +353,7 @@ * 1: clear, optimized for a very large area of memory * 2: clear, optimized for a small or medium area of memory * 3: fill with garbage + If 'zero' is set to 0, then size can be RESET_WHOLE_ARENA. """ arena_addr = getfakearenaaddress(arena_addr) arena_addr.arena.reset(zero, arena_addr.offset, size) @@ -503,6 +522,26 @@ llfakeimpl=arena_free, sandboxsafe=True) +def llimpl_arena_mmap(nbytes): + from rpython.rlib import rmmap + return rffi.cast(llmemory.Address, rmmap.allocate_memory_chunk(nbytes)) + +def llimpl_arena_munmap(addr, nbytes): + from rpython.rlib import rmmap + rmmap.free_memory_chunk(rffi.cast(rmmap.PTR, addr), nbytes) + +register_external(arena_mmap, [int], llmemory.Address, + 'll_arena.arena_mmap', + llimpl=llimpl_arena_mmap, + llfakeimpl=arena_mmap, + sandboxsafe=True) + +register_external(arena_munmap, [llmemory.Address, int], None, + 'll_arena.arena_munmap', + llimpl=llimpl_arena_munmap, + llfakeimpl=arena_munmap, + sandboxsafe=True) + def llimpl_arena_reset(arena_addr, size, zero): if zero: if zero == 1: From pypy.commits at gmail.com Wed Jul 6 12:13:31 2016 From: pypy.commits at gmail.com (arigo) Date: Wed, 06 Jul 2016 09:13:31 -0700 (PDT) Subject: [pypy-commit] pypy use-mmap: Backed out changeset c645b576c5ad Message-ID: <577d2e2b.049bc20a.dcf70.ffffc527@mx.google.com> Author: Armin Rigo Branch: use-mmap Changeset: r85580:8a06c05e563e Date: 2016-07-06 18:14 +0200 http://bitbucket.org/pypy/pypy/changeset/8a06c05e563e/ Log: Backed out changeset c645b576c5ad This was not meant to be checked in. diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py --- a/pypy/module/cpyext/api.py +++ b/pypy/module/cpyext/api.py @@ -613,7 +613,7 @@ }.items(): GLOBALS['%s#%s' % (cpyname, pypy_decl)] = ('PyTypeObject*', pypyexpr) - for cpyname in '''PyMethodObject PyListObject + for cpyname in '''PyMethodObject PyListObject PyLongObject PyDictObject PyClassObject'''.split(): FORWARD_DECLS.append('typedef struct { PyObject_HEAD } %s' % (cpyname, )) diff --git a/pypy/module/cpyext/include/longintrepr.h b/pypy/module/cpyext/include/longintrepr.h --- a/pypy/module/cpyext/include/longintrepr.h +++ b/pypy/module/cpyext/include/longintrepr.h @@ -1,28 +1,1 @@ -#ifndef Py_LONGINTREPR_H -#define Py_LONGINTREPR_H - -/* - Should not be used, but is, by a few projects out there. -*/ -#include - - -#define PYLONG_BITS_IN_DIGIT 30 -typedef uint32_t digit; -typedef int32_t sdigit; -typedef uint64_t twodigits; -typedef int64_t stwodigits; -#define PyLong_SHIFT 30 -#define _PyLong_DECIMAL_SHIFT 9 /* max(e such that 10**e fits in a digit) */ -#define _PyLong_DECIMAL_BASE ((digit)1000000000) /* 10 ** DECIMAL_SHIFT */ - -#define PyLong_BASE ((digit)1 << PyLong_SHIFT) -#define PyLong_MASK ((digit)(PyLong_BASE - 1)) - -/* b/w compatibility with Python 2.5 */ -#define SHIFT PyLong_SHIFT -#define BASE PyLong_BASE -#define MASK PyLong_MASK - - -#endif +/* empty */ diff --git a/pypy/module/cpyext/longobject.py b/pypy/module/cpyext/longobject.py --- a/pypy/module/cpyext/longobject.py +++ b/pypy/module/cpyext/longobject.py @@ -9,14 +9,6 @@ from rpython.rlib.rarithmetic import intmask -PyLongObjectStruct = lltype.ForwardReference() -PyLongObject = lltype.Ptr(PyLongObjectStruct) -Digits = rffi.CArray(PyObject) -PyTupleObjectFields = PyObjectFields + \ - (("ob_size", Py_ssize_t), ("ob_item", lltype.Ptr(ObjectItems))) -cpython_struct("PyTupleObject", PyTupleObjectFields, PyTupleObjectStruct) - - PyLong_Check, PyLong_CheckExact = build_type_checkers("Long") @cpython_api([lltype.Signed], PyObject) From pypy.commits at gmail.com Wed Jul 6 12:15:07 2016 From: pypy.commits at gmail.com (plan_rich) Date: Wed, 06 Jul 2016 09:15:07 -0700 (PDT) Subject: [pypy-commit] pypy default: fix test_compile_asmlen Message-ID: <577d2e8b.4a2d1c0a.e84ed.ffffd4c1@mx.google.com> Author: Richard Plangger Branch: Changeset: r85581:5a0e839de8ba Date: 2016-07-06 18:14 +0200 http://bitbucket.org/pypy/pypy/changeset/5a0e839de8ba/ Log: fix test_compile_asmlen diff --git a/rpython/jit/backend/zarch/test/test_runner.py b/rpython/jit/backend/zarch/test/test_runner.py --- a/rpython/jit/backend/zarch/test/test_runner.py +++ b/rpython/jit/backend/zarch/test/test_runner.py @@ -26,4 +26,4 @@ add_loop_instructions = "lg; lgr; larl; agr; cgfi; jge; j;$" bridge_loop_instructions = "lg; cgfi; jnl; lghi; " \ - "(lgfi|iilf);( iihf;)? (lgfi|iilf);( iihf;)? basr; larl; (lgfi|iilf);( iihf;)? br;$" + "(lgfi|iilf);( iihf;)? (lgfi|iilf);( iihf;)? stg; basr; larl; (lgfi|iilf);( iihf;)? br;$" From pypy.commits at gmail.com Wed Jul 6 12:16:56 2016 From: pypy.commits at gmail.com (arigo) Date: Wed, 06 Jul 2016 09:16:56 -0700 (PDT) Subject: [pypy-commit] pypy use-mmap: Test fix Message-ID: <577d2ef8.901e1c0a.eea3e.0eeb@mx.google.com> Author: Armin Rigo Branch: use-mmap Changeset: r85582:c9f27194913d Date: 2016-07-06 18:18 +0200 http://bitbucket.org/pypy/pypy/changeset/c9f27194913d/ Log: Test fix diff --git a/rpython/memory/gc/minimarktest.py b/rpython/memory/gc/minimarktest.py --- a/rpython/memory/gc/minimarktest.py +++ b/rpython/memory/gc/minimarktest.py @@ -56,3 +56,6 @@ self.mass_free_prepare() res = self.mass_free_incremental(ok_to_free_func, sys.maxint) assert res + + def kill_dying_arenas(self): + pass From pypy.commits at gmail.com Wed Jul 6 12:37:50 2016 From: pypy.commits at gmail.com (plan_rich) Date: Wed, 06 Jul 2016 09:37:50 -0700 (PDT) Subject: [pypy-commit] pypy new-jit-log: using the the same trick as done in rlib/jithook to call flush_trace_counters on the assembler Message-ID: <577d33de.571a1c0a.1a06b.12e9@mx.google.com> Author: Richard Plangger Branch: new-jit-log Changeset: r85583:f669d0340e40 Date: 2016-07-06 18:37 +0200 http://bitbucket.org/pypy/pypy/changeset/f669d0340e40/ Log: using the the same trick as done in rlib/jithook to call flush_trace_counters on the assembler diff --git a/rpython/jit/backend/llsupport/assembler.py b/rpython/jit/backend/llsupport/assembler.py --- a/rpython/jit/backend/llsupport/assembler.py +++ b/rpython/jit/backend/llsupport/assembler.py @@ -10,11 +10,16 @@ debug_print) from rpython.rlib.rarithmetic import r_uint from rpython.rlib.objectmodel import specialize, compute_unique_id +from rpython.rlib.jitlog import _log_jit_counter from rpython.rtyper.annlowlevel import cast_instance_to_gcref, llhelper from rpython.rtyper.lltypesystem import rffi, lltype -from rpython.jit.metainterp.debug import (DEBUG_COUNTER, debug_sd, - flush_debug_counters) +DEBUG_COUNTER = lltype.Struct('DEBUG_COUNTER', + # 'b'ridge, 'l'abel or # 'e'ntry point + ('i', lltype.Signed), # first field, at offset 0 + ('type', lltype.Char), + ('number', lltype.Signed) +) class GuardToken(object): def __init__(self, cpu, gcmap, faildescr, failargs, fail_locs, @@ -69,6 +74,7 @@ self.memset_addr = 0 self.rtyper = cpu.rtyper self._debug = False + self.loop_run_counters = [] def stitch_bridge(self, faildescr, target): raise NotImplementedError @@ -333,7 +339,7 @@ self._call_assembler_patch_jmp(jmp_location) def get_loop_run_counters(self, index): - return debug_sd.loop_run_counters[index] + return self.loop_run_counters[index] @specialize.argtype(1) def _inject_debugging_code(self, looptoken, operations, tp, number): @@ -366,16 +372,19 @@ else: assert token struct.number = compute_unique_id(token) - debug_sd.loop_run_counters.append(struct) + # YYY very minor leak -- we need the counters to stay alive + # forever, just because we want to report them at the end + # of the process + self.loop_run_counters.append(struct) return struct def finish_once(self): if self._debug: # TODO remove the old logging system when jitlog is complete debug_start('jit-backend-counts') - length = len(debug_sd.loop_run_counters) + length = len(self.loop_run_counters) for i in range(length): - struct = debug_sd.loop_run_counters[i] + struct = self.loop_run_counters[i] if struct.type == 'l': prefix = 'TargetToken(%d)' % struct.number else: @@ -391,7 +400,20 @@ debug_print(prefix + ':' + str(struct.i)) debug_stop('jit-backend-counts') - flush_debug_counters() + self.flush_trace_counters() + + def flush_trace_counters(self): + # this is always called, the jitlog knows if it is enabled + length = len(self.loop_run_counters) + for i in range(length): + struct = self.loop_run_counters[i] + _log_jit_counter(struct) + # reset the counter, flush in a later point in time will + # add up the counters! + struct.i = 0 + # here would be the point to free some counters + # see YYY comment above! but first we should run this every once in a while + # not just when jitlog_disable is called @staticmethod @rgc.no_collect diff --git a/rpython/jit/metainterp/debug.py b/rpython/jit/metainterp/debug.py deleted file mode 100644 --- a/rpython/jit/metainterp/debug.py +++ /dev/null @@ -1,32 +0,0 @@ -from rpython.rtyper.lltypesystem import rffi, lltype -from rpython.rlib.jitlog import _log_jit_counter - -# YYY very minor leak -- we need the counters to stay alive -# forever, just because we want to report them at the end -# of the process - -class DebugStaticData(object): - def __init__(self): - self.loop_run_counters = [] - -debug_sd = DebugStaticData() - -DEBUG_COUNTER = lltype.Struct('DEBUG_COUNTER', - # 'b'ridge, 'l'abel or # 'e'ntry point - ('i', lltype.Signed), # first field, at offset 0 - ('type', lltype.Char), - ('number', lltype.Signed) -) - -def flush_debug_counters(): - # this is always called, the jitlog knows if it is enabled - length = len(debug_sd.loop_run_counters) - for i in range(length): - struct = debug_sd.loop_run_counters[i] - _log_jit_counter(struct) - # reset the counter, flush in a later point in time will - # add up the counters! - struct.i = 0 - # here would be the point to free some counters - # see YYY comment above! but first we should run this every once in a while - # not just when jitlog_disable is called diff --git a/rpython/jit/metainterp/test/test_ajit.py b/rpython/jit/metainterp/test/test_ajit.py --- a/rpython/jit/metainterp/test/test_ajit.py +++ b/rpython/jit/metainterp/test/test_ajit.py @@ -4,6 +4,7 @@ import weakref from rpython.rlib import rgc +from rpython.rlib.debug import debug_flush_trace_counts from rpython.jit.codewriter.policy import StopAtXPolicy from rpython.jit.metainterp import history from rpython.jit.metainterp.test.support import LLJitMixin, noConst @@ -64,6 +65,20 @@ res = self.interp_operations(f, [8, 98]) assert res == 110 + def test_flush_trace_count(self): + myjitdriver = JitDriver(greens = [], reds = ['i']) + def f(i): + while i > 0: + myjitdriver.can_enter_jit(i=i) + myjitdriver.jit_merge_point(i=i) + if i == 4: + debug_flush_trace_counts(None) + print("4") + i -= 1 + return i + res = self.meta_interp(f, [40]) + assert res == 0 + def test_loop_1(self): myjitdriver = JitDriver(greens = [], reds = ['x', 'y', 'res']) def f(x, y): diff --git a/rpython/jit/metainterp/warmspot.py b/rpython/jit/metainterp/warmspot.py --- a/rpython/jit/metainterp/warmspot.py +++ b/rpython/jit/metainterp/warmspot.py @@ -168,6 +168,7 @@ def find_jit_merge_points(graphs): results = _find_jit_marker(graphs, 'jit_merge_point') + import pdb; pdb.set_trace() if not results: raise Exception("no jit_merge_point found!") seen = set([graph for graph, block, pos in results]) diff --git a/rpython/rlib/jitlog.py b/rpython/rlib/jitlog.py --- a/rpython/rlib/jitlog.py +++ b/rpython/rlib/jitlog.py @@ -12,6 +12,14 @@ from rpython.rlib.objectmodel import compute_unique_id, always_inline from rpython.rlib.objectmodel import we_are_translated, specialize from rpython.rlib.unroll import unrolling_iterable +from rpython.rlib.jit_hooks import register_helper +from rpython.annotator import model as annmodel + + at register_helper(None) +def stats_flush_trace_counts(warmrunnerdesc): + print("hello") + warmrunnerdesc.metainterp_sd.cpu.assembler.flush_trace_counters() + return True def commonprefix(a,b): "Given a list of pathnames, returns the longest common leading component" diff --git a/rpython/rlib/rvmprof/rvmprof.py b/rpython/rlib/rvmprof/rvmprof.py --- a/rpython/rlib/rvmprof/rvmprof.py +++ b/rpython/rlib/rvmprof/rvmprof.py @@ -132,8 +132,8 @@ self.cintf.jitlog_write_marked(jl.MARK_JITLOG_HEADER + blob, len(blob) + 1) def disable_jitlog(self): - from rpython.jit.metainterp.debug import flush_debug_counters - flush_debug_counters() + from rpython.rlib import debug + stats_flush_trace_counts(None) self.cintf.jitlog_teardown() def disable(self): diff --git a/rpython/rtyper/lltypesystem/lloperation.py b/rpython/rtyper/lltypesystem/lloperation.py --- a/rpython/rtyper/lltypesystem/lloperation.py +++ b/rpython/rtyper/lltypesystem/lloperation.py @@ -541,17 +541,18 @@ 'threadlocalref_enum': LLOp(sideeffects=False), # enum all threadlocalrefs # __________ debugging __________ - 'debug_view': LLOp(), - 'debug_print': LLOp(canrun=True), - 'debug_start': LLOp(canrun=True), - 'debug_stop': LLOp(canrun=True), - 'have_debug_prints': LLOp(canrun=True), - 'have_debug_prints_for':LLOp(canrun=True), - 'debug_offset': LLOp(canrun=True), - 'debug_flush': LLOp(canrun=True), - 'debug_assert': LLOp(tryfold=True), - 'debug_fatalerror': LLOp(canrun=True), - 'debug_llinterpcall': LLOp(canraise=(Exception,)), + 'debug_flush_trace_counts': LLOp(), + 'debug_view': LLOp(), + 'debug_print': LLOp(canrun=True), + 'debug_start': LLOp(canrun=True), + 'debug_stop': LLOp(canrun=True), + 'have_debug_prints': LLOp(canrun=True), + 'have_debug_prints_for': LLOp(canrun=True), + 'debug_offset': LLOp(canrun=True), + 'debug_flush': LLOp(canrun=True), + 'debug_assert': LLOp(tryfold=True), + 'debug_fatalerror': LLOp(canrun=True), + 'debug_llinterpcall': LLOp(canraise=(Exception,)), # Python func call 'res=arg[0](*arg[1:])' # in backends, abort() or whatever is fine 'debug_start_traceback': LLOp(), diff --git a/rpython/rtyper/lltypesystem/opimpl.py b/rpython/rtyper/lltypesystem/opimpl.py --- a/rpython/rtyper/lltypesystem/opimpl.py +++ b/rpython/rtyper/lltypesystem/opimpl.py @@ -580,6 +580,9 @@ return hlstr(x) return x +def op_debug_flush_log(): + debug.debug_flush_log() + def op_debug_print(*args): debug.debug_print(*map(_normalize, args)) diff --git a/rpython/translator/c/test/test_standalone.py b/rpython/translator/c/test/test_standalone.py --- a/rpython/translator/c/test/test_standalone.py +++ b/rpython/translator/c/test/test_standalone.py @@ -6,6 +6,7 @@ from rpython.rlib.objectmodel import keepalive_until_here from rpython.rlib.rarithmetic import r_longlong from rpython.rlib.debug import ll_assert, have_debug_prints, debug_flush +from rpython.rlib.jitlog import stats_flush_trace_counts from rpython.rlib.debug import debug_print, debug_start, debug_stop from rpython.rlib.debug import debug_offset, have_debug_prints_for from rpython.rlib.entrypoint import entrypoint_highlevel, secondary_entrypoints @@ -17,6 +18,7 @@ from rpython.tool.udir import udir from rpython.translator import cdir from rpython.conftest import option +from rpython.rlib.jit import JitDriver def setup_module(module): if os.name == 'nt': From pypy.commits at gmail.com Wed Jul 6 12:43:22 2016 From: pypy.commits at gmail.com (raffael_t) Date: Wed, 06 Jul 2016 09:43:22 -0700 (PDT) Subject: [pypy-commit] pypy py3.5-async: Add await token test Message-ID: <577d352a.48371c0a.f2b81.7f03@mx.google.com> Author: Raffael Tfirst Branch: py3.5-async Changeset: r85584:f361d2f549f1 Date: 2016-07-06 18:42 +0200 http://bitbucket.org/pypy/pypy/changeset/f361d2f549f1/ Log: Add await token test diff --git a/pypy/interpreter/pyparser/pytokenizer.py b/pypy/interpreter/pyparser/pytokenizer.py --- a/pypy/interpreter/pyparser/pytokenizer.py +++ b/pypy/interpreter/pyparser/pytokenizer.py @@ -253,9 +253,9 @@ if not verify_identifier(token): raise TokenError("invalid character in identifier", line, lnum, start + 1, token_list) - if token == "async": + if token == 'async': token_list.append((tokens.ASYNC, token, lnum, start, line)) - elif token == "await": + elif token == 'await': token_list.append((tokens.AWAIT, token, lnum, start, line)) else: token_list.append((tokens.NAME, token, lnum, start, line)) diff --git a/pypy/interpreter/pyparser/test/test_pyparse.py b/pypy/interpreter/pyparser/test/test_pyparse.py --- a/pypy/interpreter/pyparser/test/test_pyparse.py +++ b/pypy/interpreter/pyparser/test/test_pyparse.py @@ -168,6 +168,8 @@ py.test.raises(SyntaxError, self.parse, 'f()\nxy # blah\nblah()', "single") py.test.raises(SyntaxError, self.parse, 'x = 5 # comment\nx = 6\n', "single") + def test_await(self): + self.parse("await result = func()") class TestPythonParserWithSpace: From pypy.commits at gmail.com Wed Jul 6 13:43:31 2016 From: pypy.commits at gmail.com (arigo) Date: Wed, 06 Jul 2016 10:43:31 -0700 (PDT) Subject: [pypy-commit] pypy use-mmap: translation fix Message-ID: <577d4343.244cc20a.bc2d6.1427@mx.google.com> Author: Armin Rigo Branch: use-mmap Changeset: r85585:8e9459757f75 Date: 2016-07-06 18:51 +0100 http://bitbucket.org/pypy/pypy/changeset/8e9459757f75/ Log: translation fix diff --git a/rpython/rlib/rmmap.py b/rpython/rlib/rmmap.py --- a/rpython/rlib/rmmap.py +++ b/rpython/rlib/rmmap.py @@ -705,7 +705,7 @@ def free_memory_chunk(addr, map_size): # used by the memory allocator (in llarena.py, from minimarkpage.py) - c_munmap_safe(addr, map_size) + c_munmap_safe(addr, rffi.cast(size_t, map_size)) def alloc_hinted(hintp, map_size): flags = MAP_PRIVATE | MAP_ANONYMOUS From pypy.commits at gmail.com Wed Jul 6 13:44:05 2016 From: pypy.commits at gmail.com (raffael_t) Date: Wed, 06 Jul 2016 10:44:05 -0700 (PDT) Subject: [pypy-commit] pypy py3.5-async: Add missing self on calls Message-ID: <577d4365.06a81c0a.47527.ffffb5a6@mx.google.com> Author: Raffael Tfirst Branch: py3.5-async Changeset: r85586:54c81e3580eb Date: 2016-07-06 19:43 +0200 http://bitbucket.org/pypy/pypy/changeset/54c81e3580eb/ Log: Add missing self on calls diff --git a/pypy/interpreter/astcompiler/astbuilder.py b/pypy/interpreter/astcompiler/astbuilder.py --- a/pypy/interpreter/astcompiler/astbuilder.py +++ b/pypy/interpreter/astcompiler/astbuilder.py @@ -515,11 +515,11 @@ def handle_async_stmt(self, node): ch = node.get_child(1) if ch.type == syms.funcdef: - return handle_funcdef_impl(ch, 1) + return self.handle_funcdef_impl(ch, 1) elif ch.type == syms.with_stmt: - return handle_with_stmt(ch, 1) + return self.handle_with_stmt(ch, 1) elif ch.type == syms.for_stmt: - return handle_for_stmt(ch, 1) + return self.handle_for_stmt(ch, 1) else: raise AssertionError("invalid async statement") From pypy.commits at gmail.com Wed Jul 6 14:27:36 2016 From: pypy.commits at gmail.com (plan_rich) Date: Wed, 06 Jul 2016 11:27:36 -0700 (PDT) Subject: [pypy-commit] pypy new-jit-log: remove set trace Message-ID: <577d4d98.cc9d1c0a.19a0c.ffffcdba@mx.google.com> Author: Richard Plangger Branch: new-jit-log Changeset: r85587:a5bf76474c9d Date: 2016-07-06 20:24 +0200 http://bitbucket.org/pypy/pypy/changeset/a5bf76474c9d/ Log: remove set trace diff --git a/rpython/jit/metainterp/warmspot.py b/rpython/jit/metainterp/warmspot.py --- a/rpython/jit/metainterp/warmspot.py +++ b/rpython/jit/metainterp/warmspot.py @@ -168,7 +168,6 @@ def find_jit_merge_points(graphs): results = _find_jit_marker(graphs, 'jit_merge_point') - import pdb; pdb.set_trace() if not results: raise Exception("no jit_merge_point found!") seen = set([graph for graph, block, pos in results]) From pypy.commits at gmail.com Wed Jul 6 14:27:38 2016 From: pypy.commits at gmail.com (plan_rich) Date: Wed, 06 Jul 2016 11:27:38 -0700 (PDT) Subject: [pypy-commit] pypy new-jit-log: change import in disable_jitlog Message-ID: <577d4d9a.571a1c0a.1a06b.388e@mx.google.com> Author: Richard Plangger Branch: new-jit-log Changeset: r85588:c08693600013 Date: 2016-07-06 20:25 +0200 http://bitbucket.org/pypy/pypy/changeset/c08693600013/ Log: change import in disable_jitlog diff --git a/rpython/rlib/rvmprof/rvmprof.py b/rpython/rlib/rvmprof/rvmprof.py --- a/rpython/rlib/rvmprof/rvmprof.py +++ b/rpython/rlib/rvmprof/rvmprof.py @@ -132,7 +132,7 @@ self.cintf.jitlog_write_marked(jl.MARK_JITLOG_HEADER + blob, len(blob) + 1) def disable_jitlog(self): - from rpython.rlib import debug + from rpython.rlib.jitlog import stats_flush_trace_counts stats_flush_trace_counts(None) self.cintf.jitlog_teardown() From pypy.commits at gmail.com Wed Jul 6 14:31:50 2016 From: pypy.commits at gmail.com (arigo) Date: Wed, 06 Jul 2016 11:31:50 -0700 (PDT) Subject: [pypy-commit] pypy use-mmap: Make position hinting work on 64-bit too. At least Linux64 doesn't Message-ID: <577d4e96.972e1c0a.96156.ffffe7db@mx.google.com> Author: Armin Rigo Branch: use-mmap Changeset: r85589:638e16f1b0f4 Date: 2016-07-06 20:33 +0200 http://bitbucket.org/pypy/pypy/changeset/638e16f1b0f4/ Log: Make position hinting work on 64-bit too. At least Linux64 doesn't allow negative addresses to be used by processes, so the hint was never respected. Now I hope it will be, which would avoid the current situation: /proc/../maps shows alternating PROT_EXEC and non-PROT_EXEC maps, which can't be merged by the kernel. diff --git a/rpython/rlib/rmmap.py b/rpython/rlib/rmmap.py --- a/rpython/rlib/rmmap.py +++ b/rpython/rlib/rmmap.py @@ -727,7 +727,10 @@ # XXX is this really necessary? class Hint: - pos = -0x4fff0000 # for reproducible results + if sys.maxint <= 2**32: + pos = -0x4fff0000 # for reproducible results + else: + pos = 0x4fde00000000 hint = Hint() def alloc(map_size): From pypy.commits at gmail.com Wed Jul 6 15:01:27 2016 From: pypy.commits at gmail.com (plan_rich) Date: Wed, 06 Jul 2016 12:01:27 -0700 (PDT) Subject: [pypy-commit] pypy ppc-vsx-support: remove the flags on the cpu, and push them down to the object in the field vector_ext (saved on the cpu). simplifies the vector extension handling Message-ID: <577d5587.c5461c0a.ccb20.6883@mx.google.com> Author: Richard Plangger Branch: ppc-vsx-support Changeset: r85590:7dd9c13d12f8 Date: 2016-07-06 21:00 +0200 http://bitbucket.org/pypy/pypy/changeset/7dd9c13d12f8/ Log: remove the flags on the cpu, and push them down to the object in the field vector_ext (saved on the cpu). simplifies the vector extension handling diff --git a/rpython/jit/backend/llgraph/runner.py b/rpython/jit/backend/llgraph/runner.py --- a/rpython/jit/backend/llgraph/runner.py +++ b/rpython/jit/backend/llgraph/runner.py @@ -327,10 +327,8 @@ supports_guard_gc_type = True translate_support_code = False is_llgraph = True - vector_extension = True - vector_register_size = 16 # in bytes - vector_horizontal_operations = True - vector_pack_slots = True + vector_ext = VectorExt() + vector_ext.enable(16, accum=True) def __init__(self, rtyper, stats=None, *ignored_args, **kwds): model.AbstractCPU.__init__(self) diff --git a/rpython/jit/backend/llsupport/llmodel.py b/rpython/jit/backend/llsupport/llmodel.py --- a/rpython/jit/backend/llsupport/llmodel.py +++ b/rpython/jit/backend/llsupport/llmodel.py @@ -36,10 +36,6 @@ load_supported_factors = (1,) vector_ext = None - vector_extension = False - vector_register_size = 0 # in bytes - vector_horizontal_operations = False - vector_pack_slots = False def __init__(self, rtyper, stats, opts, translate_support_code=False, gcdescr=None): diff --git a/rpython/jit/backend/llsupport/vector_ext.py b/rpython/jit/backend/llsupport/vector_ext.py --- a/rpython/jit/backend/llsupport/vector_ext.py +++ b/rpython/jit/backend/llsupport/vector_ext.py @@ -6,6 +6,7 @@ from rpython.jit.metainterp.resoperation import rop from rpython.jit.metainterp.optimizeopt.schedule import (forwarded_vecinfo, failnbail_transformation) +from rpython.jit.metainterp.jitexc import NotAVectorizeableLoop class TypeRestrict(object): ANY_TYPE = '\x00' @@ -190,6 +191,25 @@ class VectorExt(object): + def __init__(self): + self._enabled = False + self.register_size = 0 # in bytes + self.horizontal_operations = False + + def enable(self, vec_size, accum=False): + self._enabled = vec_size != 0 + self.register_size = vec_size + self.horizontal_operations = accum + + def is_enabled(self): + return self._enabled + + def vec_size(self): + return self.register_size + + def supports_accumulation(self): + return self.horizontal_operations + # note that the following definition is x86 arch specific TR_MAPPING = { rop.VEC_INT_ADD: OR_MSTF_I, diff --git a/rpython/jit/backend/ppc/runner.py b/rpython/jit/backend/ppc/runner.py --- a/rpython/jit/backend/ppc/runner.py +++ b/rpython/jit/backend/ppc/runner.py @@ -13,9 +13,6 @@ class PPC_CPU(AbstractLLCPU): vector_ext = AltiVectorExt() - vector_extension = False # may be set to true in setup - vector_register_size = 16 - vector_horizontal_operations = False supports_floats = True # missing: supports_singlefloats @@ -49,8 +46,7 @@ def setup_once(self): self.assembler.setup_once() if detect_vsx(): - self.vector_extension = True - self.vector_horizontal_operations = True + self.vector_ext.enable(16, accum=True) self.assembler.setup_once_vector() @rgc.no_release_gil From pypy.commits at gmail.com Wed Jul 6 15:11:59 2016 From: pypy.commits at gmail.com (raffael_t) Date: Wed, 06 Jul 2016 12:11:59 -0700 (PDT) Subject: [pypy-commit] pypy py3.5-async: Change visit_annotations in symtable to have args and returns as parameters for async Message-ID: <577d57ff.4aa6c20a.cb609.ffff890d@mx.google.com> Author: Raffael Tfirst Branch: py3.5-async Changeset: r85591:b52ebf4b8d02 Date: 2016-07-06 21:11 +0200 http://bitbucket.org/pypy/pypy/changeset/b52ebf4b8d02/ Log: Change visit_annotations in symtable to have args and returns as parameters for async diff --git a/pypy/interpreter/astcompiler/symtable.py b/pypy/interpreter/astcompiler/symtable.py --- a/pypy/interpreter/astcompiler/symtable.py +++ b/pypy/interpreter/astcompiler/symtable.py @@ -366,7 +366,7 @@ assert isinstance(args, ast.arguments) self.visit_sequence(args.defaults) self.visit_kwonlydefaults(args.kw_defaults) - self._visit_annotations(func) + self._visit_annotations(func, func.args, func.returns) self.visit_sequence(func.decorator_list) new_scope = FunctionScope(func.name, func.lineno, func.col_offset) self.push_scope(new_scope, func) @@ -527,14 +527,13 @@ arg = params[i].arg self.note_symbol(arg, SYM_PARAM) - def _visit_annotations(self, func): - args = func.args + def _visit_annotations(self, func, args, returns): assert isinstance(args, ast.arguments) if args.args: self._visit_arg_annotations(args.args) if args.kwonlyargs: self._visit_arg_annotations(args.kwonlyargs) - if func.returns: + if returns: func.returns.walkabout(self) def _visit_arg_annotations(self, args): From pypy.commits at gmail.com Wed Jul 6 15:56:34 2016 From: pypy.commits at gmail.com (wlav) Date: Wed, 06 Jul 2016 12:56:34 -0700 (PDT) Subject: [pypy-commit] pypy cling-support: from Aditi: more typecode fixes Message-ID: <577d6272.4aa6c20a.cb609.ffff9604@mx.google.com> Author: Wim Lavrijsen Branch: cling-support Changeset: r85592:bbbf4b479fc5 Date: 2016-07-06 12:53 -0700 http://bitbucket.org/pypy/pypy/changeset/bbbf4b479fc5/ Log: from Aditi: more typecode fixes diff --git a/pypy/module/cppyy/converter.py b/pypy/module/cppyy/converter.py --- a/pypy/module/cppyy/converter.py +++ b/pypy/module/cppyy/converter.py @@ -284,6 +284,8 @@ def convert_argument(self, space, w_obj, address, call_local): x = rffi.cast(rffi.LONGP, address) x[0] = self._unwrap_object(space, w_obj) + ba = rffi.cast(rffi.CCHARP, address) + ba[capi.c_function_arg_typeoffset(space)] = 'b' def convert_argument_libffi(self, space, w_obj, address, call_local): x = rffi.cast(rffi.LONGP, address) @@ -307,6 +309,8 @@ def convert_argument(self, space, w_obj, address, call_local): x = rffi.cast(rffi.CCHARP, address) x[0] = self._unwrap_object(space, w_obj) + ba = rffi.cast(rffi.CCHARP, address) + ba[capi.c_function_arg_typeoffset(space)] = 'b' def convert_argument_libffi(self, space, w_obj, address, call_local): x = rffi.cast(self.c_ptrtype, address) diff --git a/pypy/module/cppyy/src/clingcwrapper.cxx b/pypy/module/cppyy/src/clingcwrapper.cxx --- a/pypy/module/cppyy/src/clingcwrapper.cxx +++ b/pypy/module/cppyy/src/clingcwrapper.cxx @@ -368,7 +368,7 @@ case 'K': /* unsigned long long */ vargs[i] = (void*)&args[i].fValue.fULongLong; break; - case 'f': /* double */ + case 'f': /* float */ vargs[i] = (void*)&args[i].fValue.fFloat; break; case 'd': /* double */ diff --git a/pypy/module/cppyy/test/test_datatypes.py b/pypy/module/cppyy/test/test_datatypes.py --- a/pypy/module/cppyy/test/test_datatypes.py +++ b/pypy/module/cppyy/test/test_datatypes.py @@ -124,8 +124,7 @@ assert isinstance(c, cppyy_test_data) # boolean types through functions - c.set_bool(True); - assert c.get_bool() == True + c.set_bool(True); assert c.get_bool() == True c.set_bool(0); assert c.get_bool() == False # boolean types through data members From pypy.commits at gmail.com Wed Jul 6 16:23:15 2016 From: pypy.commits at gmail.com (raffael_t) Date: Wed, 06 Jul 2016 13:23:15 -0700 (PDT) Subject: [pypy-commit] pypy py3.5-async: Undo visit_annotations parameter change, add asyncfunctiondef and asyncwith Message-ID: <577d68b3.e152c20a.c594a.3268@mx.google.com> Author: Raffael Tfirst Branch: py3.5-async Changeset: r85593:90196ea803fd Date: 2016-07-06 22:22 +0200 http://bitbucket.org/pypy/pypy/changeset/90196ea803fd/ Log: Undo visit_annotations parameter change, add asyncfunctiondef and asyncwith diff --git a/pypy/interpreter/astcompiler/symtable.py b/pypy/interpreter/astcompiler/symtable.py --- a/pypy/interpreter/astcompiler/symtable.py +++ b/pypy/interpreter/astcompiler/symtable.py @@ -366,13 +366,29 @@ assert isinstance(args, ast.arguments) self.visit_sequence(args.defaults) self.visit_kwonlydefaults(args.kw_defaults) - self._visit_annotations(func, func.args, func.returns) + self._visit_annotations(func) self.visit_sequence(func.decorator_list) new_scope = FunctionScope(func.name, func.lineno, func.col_offset) self.push_scope(new_scope, func) func.args.walkabout(self) self.visit_sequence(func.body) self.pop_scope() + + def visit_AsyncFunctionDef(self, func): + self.note_symbol(func.name, SYM_ASSIGNED) + # Function defaults and decorators happen in the outer scope. + args = func.args + assert isinstance(args, ast.arguments) + self.visit_sequence(args.defaults) + self.visit_kwonlydefaults(args.kw_defaults) + self._visit_annotations(func) + self.visit_sequence(func.decorator_list) + new_scope = FunctionScope(func.name, func.lineno, func.col_offset) + self.push_scope(new_scope, func) + func.args.walkabout(self) + self.visit_sequence(func.body) + self.pop_scope() + def visit_Return(self, ret): self.scope.note_return(ret) @@ -507,6 +523,13 @@ witem.context_expr.walkabout(self) if witem.optional_vars: witem.optional_vars.walkabout(self) + + def visit_AsyncWith(self, aw): + self.scope.new_temporary_name() + self.visit_sequence(aw.items) + self.scope.note_try_start(aw) + self.visit_sequence(aw.body) + self.scope.note_try_end(aw) def visit_arguments(self, arguments): scope = self.scope @@ -527,13 +550,14 @@ arg = params[i].arg self.note_symbol(arg, SYM_PARAM) - def _visit_annotations(self, func, args, returns): + def _visit_annotations(self, func): + args = func.args assert isinstance(args, ast.arguments) if args.args: self._visit_arg_annotations(args.args) if args.kwonlyargs: self._visit_arg_annotations(args.kwonlyargs) - if returns: + if func.returns: func.returns.walkabout(self) def _visit_arg_annotations(self, args): From pypy.commits at gmail.com Thu Jul 7 02:59:29 2016 From: pypy.commits at gmail.com (arigo) Date: Wed, 06 Jul 2016 23:59:29 -0700 (PDT) Subject: [pypy-commit] pypy default: Issue #2340: itertools.tee(x) uses a different logic if hasattr(iter(x), Message-ID: <577dfdd1.8f1d1c0a.ddb9c.2247@mx.google.com> Author: Armin Rigo Branch: Changeset: r85594:bca9142297fe Date: 2016-07-07 09:00 +0200 http://bitbucket.org/pypy/pypy/changeset/bca9142297fe/ Log: Issue #2340: itertools.tee(x) uses a different logic if hasattr(iter(x), '__copy__'). diff --git a/pypy/module/itertools/interp_itertools.py b/pypy/module/itertools/interp_itertools.py --- a/pypy/module/itertools/interp_itertools.py +++ b/pypy/module/itertools/interp_itertools.py @@ -835,7 +835,7 @@ data before the other iterator, it is faster to use list() instead of tee() - Equivalent to : + If iter(iterable) has no method __copy__(), this is equivalent to: def tee(iterable, n=2): def gen(next, data={}, cnt=[0]): @@ -848,17 +848,22 @@ yield item it = iter(iterable) return tuple([gen(it.next) for i in range(n)]) + + If iter(iterable) has a __copy__ method, though, we just return + a tuple t = (iterable, t[0].__copy__(), t[1].__copy__(), ...). """ if n < 0: raise oefmt(space.w_ValueError, "n must be >= 0") - if isinstance(w_iterable, W_TeeIterable): # optimization only - chained_list = w_iterable.chained_list - w_iterator = w_iterable.w_iterator + if space.findattr(w_iterable, space.wrap("__copy__")) is not None: + # In this case, we don't instantiate any W_TeeIterable. + # We just rely on doing repeated __copy__(). This case + # includes the situation where w_iterable is already + # a W_TeeIterable itself. iterators_w = [w_iterable] * n for i in range(1, n): - iterators_w[i] = space.wrap(W_TeeIterable(space, w_iterator, - chained_list)) + w_iterable = space.call_method(w_iterable, "__copy__") + iterators_w[i] = w_iterable else: w_iterator = space.iter(w_iterable) chained_list = TeeChainedListNode() @@ -891,6 +896,11 @@ self.chained_list = chained_list.next return w_obj + def copy_w(self): + space = self.space + tee_iter = W_TeeIterable(space, self.w_iterator, self.chained_list) + return space.wrap(tee_iter) + def W_TeeIterable___new__(space, w_subtype, w_iterable): # Obscure and undocumented function. PyPy only supports w_iterable # being a W_TeeIterable, because the case where it is a general @@ -906,6 +916,7 @@ __new__ = interp2app(W_TeeIterable___new__), __iter__ = interp2app(W_TeeIterable.iter_w), next = interp2app(W_TeeIterable.next_w), + __copy__ = interp2app(W_TeeIterable.copy_w), __weakref__ = make_weakref_descr(W_TeeIterable), ) W_TeeIterable.typedef.acceptable_as_base_class = False diff --git a/pypy/module/itertools/test/test_itertools.py b/pypy/module/itertools/test/test_itertools.py --- a/pypy/module/itertools/test/test_itertools.py +++ b/pypy/module/itertools/test/test_itertools.py @@ -702,6 +702,48 @@ x = d.next() assert x == 'b' + def test_tee_defines_copy(self): + import itertools + a, b = itertools.tee('abc') + c = b.__copy__() + assert list(a) == ['a', 'b', 'c'] + assert list(b) == ['a', 'b', 'c'] + assert list(c) == ['a', 'b', 'c'] + a, = itertools.tee('abc', 1) + x = a.next() + assert x == 'a' + b = a.__copy__() + x = a.next() + assert x == 'b' + x = b.next() + assert x == 'b' + + def test_tee_function_uses_copy(self): + import itertools + class MyIterator(object): + def __iter__(self): + return self + def next(self): + raise NotImplementedError + def __copy__(self): + return iter('def') + my = MyIterator() + a, = itertools.tee(my, 1) + assert a is my + a, b = itertools.tee(my) + assert a is my + assert b is not my + assert list(b) == ['d', 'e', 'f'] + # this gives AttributeError because it tries to call + # my.__copy__().__copy__() and there isn't one + raises(AttributeError, itertools.tee, my, 3) + + def test_tee_function_empty(self): + import itertools + assert itertools.tee('abc', 0) == () + a, = itertools.tee('abc', 1) + assert itertools.tee(a, 0) == () + class AppTestItertools26: spaceconfig = dict(usemodules=['itertools']) From pypy.commits at gmail.com Thu Jul 7 04:23:37 2016 From: pypy.commits at gmail.com (plan_rich) Date: Thu, 07 Jul 2016 01:23:37 -0700 (PDT) Subject: [pypy-commit] pypy ppc-vsx-support: continue refactoring started in last commit Message-ID: <577e1189.4dd11c0a.ecfe0.ffff8b0c@mx.google.com> Author: Richard Plangger Branch: ppc-vsx-support Changeset: r85595:b129dbdbe458 Date: 2016-07-07 10:22 +0200 http://bitbucket.org/pypy/pypy/changeset/b129dbdbe458/ Log: continue refactoring started in last commit diff --git a/rpython/jit/backend/llgraph/runner.py b/rpython/jit/backend/llgraph/runner.py --- a/rpython/jit/backend/llgraph/runner.py +++ b/rpython/jit/backend/llgraph/runner.py @@ -2,6 +2,7 @@ from rpython.jit.backend import model from rpython.jit.backend.llgraph import support from rpython.jit.backend.llsupport import symbolic +from rpython.jit.backend.llsupport.vector_ext import VectorExt from rpython.jit.metainterp.history import AbstractDescr from rpython.jit.metainterp.history import Const, getkind from rpython.jit.metainterp.history import INT, REF, FLOAT, VOID @@ -931,7 +932,7 @@ def build_getarrayitem(func): def method(self, struct, offset, descr, _count): values = [] - count = self.vector_register_size // descr.get_item_size_in_bytes() + count = self.vector_ext.vec_size() // descr.get_item_size_in_bytes() assert _count == count assert count > 0 for i in range(count): @@ -949,7 +950,7 @@ def _bh_vec_raw_load(self, struct, offset, descr, _count): values = [] stride = descr.get_item_size_in_bytes() - count = self.vector_register_size // descr.get_item_size_in_bytes() + count = self.vector_ext.vec_size() // descr.get_item_size_in_bytes() assert _count == count assert count > 0 for i in range(count): diff --git a/rpython/jit/backend/x86/runner.py b/rpython/jit/backend/x86/runner.py --- a/rpython/jit/backend/x86/runner.py +++ b/rpython/jit/backend/x86/runner.py @@ -63,6 +63,10 @@ if self.HAS_CODEMAP: self.codemap.setup() self.assembler.setup_once() + if self.vector_ext: + # TODO + pass + #self.vector_ext. @rgc.no_release_gil def finish_once(self): @@ -149,8 +153,9 @@ HAS_CODEMAP = True class CPU_X86_64_SSE4(CPU_X86_64): - vector_extension = True - vector_register_size = 16 - vector_horizontal_operations = True + vector_ext = X86VectorExt() + #vector_extension = True + #vector_register_size = 16 + #vector_horizontal_operations = True CPU = CPU386 diff --git a/rpython/jit/metainterp/optimizeopt/schedule.py b/rpython/jit/metainterp/optimizeopt/schedule.py --- a/rpython/jit/metainterp/optimizeopt/schedule.py +++ b/rpython/jit/metainterp/optimizeopt/schedule.py @@ -473,7 +473,7 @@ def __init__(self, graph, packset, cpu, costmodel): SchedulerState.__init__(self, cpu, graph) self.box_to_vbox = {} - self.vec_reg_size = cpu.vector_register_size + self.vec_reg_size = cpu.vector_ext.vec_size() self.expanded_map = {} self.costmodel = costmodel self.inputargs = {} diff --git a/rpython/jit/metainterp/optimizeopt/test/test_costmodel.py b/rpython/jit/metainterp/optimizeopt/test/test_costmodel.py --- a/rpython/jit/metainterp/optimizeopt/test/test_costmodel.py +++ b/rpython/jit/metainterp/optimizeopt/test/test_costmodel.py @@ -13,11 +13,6 @@ from rpython.jit.metainterp.resoperation import rop, ResOperation, AbstractValue from rpython.jit.tool.oparser import parse as opparse from rpython.jit.tool.oparser_model import get_model -from rpython.jit.backend.detect_cpu import getcpuclass - -CPU = getcpuclass() -if not CPU.vector_extension: - py.test.skip("this cpu %s has no implemented vector backend" % CPU) class FakeMemoryRef(object): def __init__(self, array, iv): diff --git a/rpython/jit/metainterp/optimizeopt/test/test_dependency.py b/rpython/jit/metainterp/optimizeopt/test/test_dependency.py --- a/rpython/jit/metainterp/optimizeopt/test/test_dependency.py +++ b/rpython/jit/metainterp/optimizeopt/test/test_dependency.py @@ -35,6 +35,8 @@ def setup_method(self, method): self.test_name = method.__name__ + if not self.cpu.vector_ext.is_enabled(): + py.test.skip("cpu %s needs to implement the vector backend" % self.cpu) def build_dependency(self, ops): loop = self.parse_loop(ops) diff --git a/rpython/jit/metainterp/optimizeopt/test/test_schedule.py b/rpython/jit/metainterp/optimizeopt/test/test_schedule.py --- a/rpython/jit/metainterp/optimizeopt/test/test_schedule.py +++ b/rpython/jit/metainterp/optimizeopt/test/test_schedule.py @@ -15,11 +15,6 @@ from rpython.jit.metainterp.resoperation import rop, ResOperation, VectorizationInfo from rpython.jit.tool.oparser import parse as opparse from rpython.jit.tool.oparser_model import get_model -from rpython.jit.backend.detect_cpu import getcpuclass - -CPU = getcpuclass() -if not CPU.vector_extension: - py.test.skip("this cpu %s has no implemented vector backend" % CPU) class FakeVecScheduleState(VecScheduleState): def __init__(self): diff --git a/rpython/jit/metainterp/optimizeopt/test/test_vecopt.py b/rpython/jit/metainterp/optimizeopt/test/test_vecopt.py --- a/rpython/jit/metainterp/optimizeopt/test/test_vecopt.py +++ b/rpython/jit/metainterp/optimizeopt/test/test_vecopt.py @@ -28,8 +28,6 @@ from rpython.jit.backend.detect_cpu import getcpuclass CPU = getcpuclass() -if not CPU.vector_extension: - py.test.skip("this cpu %s has no implemented vector backend" % CPU) class FakeJitDriverStaticData(object): vec=True @@ -111,7 +109,7 @@ def earlyexit(self, loop): opt = self.vectoroptimizer(loop) graph = opt.analyse_index_calculations(loop) - state = SchedulerState(graph) + state = SchedulerState(self.cpu, graph) opt.schedule(state) return graph.loop @@ -133,7 +131,7 @@ print "CYCLE found %s" % cycle self.show_dot_graph(graph, "early_exit_" + self.test_name) assert cycle is None - state = SchedulerState(graph) + state = SchedulerState(self.cpu, graph) opt.schedule(state) opt.unroll_loop_iterations(loop, unroll_factor) self.debug_print_operations(loop) @@ -269,11 +267,11 @@ pack = Pack([Node(ResOperation(rop.RAW_STORE, [0,0,arg('f',4)], descr), 0), Node(ResOperation(rop.RAW_STORE, [0,0,arg('f',4)], descr), 0), ]) - assert pack.opcount_filling_vector_register(16) == 2 + assert pack.opcount_filling_vector_register(16, self.cpu.vector_ext) == 2 def test_opcount_filling_guard(self): descr = ArrayDescr(0,4, None, 'S') - vec = ResOperation(rop.VEC_RAW_LOAD_I, ['a','i'], descr=descr) + vec = ResOperation(rop.VEC_LOAD_I, ['a','i'], descr=descr) vec.count = 4 pack = Pack([Node(ResOperation(rop.GUARD_TRUE, [vec]), 0), Node(ResOperation(rop.GUARD_TRUE, [vec]), 1), @@ -285,12 +283,13 @@ assert pack.pack_load(16) == 24-16 assert pack.pack_load(8) == 24-8 assert pack.pack_load(32) == 24-32 - assert pack.opcount_filling_vector_register(16) == 4 - ops, newops = pack.slice_operations(16) + ext = self.cpu.vector_ext + assert pack.opcount_filling_vector_register(16, ext) == 4 + ops, newops = pack.slice_operations(16, ext) assert len(ops) == 4 assert len(newops) == 2 - assert pack.opcount_filling_vector_register(8) == 2 - ops, newops = pack.slice_operations(8) + assert pack.opcount_filling_vector_register(8, ext) == 2 + ops, newops = pack.slice_operations(8, ext) assert len(ops) == 2 assert len(newops) == 4 @@ -1044,10 +1043,10 @@ i18 = int_lt(i13, i1) guard_true(i18) [i11,i1,i2,i3,i4] i14 = int_mul(i11, 8) - v19[2xi64] = vec_raw_load_i(i2, i6, descr=arraydescr) - v20[2xi64] = vec_raw_load_i(i3, i6, descr=arraydescr) + v19[2xi64] = vec_load_i(i2, i6, descr=arraydescr) + v20[2xi64] = vec_load_i(i3, i6, descr=arraydescr) v21[2xi64] = vec_int_add(v19, v20) - vec_raw_store(i4, i6, v21, descr=arraydescr) + vec_store(i4, i6, v21, descr=arraydescr) jump(i13, i1, i2, i3, i4) """ loop = self.parse_loop(ops) @@ -1182,7 +1181,7 @@ guard_true(i2) [p0, i0, v2[2xf64]] i10 = int_add(i0, 16) i20 = int_lt(i10, 100) - v1[2xf64] = vec_raw_load_f(p0, i0, descr=floatarraydescr) + v1[2xf64] = vec_load_f(p0, i0, descr=floatarraydescr) v3[2xf64] = vec_float_add(v2[2xf64], v1[2xf64]) jump(p0, i1, v3[2xf64]) """ @@ -1222,10 +1221,10 @@ i55 = int_add(i44, 16) i629 = int_add(i41, 16) i637 = int_add(i37, 16) - v61[2xf64] = vec_raw_load_f(i21, i44, descr=floatarraydescr) - v62[2xf64] = vec_raw_load_f(i4, i41, descr=floatarraydescr) + v61[2xf64] = vec_load_f(i21, i44, descr=floatarraydescr) + v62[2xf64] = vec_load_f(i4, i41, descr=floatarraydescr) v63[2xf64] = vec_float_add(v61, v62) - vec_raw_store(i0, i37, v63, descr=floatarraydescr) + vec_store(i0, i37, v63, descr=floatarraydescr) f100 = vec_unpack_f(v61, 1, 1) f101 = vec_unpack_f(v62, 1, 1) jump(p36, i637, p9, i56, p14, f100, p12, p38, f101, p39, i40, i54, p42, i43, i55, i21, i4, i0, i18) @@ -1302,11 +1301,11 @@ i400 = int_add(i4, 16) i401= int_lt(i400, 100) i402 = int_add(i0, 16) - v228[4xi32] = vec_raw_load_i(p0, i0, descr=float32arraydescr) + v228[4xi32] = vec_load_i(p0, i0, descr=float32arraydescr) v229[2xf64] = vec_cast_singlefloat_to_float(v228) v230 = vec_unpack_i(v228, 2, 2) v231 = vec_cast_singlefloat_to_float(v230) - v232 = vec_raw_load_i(p1, i189, descr=float32arraydescr) + v232 = vec_load_i(p1, i189, descr=float32arraydescr) v233 = vec_cast_singlefloat_to_float(v232) v236 = vec_float_add(v229, v233) v238 = vec_cast_float_to_singlefloat(v236) @@ -1315,7 +1314,7 @@ v237 = vec_float_add(v231, v235) v239 = vec_cast_float_to_singlefloat(v237) v240 = vec_pack_i(v238, v239, 2, 2) - vec_raw_store(p2, i4, v240, descr=float32arraydescr) + vec_store(p2, i4, v240, descr=float32arraydescr) jump(p0, p1, p2, i207, i500) """) vopt = self.vectorize(trace) @@ -1355,7 +1354,7 @@ """) vopt = self.schedule(trace) self.ensure_operations([ - 'v10[2xf64] = vec_raw_load_f(p0,i0,descr=floatarraydescr)', + 'v10[2xf64] = vec_load_f(p0,i0,descr=floatarraydescr)', 'v11[2xf64] = vec_float_mul(v10[2xf64], v9[2xf64])', 'v12[2xf64] = vec_float_eq(v11[2xf64], v11[2xf64])', 'i100 = vec_unpack_f(v12[4xi32], 0, 1)', diff --git a/rpython/jit/metainterp/optimizeopt/vector.py b/rpython/jit/metainterp/optimizeopt/vector.py --- a/rpython/jit/metainterp/optimizeopt/vector.py +++ b/rpython/jit/metainterp/optimizeopt/vector.py @@ -216,6 +216,7 @@ def __init__(self, metainterp_sd, jitdriver_sd, cost_threshold): Optimizer.__init__(self, metainterp_sd, jitdriver_sd) self.cpu = metainterp_sd.cpu + self.vector_ext = self.cpu.vector_ext self.cost_threshold = cost_threshold self.packset = None self.unroll_count = 0 @@ -226,7 +227,7 @@ self.orig_label_args = loop.label.getarglist_copy() self.linear_find_smallest_type(loop) byte_count = self.smallest_type_bytes - vsize = self.cpu.vector_register_size + vsize = self.vector_ext.vec_size() if vsize == 0 or byte_count == 0 or loop.label.getopnum() != rop.LABEL: # stop, there is no chance to vectorize this trace # we cannot optimize normal traces (if there is no label) @@ -345,7 +346,7 @@ loop = graph.loop operations = loop.operations - self.packset = PackSet(self.cpu.vector_register_size) + self.packset = PackSet(self.vector_ext.vec_size()) memory_refs = graph.memory_refs.items() # initialize the pack set for node_a,memref_a in memory_refs: @@ -558,7 +559,7 @@ """ def __init__(self, cpu, threshold): self.threshold = threshold - self.vec_reg_size = cpu.vector_register_size + self.vec_reg_size = cpu.vector_ext.vec_size() self.savings = 0 def reset_savings(self): From pypy.commits at gmail.com Thu Jul 7 06:21:32 2016 From: pypy.commits at gmail.com (arigo) Date: Thu, 07 Jul 2016 03:21:32 -0700 (PDT) Subject: [pypy-commit] cffi default: Link to the issue's comment Message-ID: <577e2d2c.d11b1c0a.38cb1.057f@mx.google.com> Author: Armin Rigo Branch: Changeset: r2722:28b9b2f71a46 Date: 2016-07-07 12:23 +0200 http://bitbucket.org/cffi/cffi/changeset/28b9b2f71a46/ Log: Link to the issue's comment diff --git a/doc/source/cdef.rst b/doc/source/cdef.rst --- a/doc/source/cdef.rst +++ b/doc/source/cdef.rst @@ -506,7 +506,10 @@ ``long *``) or in other locations (e.g. a global array ``int a[5];`` must not be misdeclared ``long a[5];``). CFFI considers `all types listed above`_ as primitive (so ``long long a[5];`` and ``int64_t a[5]`` are -different declarations). +different declarations). The reason for that is detailed in `a comment +about an issue.`__ + +.. __: https://bitbucket.org/cffi/cffi/issues/265/cffi-doesnt-allow-creating-pointers-to#comment-28406958 ffibuilder.compile() etc.: compiling out-of-line modules From pypy.commits at gmail.com Thu Jul 7 07:57:55 2016 From: pypy.commits at gmail.com (plan_rich) Date: Thu, 07 Jul 2016 04:57:55 -0700 (PDT) Subject: [pypy-commit] pypy ppc-vsx-support: call mulld, codebuilder has no method mul Message-ID: <577e43c3.88c11c0a.1bc0c.0fb5@mx.google.com> Author: Richard Plangger Branch: ppc-vsx-support Changeset: r85596:6d2e011d1895 Date: 2016-07-07 13:57 +0200 http://bitbucket.org/pypy/pypy/changeset/6d2e011d1895/ Log: call mulld, codebuilder has no method mul diff --git a/rpython/jit/backend/ppc/vector_ext.py b/rpython/jit/backend/ppc/vector_ext.py --- a/rpython/jit/backend/ppc/vector_ext.py +++ b/rpython/jit/backend/ppc/vector_ext.py @@ -319,7 +319,7 @@ if op == '+': self.mc.add(tgt, tgt, acc) elif op == '*': - self.mc.mul(tgt, tgt, acc) + self.mc.mulld(tgt, tgt, acc) else: not_implemented("sum not implemented") return From pypy.commits at gmail.com Thu Jul 7 07:59:07 2016 From: pypy.commits at gmail.com (plan_rich) Date: Thu, 07 Jul 2016 04:59:07 -0700 (PDT) Subject: [pypy-commit] pypy ppc-vsx-support: merge default Message-ID: <577e440b.cc9d1c0a.19a0c.fffff2a0@mx.google.com> Author: Richard Plangger Branch: ppc-vsx-support Changeset: r85597:fb6cdfd34d1b Date: 2016-07-07 13:58 +0200 http://bitbucket.org/pypy/pypy/changeset/fb6cdfd34d1b/ Log: merge default diff --git a/pypy/module/_cffi_backend/ctypestruct.py b/pypy/module/_cffi_backend/ctypestruct.py --- a/pypy/module/_cffi_backend/ctypestruct.py +++ b/pypy/module/_cffi_backend/ctypestruct.py @@ -90,7 +90,7 @@ self.force_lazy_struct() space = self.space try: - cfield = self._fields_dict[fieldname] + cfield = self._getcfield_const(fieldname) except KeyError: raise OperationError(space.w_KeyError, space.wrap(fieldname)) if cfield.bitshift >= 0: diff --git a/pypy/module/cpyext/test/test_cpyext.py b/pypy/module/cpyext/test/test_cpyext.py --- a/pypy/module/cpyext/test/test_cpyext.py +++ b/pypy/module/cpyext/test/test_cpyext.py @@ -129,6 +129,7 @@ return str(soname) def freeze_refcnts(self): + rawrefcount._dont_free_any_more() return #ZZZ state = self.space.fromcache(RefcountState) self.frozen_refcounts = {} diff --git a/pypy/module/itertools/interp_itertools.py b/pypy/module/itertools/interp_itertools.py --- a/pypy/module/itertools/interp_itertools.py +++ b/pypy/module/itertools/interp_itertools.py @@ -835,7 +835,7 @@ data before the other iterator, it is faster to use list() instead of tee() - Equivalent to : + If iter(iterable) has no method __copy__(), this is equivalent to: def tee(iterable, n=2): def gen(next, data={}, cnt=[0]): @@ -848,17 +848,22 @@ yield item it = iter(iterable) return tuple([gen(it.next) for i in range(n)]) + + If iter(iterable) has a __copy__ method, though, we just return + a tuple t = (iterable, t[0].__copy__(), t[1].__copy__(), ...). """ if n < 0: raise oefmt(space.w_ValueError, "n must be >= 0") - if isinstance(w_iterable, W_TeeIterable): # optimization only - chained_list = w_iterable.chained_list - w_iterator = w_iterable.w_iterator + if space.findattr(w_iterable, space.wrap("__copy__")) is not None: + # In this case, we don't instantiate any W_TeeIterable. + # We just rely on doing repeated __copy__(). This case + # includes the situation where w_iterable is already + # a W_TeeIterable itself. iterators_w = [w_iterable] * n for i in range(1, n): - iterators_w[i] = space.wrap(W_TeeIterable(space, w_iterator, - chained_list)) + w_iterable = space.call_method(w_iterable, "__copy__") + iterators_w[i] = w_iterable else: w_iterator = space.iter(w_iterable) chained_list = TeeChainedListNode() @@ -891,6 +896,11 @@ self.chained_list = chained_list.next return w_obj + def copy_w(self): + space = self.space + tee_iter = W_TeeIterable(space, self.w_iterator, self.chained_list) + return space.wrap(tee_iter) + def W_TeeIterable___new__(space, w_subtype, w_iterable): # Obscure and undocumented function. PyPy only supports w_iterable # being a W_TeeIterable, because the case where it is a general @@ -906,6 +916,7 @@ __new__ = interp2app(W_TeeIterable___new__), __iter__ = interp2app(W_TeeIterable.iter_w), next = interp2app(W_TeeIterable.next_w), + __copy__ = interp2app(W_TeeIterable.copy_w), __weakref__ = make_weakref_descr(W_TeeIterable), ) W_TeeIterable.typedef.acceptable_as_base_class = False diff --git a/pypy/module/itertools/test/test_itertools.py b/pypy/module/itertools/test/test_itertools.py --- a/pypy/module/itertools/test/test_itertools.py +++ b/pypy/module/itertools/test/test_itertools.py @@ -702,6 +702,48 @@ x = d.next() assert x == 'b' + def test_tee_defines_copy(self): + import itertools + a, b = itertools.tee('abc') + c = b.__copy__() + assert list(a) == ['a', 'b', 'c'] + assert list(b) == ['a', 'b', 'c'] + assert list(c) == ['a', 'b', 'c'] + a, = itertools.tee('abc', 1) + x = a.next() + assert x == 'a' + b = a.__copy__() + x = a.next() + assert x == 'b' + x = b.next() + assert x == 'b' + + def test_tee_function_uses_copy(self): + import itertools + class MyIterator(object): + def __iter__(self): + return self + def next(self): + raise NotImplementedError + def __copy__(self): + return iter('def') + my = MyIterator() + a, = itertools.tee(my, 1) + assert a is my + a, b = itertools.tee(my) + assert a is my + assert b is not my + assert list(b) == ['d', 'e', 'f'] + # this gives AttributeError because it tries to call + # my.__copy__().__copy__() and there isn't one + raises(AttributeError, itertools.tee, my, 3) + + def test_tee_function_empty(self): + import itertools + assert itertools.tee('abc', 0) == () + a, = itertools.tee('abc', 1) + assert itertools.tee(a, 0) == () + class AppTestItertools26: spaceconfig = dict(usemodules=['itertools']) diff --git a/rpython/jit/backend/zarch/assembler.py b/rpython/jit/backend/zarch/assembler.py --- a/rpython/jit/backend/zarch/assembler.py +++ b/rpython/jit/backend/zarch/assembler.py @@ -317,7 +317,7 @@ self._push_fp_regs_to_jitframe(mc) - # First argument is SPP (= r31), which is the jitframe + # First argument is SPP, which is the jitframe mc.LGR(r.r2, r.SPP) # no need to move second argument (frame_depth), @@ -385,14 +385,12 @@ # signature of these cond_call_slowpath functions: # * on entry, r11 contains the function to call # * r2, r3, r4, r5 contain arguments for the call - # * r0 is the gcmap + # * gcmap is pushed # * the old value of these regs must already be stored in the jitframe # * on exit, all registers are restored from the jitframe mc = InstrBuilder() self.mc = mc - ofs2 = self.cpu.get_ofs_of_frame_field('jf_gcmap') - mc.STG(r.SCRATCH2, l.addr(ofs2,r.SPP)) mc.store_link() mc.push_std_frame() @@ -502,7 +500,6 @@ # r.RSZ is loaded from [r1], to make the caller's store a no-op here mc.load(r.RSZ, r.r1, 0) # - self.pop_gcmap(mc) # push_gcmap(store=True) done by the caller mc.restore_link() mc.BCR(c.ANY, r.r14) self.mc = None diff --git a/rpython/jit/backend/zarch/opassembler.py b/rpython/jit/backend/zarch/opassembler.py --- a/rpython/jit/backend/zarch/opassembler.py +++ b/rpython/jit/backend/zarch/opassembler.py @@ -387,8 +387,7 @@ if reg in self._COND_CALL_SAVE_REGS] self._push_core_regs_to_jitframe(self.mc, should_be_saved) - # load gc map into unusual location: r0 - self.load_gcmap(self.mc, r.SCRATCH2, regalloc.get_gcmap()) + self.push_gcmap(self.mc, regalloc.get_gcmap()) # # load the 0-to-4 arguments into these registers, with the address of # the function to call into r11 diff --git a/rpython/jit/backend/zarch/test/test_runner.py b/rpython/jit/backend/zarch/test/test_runner.py --- a/rpython/jit/backend/zarch/test/test_runner.py +++ b/rpython/jit/backend/zarch/test/test_runner.py @@ -26,4 +26,4 @@ add_loop_instructions = "lg; lgr; larl; agr; cgfi; jge; j;$" bridge_loop_instructions = "lg; cgfi; jnl; lghi; " \ - "(lgfi|iilf);( iihf;)? (lgfi|iilf);( iihf;)? basr; larl; (lgfi|iilf);( iihf;)? br;$" + "(lgfi|iilf);( iihf;)? (lgfi|iilf);( iihf;)? stg; basr; larl; (lgfi|iilf);( iihf;)? br;$" diff --git a/rpython/jit/metainterp/optimizeopt/intdiv.py b/rpython/jit/metainterp/optimizeopt/intdiv.py --- a/rpython/jit/metainterp/optimizeopt/intdiv.py +++ b/rpython/jit/metainterp/optimizeopt/intdiv.py @@ -1,5 +1,5 @@ from rpython.rlib.rarithmetic import LONG_BIT, intmask, r_uint -from rpython.rlib.rbigint import rbigint, ONERBIGINT + from rpython.jit.metainterp.history import ConstInt from rpython.jit.metainterp.resoperation import ResOperation, rop @@ -17,10 +17,19 @@ while (r_uint(1) << (i+1)) < r_uint(m): i += 1 - # k = 2**(64+i) // m + 1, computed manually using rbigint - # because that's the easiest - k1 = ONERBIGINT.lshift(LONG_BIT + i).floordiv(rbigint.fromint(m)) - k = k1.touint() + r_uint(1) + # quotient = 2**(64+i) // m + high_word_dividend = r_uint(1) << i + quotient = r_uint(0) + for bit in range(LONG_BIT-1, -1, -1): + t = quotient + (r_uint(1) << bit) + # check: is 't * m' small enough to be < 2**(64+i), or not? + # note that we're really computing (2**(64+i)-1) // m, but the result + # is the same, because powers of two are not multiples of m. + if unsigned_mul_high(t, r_uint(m)) < high_word_dividend: + quotient = t # yes, small enough + + # k = 2**(64+i) // m + 1 + k = quotient + r_uint(1) assert k != r_uint(0) # Proof that k < 2**64 holds in all cases, even with the "+1": From pypy.commits at gmail.com Thu Jul 7 08:17:05 2016 From: pypy.commits at gmail.com (Raemi) Date: Thu, 07 Jul 2016 05:17:05 -0700 (PDT) Subject: [pypy-commit] stmgc default: fix for previously committed test and do some more resetting Message-ID: <577e4841.46c21c0a.e7ac2.ffff8abd@mx.google.com> Author: Remi Meier Branch: Changeset: r1994:bfbab11bd95f Date: 2016-07-07 14:14 +0200 http://bitbucket.org/pypy/stmgc/changeset/bfbab11bd95f/ Log: fix for previously committed test and do some more resetting After an in-validation-reset of a noconflict obj, the obj was still marked as having been read before. Also, an obj with cards is not in old_objects_with_cards_set anymore after a collection. Thus we should clear all cards unconditionally during a reset. diff --git a/c8/stm/core.c b/c8/stm/core.c --- a/c8/stm/core.c +++ b/c8/stm/core.c @@ -233,7 +233,7 @@ for (; undo < end; undo++) { object_t *obj; - if (undo->type != TYPE_POSITION_MARKER) { + if (LIKELY(undo->type != TYPE_POSITION_MARKER)) { /* common case: 'undo->object' was written to in this past commit, so we must check that it was not read by us. */ @@ -263,7 +263,10 @@ an abort. However, from now on, we also assume that an abort would not roll-back to what is in the backup copy, as we don't trace the bkcpy - during major GCs. + during major GCs. (Seg0 may contain the version + found in the other segment and thus not have to + content of our bk_copy) + We choose the approach to reset all our changes to this obj here, so that we can throw away the backup copy completely: */ @@ -396,11 +399,15 @@ */ static void _validate_and_attach(struct stm_commit_log_entry_s *new) { + uintptr_t cle_length = 0; struct stm_commit_log_entry_s *old; OPT_ASSERT(new != NULL); OPT_ASSERT(new != INEV_RUNNING); + cle_length = list_count(STM_PSEGMENT->modified_old_objects); + assert(cle_length == new->written_count * 3); + soon_finished_or_inevitable_thread_segment(); retry_from_start: @@ -409,6 +416,16 @@ stm_abort_transaction(); } + if (cle_length != list_count(STM_PSEGMENT->modified_old_objects)) { + /* something changed the list of modified objs during _stm_validate; or + * during a major GC that also does _stm_validate(). That "something" + * can only be a reset of a noconflict obj. Thus, we recreate the CL + * entry */ + free_cle(new); + new = _create_commit_log_entry(); + cle_length = list_count(STM_PSEGMENT->modified_old_objects); + } + #if STM_TESTS if (STM_PSEGMENT->transaction_state != TS_INEVITABLE && STM_PSEGMENT->last_commit_log_entry->next == INEV_RUNNING) { @@ -607,6 +624,10 @@ size_t start_offset; if (first_call) { start_offset = 0; + + /* flags like a never-touched obj */ + assert(obj->stm_flags & GCFLAG_WRITE_BARRIER); + assert(!(obj->stm_flags & GCFLAG_WB_EXECUTED)); } else { start_offset = -1; } @@ -1288,10 +1309,21 @@ reset_modified_from_backup_copies(segment_num, obj); + /* reset read marker (must not be considered read either) */ + ((struct stm_read_marker_s *) + (pseg->pub.segment_base + (((uintptr_t)obj) >> 4)))->rm = 0; + + /* reset possibly marked cards */ + if (get_page_status_in(segment_num, (uintptr_t)obj / 4096) == PAGE_ACCESSIBLE + && obj_should_use_cards(pseg->pub.segment_base, obj)) { + /* if header is not accessible, we didn't mark any cards */ + _reset_object_cards(pseg, obj, CARD_CLEAR, false, false); + } + + /* remove from all other lists */ LIST_FOREACH_R(pseg->old_objects_with_cards_set, object_t * /*item*/, { if (item == obj) { - _reset_object_cards(pseg, item, CARD_CLEAR, false, false); /* copy last element over this one (HACK) */ _lst->count -= 1; _lst->items[_i] = _lst->items[_lst->count]; @@ -1330,7 +1362,7 @@ continue; object_t *obj = undo->object; - if (only_obj != NULL && obj != only_obj) + if (UNLIKELY(only_obj != NULL) && LIKELY(obj != only_obj)) continue; char *dst = REAL_ADDRESS(pseg->pub.segment_base, obj); @@ -1345,17 +1377,15 @@ free_bk(undo); - if (only_obj != NULL) { - /* WB_EXECUTED should be set on small objs, but not on card objs */ - assert(IMPLY(only_obj != NULL, - (((struct object_s *)dst)->stm_flags - & GCFLAG_NO_CONFLICT))); + if (UNLIKELY(only_obj != NULL)) { + assert(((struct object_s *)dst)->stm_flags & GCFLAG_NO_CONFLICT); + /* copy last element over this one */ end--; list->count -= 3; - if (undo < end) - *undo = *end; - undo--; /* next itr */ + *undo = *end; + /* to neutralise the increment for the next iter: */ + undo--; } } @@ -1664,13 +1694,13 @@ assert(STM_PSEGMENT->privatization_lock); assert(obj->stm_flags & GCFLAG_WRITE_BARRIER); assert(!(obj->stm_flags & GCFLAG_WB_EXECUTED)); + assert(!(obj->stm_flags & GCFLAG_CARDS_SET)); ssize_t obj_size = stmcb_size_rounded_up( (struct object_s *)REAL_ADDRESS(STM_SEGMENT->segment_base, obj)); OPT_ASSERT(obj_size >= 16); if (LIKELY(is_small_uniform(obj))) { - assert(!(obj->stm_flags & GCFLAG_CARDS_SET)); OPT_ASSERT(obj_size <= GC_LAST_SMALL_SIZE); _synchronize_fragment((stm_char *)obj, obj_size); return; diff --git a/c8/test/test_noconfl.py b/c8/test/test_noconfl.py --- a/c8/test/test_noconfl.py +++ b/c8/test/test_noconfl.py @@ -175,6 +175,8 @@ stm_set_char(o, 'a') stm_set_char(oh, 'x', use_cards=True) + assert stm_was_read(o) + assert stm_was_read(oh) assert o in modified_old_objects() assert oh in modified_old_objects() assert o in objects_pointing_to_nursery() @@ -189,6 +191,8 @@ self.commit_transaction() self.switch(0, False) + assert stm_was_read(o) + assert stm_was_read(oh) assert stm_get_char(o) == 'a' assert stm_get_char(oh) == 'x' assert o in modified_old_objects() @@ -197,6 +201,8 @@ assert oh not in objects_pointing_to_nursery() assert oh in old_objects_with_cards_set() stm_validate() + assert not stm_was_read(o) + assert not stm_was_read(oh) assert stm_get_char(o) == 'b' assert stm_get_char(oh) == 'y' assert o not in modified_old_objects() From pypy.commits at gmail.com Thu Jul 7 08:17:50 2016 From: pypy.commits at gmail.com (Raemi) Date: Thu, 07 Jul 2016 05:17:50 -0700 (PDT) Subject: [pypy-commit] pypy stmgc-c8: import stmgc with fixes for noconfl objs Message-ID: <577e486e.49a4c20a.87381.ffffc27a@mx.google.com> Author: Remi Meier Branch: stmgc-c8 Changeset: r85598:c458e253bab4 Date: 2016-07-07 14:16 +0200 http://bitbucket.org/pypy/pypy/changeset/c458e253bab4/ Log: import stmgc with fixes for noconfl objs diff --git a/rpython/translator/stm/src_stm/revision b/rpython/translator/stm/src_stm/revision --- a/rpython/translator/stm/src_stm/revision +++ b/rpython/translator/stm/src_stm/revision @@ -1,1 +1,1 @@ -5e3551b4e599 +bfbab11bd95f+ diff --git a/rpython/translator/stm/src_stm/stm/core.c b/rpython/translator/stm/src_stm/stm/core.c --- a/rpython/translator/stm/src_stm/stm/core.c +++ b/rpython/translator/stm/src_stm/stm/core.c @@ -151,6 +151,7 @@ } static void reset_modified_from_backup_copies(int segment_num, object_t *only_obj); /* forward */ +static void undo_modifications_to_single_obj(int segment_num, object_t *only_obj); /* forward */ static bool _stm_validate(void) { @@ -232,7 +233,7 @@ for (; undo < end; undo++) { object_t *obj; - if (undo->type != TYPE_POSITION_MARKER) { + if (LIKELY(undo->type != TYPE_POSITION_MARKER)) { /* common case: 'undo->object' was written to in this past commit, so we must check that it was not read by us. */ @@ -262,13 +263,17 @@ an abort. However, from now on, we also assume that an abort would not roll-back to what is in the backup copy, as we don't trace the bkcpy - during major GCs. + during major GCs. (Seg0 may contain the version + found in the other segment and thus not have to + content of our bk_copy) + We choose the approach to reset all our changes to this obj here, so that we can throw away the backup copy completely: */ /* XXX: this browses through the whole list of modified fragments; this may become a problem... */ - reset_modified_from_backup_copies(my_segnum, obj); + undo_modifications_to_single_obj(my_segnum, obj); + continue; } @@ -394,11 +399,15 @@ */ static void _validate_and_attach(struct stm_commit_log_entry_s *new) { + uintptr_t cle_length = 0; struct stm_commit_log_entry_s *old; OPT_ASSERT(new != NULL); OPT_ASSERT(new != INEV_RUNNING); + cle_length = list_count(STM_PSEGMENT->modified_old_objects); + assert(cle_length == new->written_count * 3); + soon_finished_or_inevitable_thread_segment(); retry_from_start: @@ -407,6 +416,16 @@ stm_abort_transaction(); } + if (cle_length != list_count(STM_PSEGMENT->modified_old_objects)) { + /* something changed the list of modified objs during _stm_validate; or + * during a major GC that also does _stm_validate(). That "something" + * can only be a reset of a noconflict obj. Thus, we recreate the CL + * entry */ + free_cle(new); + new = _create_commit_log_entry(); + cle_length = list_count(STM_PSEGMENT->modified_old_objects); + } + #if STM_TESTS if (STM_PSEGMENT->transaction_state != TS_INEVITABLE && STM_PSEGMENT->last_commit_log_entry->next == INEV_RUNNING) { @@ -605,6 +624,10 @@ size_t start_offset; if (first_call) { start_offset = 0; + + /* flags like a never-touched obj */ + assert(obj->stm_flags & GCFLAG_WRITE_BARRIER); + assert(!(obj->stm_flags & GCFLAG_WB_EXECUTED)); } else { start_offset = -1; } @@ -1276,6 +1299,48 @@ invoke_general_finalizers(tl); } +static void undo_modifications_to_single_obj(int segment_num, object_t *obj) +{ + /* special function used for noconflict objs to reset all their + * modifications and make them appear untouched in the current transaction. + * I.e., reset modifications and remove from all lists. */ + + struct stm_priv_segment_info_s *pseg = get_priv_segment(segment_num); + + reset_modified_from_backup_copies(segment_num, obj); + + /* reset read marker (must not be considered read either) */ + ((struct stm_read_marker_s *) + (pseg->pub.segment_base + (((uintptr_t)obj) >> 4)))->rm = 0; + + /* reset possibly marked cards */ + if (get_page_status_in(segment_num, (uintptr_t)obj / 4096) == PAGE_ACCESSIBLE + && obj_should_use_cards(pseg->pub.segment_base, obj)) { + /* if header is not accessible, we didn't mark any cards */ + _reset_object_cards(pseg, obj, CARD_CLEAR, false, false); + } + + /* remove from all other lists */ + LIST_FOREACH_R(pseg->old_objects_with_cards_set, object_t * /*item*/, + { + if (item == obj) { + /* copy last element over this one (HACK) */ + _lst->count -= 1; + _lst->items[_i] = _lst->items[_lst->count]; + break; + } + }); + LIST_FOREACH_R(pseg->objects_pointing_to_nursery, object_t * /*item*/, + { + if (item == obj) { + /* copy last element over this one (HACK) */ + _lst->count -= 1; + _lst->items[_i] = _lst->items[_lst->count]; + break; + } + }); +} + static void reset_modified_from_backup_copies(int segment_num, object_t *only_obj) { #pragma push_macro("STM_PSEGMENT") @@ -1284,6 +1349,9 @@ #undef STM_SEGMENT assert(modification_lock_check_wrlock(segment_num)); + /* WARNING: resetting the obj will remove the WB flag. Make sure you either + * re-add it or remove it from lists where it was added based on the flag. */ + struct stm_priv_segment_info_s *pseg = get_priv_segment(segment_num); struct list_s *list = pseg->modified_old_objects; struct stm_undo_s *undo = (struct stm_undo_s *)list->items; @@ -1294,7 +1362,7 @@ continue; object_t *obj = undo->object; - if (only_obj != NULL && obj != only_obj) + if (UNLIKELY(only_obj != NULL) && LIKELY(obj != only_obj)) continue; char *dst = REAL_ADDRESS(pseg->pub.segment_base, obj); @@ -1309,19 +1377,15 @@ free_bk(undo); - if (only_obj != NULL) { - assert(IMPLY(only_obj != NULL, - (((struct object_s *)dst)->stm_flags - & (GCFLAG_NO_CONFLICT - | GCFLAG_WRITE_BARRIER - | GCFLAG_WB_EXECUTED)) - == (GCFLAG_NO_CONFLICT | GCFLAG_WRITE_BARRIER))); + if (UNLIKELY(only_obj != NULL)) { + assert(((struct object_s *)dst)->stm_flags & GCFLAG_NO_CONFLICT); + /* copy last element over this one */ end--; list->count -= 3; - if (undo < end) - *undo = *end; - undo--; /* next itr */ + *undo = *end; + /* to neutralise the increment for the next iter: */ + undo--; } } @@ -1630,13 +1694,13 @@ assert(STM_PSEGMENT->privatization_lock); assert(obj->stm_flags & GCFLAG_WRITE_BARRIER); assert(!(obj->stm_flags & GCFLAG_WB_EXECUTED)); + assert(!(obj->stm_flags & GCFLAG_CARDS_SET)); ssize_t obj_size = stmcb_size_rounded_up( (struct object_s *)REAL_ADDRESS(STM_SEGMENT->segment_base, obj)); OPT_ASSERT(obj_size >= 16); if (LIKELY(is_small_uniform(obj))) { - assert(!(obj->stm_flags & GCFLAG_CARDS_SET)); OPT_ASSERT(obj_size <= GC_LAST_SMALL_SIZE); _synchronize_fragment((stm_char *)obj, obj_size); return; diff --git a/rpython/translator/stm/src_stm/stm/nursery.c b/rpython/translator/stm/src_stm/stm/nursery.c --- a/rpython/translator/stm/src_stm/stm/nursery.c +++ b/rpython/translator/stm/src_stm/stm/nursery.c @@ -118,6 +118,9 @@ char *realnobj = REAL_ADDRESS(STM_SEGMENT->segment_base, nobj); memcpy(realnobj, realobj, size); + /* uint64_t ttt = 0xa0a0a0a0a0a0a0a0; */ + /* assert(memmem(realobj, size, &ttt, sizeof(uint64_t)) == NULL); */ + nobj_sync_now = ((uintptr_t)nobj) | FLAG_SYNC_LARGE; pforwarded_array[0] = GCWORD_MOVED; diff --git a/rpython/translator/stm/src_stm/stmgc.h b/rpython/translator/stm/src_stm/stmgc.h --- a/rpython/translator/stm/src_stm/stmgc.h +++ b/rpython/translator/stm/src_stm/stmgc.h @@ -205,7 +205,7 @@ threads than the number of segments, it will block, waiting for the next segment to become free. */ -#define STM_NB_SEGMENTS 4 +#define STM_NB_SEGMENTS 8 /* Structure of objects -------------------- From pypy.commits at gmail.com Thu Jul 7 11:15:09 2016 From: pypy.commits at gmail.com (mattip) Date: Thu, 07 Jul 2016 08:15:09 -0700 (PDT) Subject: [pypy-commit] pypy default: fix tests for win32 Message-ID: <577e71fd.e85dc20a.504c7.6047@mx.google.com> Author: Matti Picus Branch: Changeset: r85599:8ab2ec92dcfc Date: 2016-07-07 18:11 +0300 http://bitbucket.org/pypy/pypy/changeset/8ab2ec92dcfc/ Log: fix tests for win32 diff --git a/pypy/module/cpyext/test/foo.c b/pypy/module/cpyext/test/foo.c --- a/pypy/module/cpyext/test/foo.c +++ b/pypy/module/cpyext/test/foo.c @@ -747,6 +747,7 @@ if (PyType_Ready(&UnicodeSubtype3) < 0) INITERROR; + TupleLike.tp_flags = Py_TPFLAGS_DEFAULT; TupleLike.tp_base = &PyTuple_Type; if (PyType_Ready(&TupleLike) < 0) INITERROR; diff --git a/pypy/module/cpyext/test/test_bytesobject.py b/pypy/module/cpyext/test/test_bytesobject.py --- a/pypy/module/cpyext/test/test_bytesobject.py +++ b/pypy/module/cpyext/test/test_bytesobject.py @@ -113,9 +113,10 @@ module = self.import_extension('foo', [ ("getbytes", "METH_NOARGS", """ - PyObject* s1 = PyBytes_FromStringAndSize("test", 4); - char* c = PyBytes_AsString(s1); - PyObject* s2 = PyBytes_FromStringAndSize(c, 4); + char *c; + PyObject* s2, *s1 = PyBytes_FromStringAndSize("test", 4); + c = PyBytes_AsString(s1); + s2 = PyBytes_FromStringAndSize(c, 4); Py_DECREF(s1); return s2; """), From pypy.commits at gmail.com Thu Jul 7 12:20:17 2016 From: pypy.commits at gmail.com (raffael_t) Date: Thu, 07 Jul 2016 09:20:17 -0700 (PDT) Subject: [pypy-commit] pypy py3.5-async: Add stack effect for async opcodes Message-ID: <577e8141.c1941c0a.e2fe0.ffff9e00@mx.google.com> Author: Raffael Tfirst Branch: py3.5-async Changeset: r85600:d00cb920f5c1 Date: 2016-07-07 16:40 +0200 http://bitbucket.org/pypy/pypy/changeset/d00cb920f5c1/ Log: Add stack effect for async opcodes diff --git a/pypy/interpreter/astcompiler/assemble.py b/pypy/interpreter/astcompiler/assemble.py --- a/pypy/interpreter/astcompiler/assemble.py +++ b/pypy/interpreter/astcompiler/assemble.py @@ -599,7 +599,7 @@ ops.PRINT_EXPR: -1, - ops.WITH_CLEANUP_START: -1, + ops.WITH_CLEANUP_START: 1, ops.WITH_CLEANUP_FINISH: -1, # XXX Sometimes more ops.LOAD_BUILD_CLASS: 1, ops.POP_BLOCK: 0, @@ -640,6 +640,12 @@ ops.LOAD_DEREF: 1, ops.STORE_DEREF: -1, ops.DELETE_DEREF: 0, + + ops.GET_AWAITABLE: 0, + ops.SETUP_ASYNC_WITH: 6, + ops.BEFORE_ASYNC_WITH: 1, + ops.GET_AITER: 0, + ops.GET_ANEXT: 1, ops.LOAD_CONST: 1, From pypy.commits at gmail.com Thu Jul 7 12:20:18 2016 From: pypy.commits at gmail.com (raffael_t) Date: Thu, 07 Jul 2016 09:20:18 -0700 (PDT) Subject: [pypy-commit] pypy py3.5-async: Implement visit_Await in codegen Message-ID: <577e8142.47c9c20a.84d0f.fffffcb4@mx.google.com> Author: Raffael Tfirst Branch: py3.5-async Changeset: r85601:7a4cdd986ad0 Date: 2016-07-07 18:03 +0200 http://bitbucket.org/pypy/pypy/changeset/7a4cdd986ad0/ Log: Implement visit_Await in codegen diff --git a/pypy/interpreter/astcompiler/codegen.py b/pypy/interpreter/astcompiler/codegen.py --- a/pypy/interpreter/astcompiler/codegen.py +++ b/pypy/interpreter/astcompiler/codegen.py @@ -909,6 +909,13 @@ self.emit_op(ops.GET_ITER) self.load_const(self.space.w_None) self.emit_op(ops.YIELD_FROM) + + def visit_Await(self, aw): + self.update_position(aw.lineno) + aw.value.walkabout(self) + self.emit_op(ops.GET_AWAITABLE) + self.load_const(self.space.w_None) + self.emit_op(ops.YIELD_FROM) def visit_Num(self, num): self.update_position(num.lineno) From pypy.commits at gmail.com Thu Jul 7 12:20:20 2016 From: pypy.commits at gmail.com (raffael_t) Date: Thu, 07 Jul 2016 09:20:20 -0700 (PDT) Subject: [pypy-commit] pypy py3.5-async: Implement visit_AsyncWith in codegen Message-ID: <577e8144.83261c0a.8d9aa.ffffe35d@mx.google.com> Author: Raffael Tfirst Branch: py3.5-async Changeset: r85602:22be1589e771 Date: 2016-07-07 18:16 +0200 http://bitbucket.org/pypy/pypy/changeset/22be1589e771/ Log: Implement visit_AsyncWith in codegen diff --git a/pypy/interpreter/astcompiler/codegen.py b/pypy/interpreter/astcompiler/codegen.py --- a/pypy/interpreter/astcompiler/codegen.py +++ b/pypy/interpreter/astcompiler/codegen.py @@ -863,6 +863,44 @@ self.emit_op(ops.END_FINALLY) self.pop_frame_block(F_BLOCK_FINALLY_END, cleanup) + def visit_AsyncWith(self, wih): + self.update_position(wih.lineno, True) + self.handle_asyncwithitem(wih, 0) + + def handle_asyncwithitem(self, wih, pos): + body_block = self.new_block() + cleanup = self.new_block() + witem = wih.items[pos] + witem.context_expr.walkabout(self) + self.emit_op(ops.BEFORE_ASYNC_WITH) + self.emit_op(ops.GET_AWAITABLE) + self.load_const(self.space.w_None) + self.emit_op(ops.YIELD_FROM) + self.emit_jump(ops.SETUP_ASYNC_WITH, cleanup) + self.use_next_block(body_block) + self.push_frame_block(F_BLOCK_FINALLY, body_block) + if witem.optional_vars: + witem.optional_vars.walkabout(self) + else: + self.emit_op(ops.POP_TOP) + if pos == len(wih.items) - 1: + self.visit_sequence(wih.body) + else: + self.handle_withitem(wih, pos + 1) + self.emit_op(ops.POP_BLOCK) + self.pop_frame_block(F_BLOCK_FINALLY, body_block) + self.load_const(self.space.w_None) + self.use_next_block(cleanup) + self.push_frame_block(F_BLOCK_FINALLY_END, cleanup) + self.emit_op(ops.WITH_CLEANUP_START) + self.emit_op(ops.GET_AWAITABLE) + self.load_const(self.space.w_None) + self.emit_op(ops.YIELD_FROM) + self.emit_op(ops.WITH_CLEANUP_FINISH) + self.emit_op(ops.END_FINALLY) + self.pop_frame_block(F_BLOCK_FINALLY_END, cleanup) + + def visit_Raise(self, rais): self.update_position(rais.lineno, True) arg = 0 From pypy.commits at gmail.com Thu Jul 7 12:20:22 2016 From: pypy.commits at gmail.com (raffael_t) Date: Thu, 07 Jul 2016 09:20:22 -0700 (PDT) Subject: [pypy-commit] pypy py3.5-async: Correct function call Message-ID: <577e8146.c5461c0a.7ed19.7b07@mx.google.com> Author: Raffael Tfirst Branch: py3.5-async Changeset: r85603:fc539047f9a8 Date: 2016-07-07 18:19 +0200 http://bitbucket.org/pypy/pypy/changeset/fc539047f9a8/ Log: Correct function call diff --git a/pypy/interpreter/astcompiler/codegen.py b/pypy/interpreter/astcompiler/codegen.py --- a/pypy/interpreter/astcompiler/codegen.py +++ b/pypy/interpreter/astcompiler/codegen.py @@ -886,7 +886,7 @@ if pos == len(wih.items) - 1: self.visit_sequence(wih.body) else: - self.handle_withitem(wih, pos + 1) + self.handle_asyncwithitem(wih, pos + 1) self.emit_op(ops.POP_BLOCK) self.pop_frame_block(F_BLOCK_FINALLY, body_block) self.load_const(self.space.w_None) From pypy.commits at gmail.com Thu Jul 7 15:40:10 2016 From: pypy.commits at gmail.com (raffael_t) Date: Thu, 07 Jul 2016 12:40:10 -0700 (PDT) Subject: [pypy-commit] pypy py3.5-async: Implement visit_AsyncFor in codegen Message-ID: <577eb01a.e5acc20a.d9c9c.497a@mx.google.com> Author: Raffael Tfirst Branch: py3.5-async Changeset: r85604:d67ac6d8fc48 Date: 2016-07-07 21:39 +0200 http://bitbucket.org/pypy/pypy/changeset/d67ac6d8fc48/ Log: Implement visit_AsyncFor in codegen diff --git a/pypy/interpreter/astcompiler/codegen.py b/pypy/interpreter/astcompiler/codegen.py --- a/pypy/interpreter/astcompiler/codegen.py +++ b/pypy/interpreter/astcompiler/codegen.py @@ -565,6 +565,60 @@ self.pop_frame_block(F_BLOCK_LOOP, start) self.visit_sequence(fr.orelse) self.use_next_block(end) + + def visit_AsyncFor(self, fr): + self.update_position(fr.lineno, True) + b_try = self.new_block() + b_except = self.new_block() + b_end = self.new_block() + b_after_try = self.new_block() + b_try_cleanup = self.new_block() + b_after_loop = self.new_block() + b_after_loop_else = self.new_block() + self.emit_jump(ops.SETUP_LOOP, b_after_loop) + self.push_frame_block(F_BLOCK_LOOP, b_try) + fr.iter.walkabout(self) + self.emit_op(ops.GET_AITER) + self.load_const(self.space.w_None) + self.emit_op(ops.YIELD_FROM) + self.use_next_block(b_try) + # This adds another line, so each for iteration can be traced. + self.lineno_set = False + self.emit_jump(ops.SETUP_EXCEPT, b_except) + self.push_frame_block(F_BLOCK_EXCEPT, b_try) + self.emit_op(ops.GET_ANEXT) + self.load_const(self.space.w_None) + self.emit_op(ops.YIELD_FROM) + fr.target.walkabout(self) + self.emit_op(ops.POP_BLOCK) + self.pop_frame_block(F_BLOCK_EXCEPT, b_try) + self.emit_jump(ops.JUMP_FORWARD, b_after_try) + self.use_next_block(b_except) + self.emit_op(ops.POP_TOP) + self.emit_op_name(ops.LOAD_GLOBAL, self.names, "StopIterError") + self.emit_op_arg(ops.COMPARE_OP, 10) + self.emit_jump(ops.POP_JUMP_IF_FALSE, b_try_cleanup, True) + + self.emit_op(ops.POP_TOP) + self.emit_op(ops.POP_TOP) + self.emit_op(ops.POP_TOP) + self.emit_op(ops.POP_EXCEPT) # for SETUP_EXCEPT + self.emit_op(ops.POP_BLOCK) # for SETUP_LOOP + self.emit_jump(ops.JUMP_ABSOLUTE, b_after_loop_else, True) + + self.use_next_block(b_try_cleanup) + self.emit_op(ops.END_FINALLY) + self.use_next_block(b_after_try) + self.visit_sequence(fr.body) + self.emit_jump(ops.JUMP_ABSOLUTE, b_try, True) + self.emit_op(ops.POP_BLOCK) # for SETUP_LOOP + self.pop_frame_block(F_BLOCK_LOOP, b_try) + self.use_next_block(b_after_loop) + self.emit_jump(ops.JUMP_ABSOLUTE, b_end, True) + self.use_next_block(b_after_loop_else) + self.visit_sequence(fr.orelse) + + self.use_next_block(b_end) def visit_While(self, wh): self.update_position(wh.lineno, True) From pypy.commits at gmail.com Fri Jul 8 05:15:35 2016 From: pypy.commits at gmail.com (arigo) Date: Fri, 08 Jul 2016 02:15:35 -0700 (PDT) Subject: [pypy-commit] extradoc extradoc: Extensive rewrite. Message-ID: <577f6f37.aa6ec20a.90fb5.fffff547@mx.google.com> Author: Armin Rigo Branch: extradoc Changeset: r5647:6273113ca565 Date: 2016-07-08 11:17 +0200 http://bitbucket.org/pypy/extradoc/changeset/6273113ca565/ Log: Extensive rewrite. diff --git a/blog/draft/revdb.rst b/blog/draft/revdb.rst --- a/blog/draft/revdb.rst +++ b/blog/draft/revdb.rst @@ -1,4 +1,397 @@ -Hi all, +============================ +Reverse debugging for Python +============================ + +RevPDB +------ + +A "reverse debugger" is a debugger where you can go forward and +backward in time. It is an uncommon feature, at least in the open +source world, but I have no idea why. I have used `undodb-gdb`_ and +`rr`_, which are reverse debugger for C code, and I can only say that +it saved me many, many days of poking around blindly in gdb. + +The PyPy team is pleased to give you "RevPDB", a reverse-debugger +similar to ``rr`` but for Python. + +An example is worth a thousand words. Let's say your big Python +program has a bug that shows up inconsistently. You have nailed it +down to something like: + +* start ``x.py``, which does stuff (maybe involving processing files, + answering some web requests that you simulate from another terminal, + etc.); + +* sometimes, after a few minutes, your program's state becomes + inconsistent and you get a failing assert or another exception. + +This is the case where RevPDB is useful. + +RevPDB is only available only on 64-bit Linux right now, but should +not be too hard to port to other OSes. It is very much *alpha-level!* +(It is a debugger full of bugs. Sorry about that.) I believe it is +still useful---it helped me in one `real use case`_ already. + +.. _`real use case`: https://bitbucket.org/pypy/pypy/commits/bd220c268bc9 + + +How to get RevPDB +----------------- + +The following demo was done with an alpha version for 64-bit Linux, +compiled for Arch Linux. I won't provide the binary; it should be +easy enough to retranslate (much faster than a regular PyPy because it +contains neither a JIT nor a custom GC). Grab the `PyPy sources`_ from +Mercurial, and then:: + + hg update reverse-debugger # this demo done with "hg update 4d82621df5ed" + cd pypy/goal + ../../rpython/bin/rpython -O2 --revdb targetpypystandalone.py --withoutmod-cpyext --withoutmod-micronumpy + +and possibly rename the final ``pypy-c`` to ``pypy-revdb`` to avoid +confusion. + +Other platforms than 64-bit Linux need some fixes before they work. + +.. _`PyPy sources`: http://pypy.org/download.html#building-from-source + + +Demo +---- + +For this demo, we're going to use this ``x.py`` as the "big program":: + + import os + + class Foo(object): + value = 5 + + lst1 = [Foo() for i in range(100)] + lst1[50].value += 1 + for x in lst1: + x.value += 1 + + for x in lst1: + if x.value != 6: + print 'oops!' + os._exit(1) + +Of course, it is clear what occurs in this small example: the check +fails on item 50. For this demo, the check has been written with +``os._exit(1)``, because this exits immediately the program. If it +was written with an ``assert``, then its failure would execute things +in the ``traceback`` module afterwards, to print the traceback; it +would be a minor mess just to find the exact point of the failing +``assert``. (This and other issues are supposed to be fixed in the +future, but for now it is alpha-level.) + +Anyway, with a regular ``assert`` and a regular post-mortem ``pdb``, +we could observe that ``x.value`` is indeed 7 instead of 6 when the +assert fails. Imagine that the program is much bigger: how would we +find the exact chain of events that caused this value 7 to show up on +this particular ``Foo`` object? This is what RevPDB is for. + +First, we need for now to disable Address Space Layout Randomization +(ASLR), otherwise replaying will not work. This is done once with the +following command line, which changes the state until the next +reboot:: + + echo 0 | sudo tee /proc/sys/kernel/randomize_va_space + +Run ``x.py`` with RevPDB's version of PyPy instead of the regular +interpreter (CPython or PyPy):: + + PYPYRDB=log.rdb ./pypy-revdb x.py + +This ``pypy-revdb`` executable is like a slow PyPy executable, running +(for now) without a JIT. This produces a file ``log.rdb`` which +contains a complete log of this execution. (If the bug we are +tracking occurs rarely, we need to re-run it several times until we +get the failure. But once we got the failure, then we're done with +this step.) + +Start ``rpython/translator/revdb/revdb.py log.rdb``. We get a +pdb-style debugger. Initially, we are at the start of the program +(not at the end, like we'd get in a regular debugger):: + + File "/app_main.py", line 787 in setup_bootstrap_path: + (1)$ + +The list of commands is available with ``help``. + +Go to the end with ``continue`` (or ``c``):: + + File "/tmp/x.py", line 14 in : + ... + lst1 = [Foo() for i in range(100)] + lst1[50].value += 1 + for x in lst1: + x.value += 1 + + for x in lst1: + if x.value != 6: + print 'oops!' + > os._exit(1) + (19727)$ + +We are now at the beginning of the last executed line. The number +19727 is the "time", measured in number of lines executed. We can go +backward with the ``bstep`` command (backward step, or ``bs``), line +by line, and forward again with the ``step`` command. There are also +commands ``bnext``, ``bcontinue`` and ``bfinish`` and their forward +equivalents. There is also ``go TIME`` to jump directly to the specified +time. (Right now the debugger only stops at "line start" +events, not at function entry or exit, which makes some cases a bit +surprizing: for example, a ``step`` from the return statement of +function ``foo()`` will jump directly to the caller's caller, if the +caller's current line was ``return foo() + 2``, because no "line +start" event occurs in the caller after ``foo()`` returns to it.) + +We can print Python expressions and statements using the ``p`` +command:: + + (19727)$ p x + $0 = <__main__.Foo object at 0xfffffffffffeab3e> + (19727)$ p x.value + $1 = 7 + (19727)$ p x.value + 1 + 8 + +The "``$NUM =``" prefix is only shown when we print an object that +really exists in the debugged program; that's why the last line does +not contain it. Once a ``$NUM`` has been printed, then we can use +it in further expressions---even at a different point time. It +becomes an anchor that always refers to the same object:: + + (19727)$ bstep + + File "/tmp/x.py", line 13 in : + ... + + lst1 = [Foo() for i in range(100)] + lst1[50].value += 1 + for x in lst1: + x.value += 1 + + for x in lst1: + if x.value != 6: + > print 'oops!' + os._exit(1) + (19726)$ p $0.value + $1 = 7 + +In this case, we want to know when this value 7 was put in this +attribute. This is the job of a watchpoint:: + + (19726)$ watch $0.value + Watchpoint 1 added + updating watchpoint value: $0.value => 7 + +This watchpoint means that ``$0.value`` will be evaluated at each line. +When the ``repr()`` of this expression changes, the watchpoint activates +and execution stops:: + + (19726)$ bcontinue + [searching 19629..19726] + [searching 19338..19629] + + updating watchpoint value: $0.value => 6 + Reverse-hit watchpoint 1: $0.value + File "/tmp/x.py", line 9 in : + import os + + class Foo(object): + value = 5 + + lst1 = [Foo() for i in range(100)] + lst1[50].value += 1 + for x in lst1: + > x.value += 1 + + for x in lst1: + if x.value != 6: + print 'oops!' + os._exit(1) + (19524)$ + +Note that using the ``$NUM`` syntax is essential in watchpoints. You +can't say "``watch x.value``", because the variable ``x`` will go out +of scope very soon when we move forward or backward in time. In fact +the watchpoint expression is always evaluated inside an environment +that contains the builtins but not the current locals and globals. +But it also contains all the ``$NUM``, which can be used to refer to +known objects. It is thus common to watch ``$0.attribute`` if ``$0`` +is an object, or to watch ``len($1)`` if ``$1`` is some list. The +watch expression can also be a simple boolean: for example, "``watch +$2 in $3``" where ``$3`` is some dict and ``$2`` is some object that +you find now in the dict; you would use this to find out the time when +``$2`` was put inside ``$3``, or removed from it. + +Use "``info watchpoints``" and "``delete ``" to manage +watchpoints. + +There are also regular breakpoints, which you set with "``b +FUNCNAME``". It breaks whenever there is a call to a function that +happens to have the given name. (It might be annoying to use for a +function like ``__init__()`` which has many homonyms. There is no +support for breaking on a fully-qualified name or at a given line +number for now.) + +In our demo, we stop at the line ``x.value += 1``, which is where the +value was changed from 6 to 7. Use ``bcontinue`` again to stop at the +line ``lst1[50].value += 1``, which is where the value was changed from +5 to 6. Now we know how this ``value`` attribute ends up being 7. + +:: + + (19524)$ bcontinue + [searching 19427..19524] + [searching 19136..19427] + + updating watchpoint value: $0.value => 5 + Reverse-hit watchpoint 1: $0.value + File "/tmp/x.py", line 7 in : + import os + + class Foo(object): + value = 5 + + lst1 = [Foo() for i in range(100)] + > lst1[50].value += 1 + for x in lst1: + x.value += 1 + + for x in lst1: + if x.value != 6: + ... + (19422)$ + +Try to use ``bcontinue`` yet another time. It will stop now just before +``$0`` is created. At that point in time, ``$0`` refers to +an object that does not exist yet, so the watchpoint now evaluates to +an error message (but it continues to work as before, with that error +message as the string it currently evaluates to). + +:: + + (19422)$ bcontinue + [searching 19325..19422] + + updating watchpoint value: $0.value => RuntimeError: '$0' refers to an object created later in time + Reverse-hit watchpoint 1: $0.value + File "/tmp/x.py", line 6 in : + import os + + class Foo(object): + value = 5 + + > lst1 = [Foo() for i in range(100)] + lst1[50].value += 1 + for x in lst1: + x.value += 1 + + for x in lst1: + ... + (19371)$ + +In big programs, the workflow is similar, just more complex. Usually +it works this way: we find interesting points in time with some +combination of watchpoints and some direct commands to move around. +We write down on a piece of (real or virtual) paper these points in +history, including most importantly their time, so that we can +construct an ordered understanding of what is going on. + +The current ``revdb`` can be annoying and sometimes even crash; but +the history you reconstruct can be kept. All the times and +expressions printed are still valid when you restart ``revdb``. The +only thing "lost" is the ``$NUM`` objects, which you need to print +again. (Maybe instead of ``$0``, ``$1``, ... we should use ``$``, where the big number identifies uniquely the object by its +creation time. These numbers would continue to be valid even after +``revdb`` is restarted. They are more annoying to use than just +``$0`` though.) + + +Current issues +-------------- + +General issues: + +* If you are using ``revdb`` on a log that took more than a few + minutes to record, then it can be painfully slow. This is because + ``revdb`` needs to replay again big parts of the log for some + operations. + +* The ``pypy-revdb`` is currently missing the following modules: + + - ``thread`` (implementing multithreading is possible, but not done + yet); + + - ``cpyext`` (the CPython C API compatibility layer); + + - ``micronumpy`` (minor issue only); + + - ``_continuation`` (for greenlets). + +* Does not contain a JIT, and does not use our fast garbage + collectors. You can expect ``pypy-revdb`` to be maybe 3 times + slower than CPython. + +* Only works on Linux, and only with Address Space Layout + Randomization (ASLR) disabled. There is no fundamental reason for + either restriction, but it is some work to fix. + +* Replaying a program uses a *lot* more memory; maybe 15x as much than + during the recording. This is because it creates many forks. If + you have a program that consumes 10% of your RAM or more, you will + need to reduce ``MAX_SUBPROCESSES`` in ``process.py``. + +Replaying also comes with a bunch of user interface issues: + +- ``Attempted to do I/O or access raw memory``: we get this whenever + trying to ``print`` some expression that cannot be evaluated with + only the GC memory---or which can, but then the ``__repr__()`` + method of the result cannot. We need to reset the state with + ``bstep`` + ``step`` before we can print anything else. However, + if only the ``__repr__()`` crashes, you still see the ``$NUM =`` + prefix, and you can use that ``$NUM`` afterwards. + +- ``id()`` is globally unique, returning a reproducible 64-bit number, + so sometimes using ``id(x)`` is a workaround for when using ``x`` + doesn't work because of ``Attempted to do I/O`` issues (e.g. ``p + [id(x) for x in somelist]``). + +- as explained in the demo, next/bnext/finish/bfinish might jump + around a bit non-predictably. + +- similarly, breaks on watchpoints can stop at apparently unexpected + places (when going backward, try to do "step" once). The issue is + that it can only stop at the beginning of every line. In the + extreme example, if a line is ``foo(somelist.pop(getindex()))``, + then ``somelist`` is modified in the middle. Immediately before + this modification occurs, we are in ``getindex()``, and + immediately afterwards we are in ``foo()``. The watchpoint will + stop the program at the end of ``getindex()`` if running backward, + and at the start of ``foo()`` if running forward, but never + actually on the line doing the change. + +- watchpoint expressions *must not* have any side-effect at all. If + they do, the replaying will get out of sync and ``revdb.py`` will + complain about that. Regular ``p`` expressions and statements can + have side-effects; these effects are discarded as soon as you move + in time again. + +- sometimes even ``p import foo`` will fail with ``Attempted to do + I/O``. Use instead ``p import sys; foo = sys.modules['foo']``. + +- use ``help`` to see all commands. ``backtrace`` can be useful. + There is no ``up`` command; you have to move in time instead, + e.g. using ``bfinish`` to go back to the point where the current + function was called. + + +How RevPDB is done +------------------ If I had to pick the main advantage of PyPy over CPython, it is that we have got with the RPython translation toolchain a real place for @@ -11,138 +404,78 @@ so-called "stackless transformation" in the early days, to the STM version of PyPy. -Today I would like to present you with last month's work (still very -much in alpha state). It is a RPython transformation that gives -support for a *reverse debugger* in PyPy or in any other interpreter -written in RPython. +RevPDB works in a similar way. It is a version of PyPy in which some +operations are systematically replaced with other operations. +To keep the log file at a reasonable size, we duplicate the content of +all GC objects during replaying---by repeating the same actions on +them, without writing anything in the log file. So that means that in +the ``pypy-revdb`` binary, the operations that do arithmetic or +read/write GC-managed memory are not modified. Most operations are +like that. However, the other operations, the ones that involve +either non-GC memory or calls to external C functions, are tweaked. +Each of these operations is replaced with code that works in two +modes, based on a global flag: -Reverse debugging ------------------ +* in "recording" mode, we log the result of the operation (but not the + arguments); -What is `reverse debugging`__? It is a debugger where you can go -forward and backward in time. It is still a not commonly used -feature, and I have no idea why not. I have used UndoDB's reverse -debugger for C code, and I can only say that it saved me many, many -days of poking around blindly in gdb. +* in "replaying" mode, we don't really do the operation at all, but + instead just fetch the result from the log. -.. __: https://en.wikipedia.org/wiki/Debugger#Reverse_debugging +Hopefully, all remaining unmodified operations (arithmetic and GC +load/store) are completely deterministic. So during replaying, every +integer or non-GC pointer variable will have exactly the same value as +it had during recording. Interestingly, it means that if the +recording process had a big array in non-GC memory, then in the +replaying process, the array is not allocated at all; it is just +represented by the same address, but there is nothing there. When we +record "read item 123 from the array", we record the result of the +read (but not the "123"). When we replay, we're seeing again the same +"read item 123 from the array" operation. At that point, we don't +read anything; we just return the result from the log. Similarly, +when recording a "write" to the array, we record nothing (this write +operation has no result); so that when replaying, we redo nothing. -There are already some Python experiments about reverse debugging. -However, I claim that they are not very useful. How they work is -typically by recording changes to some objects, like lists and -dictionaries, in addition to recording the history of where your -program passed through. However, the problem of Python is, again, -that lists and dictionaries are not the end of the story. There are -many, many, many types of objects written in C which are mutable---in -fact, the immutable ones are the exception. You can try to -systematically record all changes, but it is a huge task and easy to -forget a detail. +Note how that differs from anything managed by GC memory: GC objects +(including GC arrays) are really allocated, writes really occur, and +reads are redone. We don't touch the log in this case. + + +Other reverse debuggers for Python +---------------------------------- + +There are already some Python experiments about `reverse debugging`_. +This is also known as "omniscient debugging". However, I claim that +the result they get to is not very useful (for the purpose presented +here). How they work is typically by recording changes to some +objects, like lists and dictionaries, in addition to recording the +history of where your program passed through. However, the problem of +Python is that lists and dictionaries are not the end of the story. +There are many, many, many types of objects written in C which are +mutable---in fact, the immutable ones are the exception. You can try +to systematically record all changes, but it is a huge task and easy +to forget a detail. In other words it is a typical use case for tweaking the RPython -translation toolchain rather than the CPython or PyPy interpreter -directly. +translation toolchain, rather than tweaking the CPython (or PyPy) +interpreter directly. The result that we get here with RevPDB is more +similar to `rr`_ anyway, in that only a relatively small number of +external events are recorded---not every single change to every single +list and dictionary. +Some links: -RevDB in PyPy -------------- +* epdb: https://github.com/native-human/epdb -Right now, RevDB works barely enough to start being useful. I have -used it to track one real bug (for the interested people, see -bd220c268bc9_). So here is what it is, what it is not, and how to use -it. +* pode: https://github.com/rodsenra/pode -.. _bd220c268bc9: https://bitbucket.org/pypy/pypy/commits/bd220c268bc9 +For C: -RevDB is a Python debugger. It will not help track issues like -segfaults or crashes of the interpreter, but it will help track any -Python-level bugs. Think about bugs that end up as a Python traceback -or another wrong answer, but where the problem is really caused by -something earlier going wrong in your Python logic. +* rr: http://rr-project.org/ -RevDB is a logging system, similar to http://rr-project.org/ . You -first run your Python program by using a special version of PyPy. It -creates a log file which records the I/O actions. Sometimes you are -tracking a rare bug: you may need to run your program many times until -it shows the crash. That should still be reasonable: the special -version of PyPy is very slow (it does not contain any JIT nor one of -our high-performance GCs), but still not incredibly so---it is a few -times slower than running the same program on CPython. The point is -also that normally, what you need is just one recorded run of the -program showing the bug. You may struggle a bit to get that, but once -you have it, this part is done. +* undodb-gdb: http://undo.io/ -Then you use the debugger on the log file. The debugger will also -internally re-run the special version of PyPy in a different mode. -This feels like a debugger, though it is really a program that -inspects any point of the history. Like in a normal pdb, you can use -commands like "next" and "p foo.bar" and even run more complicated -bits of Python code. You also have new commands like "bnext" to go -backwards. Most importantly, you can set *watchpoints*. More about -that later. - - -XXX CF: it's not clear to me what "doing any input/output from the debugger" means - -What you cannot do is do any input/output from the debugger. Indeed, -the log file records all imports that were done and what the imported -modules contained. Running the debugger on the log file gives an -exact replay of what was recorded. - - - - - - - - - - -- no thread module for now. And, no cpyext module for now (the - CPython C API compatibility layer), because it depends on threads. - No micronumpy either. - These missing modules are probably blockers for large programs. - -- does not contain a JIT, and does not use our fast garbage collector. - -- for now, the process needs to get the same addresses (of C functions - and static data) when recording and when replaying. On the Linux I - tried it with, you get this result by disabling Address Space Layout - Randomization (ASLR):: - - echo 0 | sudo tee /proc/sys/kernel/randomize_va_space - -- OS/X and other Posix platforms are probably just a few fixes away. - Windows support will require some custom logic to replace the - forking done when replaying. This is more involved but should still - be possible. - -- maybe 15x memory usage on replaying (adjust number of forks in - process.py, MAX_SUBPROCESSES). - -- replaying issues: - - - Attempted to do I/O or access raw memory: we get this often, and - then we need "bstep+step" before we can print anything else - - - id() is globally unique, returning a reproducible 64-bit number, - so sometimes using id(x) is a workaround for when using x doesn't - work because of "Attempt to do I/O" issues (e.g. - ``p [id(x) for x in somelist]``) - - - next/bnext/finish/bfinish might jump around a bit non-predictably. - - - similarly, breaks on watchpoints can stop at apparently unexpected - places (when going backward, try to do "step" once). The issue is - that it can only stop at the beginning of every line. In the - extreme example, if a line is ``foo(somelist.pop(getindex()))``, - then ``somelist`` is modified in the middle. Immediately before - this modification occurs, we are in ``getindex()``, and - immediately afterwards we are in ``foo()``. The watchpoint will - stop the program at the end of ``getindex()`` if running backward, - and at the start of ``foo()`` if running forward, but never - actually on the line doing the change. - - - the first time you use $NUM to refer to an object, if it was - created long ago, then replaying might need to replay again from - that long-ago time +.. _`reverse debugging`: https://en.wikipedia.org/wiki/Debugger#Reverse_debugging +.. _`undodb-gdb`: http://undo.io/ +.. _`rr`: http://rr-project.org/ From pypy.commits at gmail.com Fri Jul 8 05:21:25 2016 From: pypy.commits at gmail.com (raffael_t) Date: Fri, 08 Jul 2016 02:21:25 -0700 (PDT) Subject: [pypy-commit] pypy py3.5: Add parser test for unpack Message-ID: <577f7095.8a40c20a.935af.ffff9e85@mx.google.com> Author: Raffael Tfirst Branch: py3.5 Changeset: r85605:fe3dcd041142 Date: 2016-07-08 11:20 +0200 http://bitbucket.org/pypy/pypy/changeset/fe3dcd041142/ Log: Add parser test for unpack diff --git a/pypy/interpreter/pyparser/test/test_pyparse.py b/pypy/interpreter/pyparser/test/test_pyparse.py --- a/pypy/interpreter/pyparser/test/test_pyparse.py +++ b/pypy/interpreter/pyparser/test/test_pyparse.py @@ -167,6 +167,12 @@ py.test.raises(SyntaxError, self.parse, 'f()\n# blah\nblah()', "single") py.test.raises(SyntaxError, self.parse, 'f()\nxy # blah\nblah()', "single") py.test.raises(SyntaxError, self.parse, 'x = 5 # comment\nx = 6\n', "single") + + def test_unpack(self): + self.parse('[*{2}, 3, *[4]]') + self.parse('{*{2}, 3, *[4]}') + self.parse('{**{}, 3:4, **{5:6, 7:8}}') + self.parse('f(2, *a, *b, **b, **c, **d)') class TestPythonParserWithSpace: From pypy.commits at gmail.com Fri Jul 8 05:43:29 2016 From: pypy.commits at gmail.com (cfbolz) Date: Fri, 08 Jul 2016 02:43:29 -0700 (PDT) Subject: [pypy-commit] extradoc extradoc: some typos and a link Message-ID: <577f75c1.8438c20a.e25a3.4c6e@mx.google.com> Author: Carl Friedrich Bolz Branch: extradoc Changeset: r5648:77b4cdab6a0a Date: 2016-07-08 11:43 +0200 http://bitbucket.org/pypy/extradoc/changeset/77b4cdab6a0a/ Log: some typos and a link diff --git a/blog/draft/revdb.rst b/blog/draft/revdb.rst --- a/blog/draft/revdb.rst +++ b/blog/draft/revdb.rst @@ -8,8 +8,8 @@ A "reverse debugger" is a debugger where you can go forward and backward in time. It is an uncommon feature, at least in the open source world, but I have no idea why. I have used `undodb-gdb`_ and -`rr`_, which are reverse debugger for C code, and I can only say that -it saved me many, many days of poking around blindly in gdb. +`rr`_, which are reverse debuggers for C code, and I can only say that +they saved me many, many days of poking around blindly in gdb. The PyPy team is pleased to give you "RevPDB", a reverse-debugger similar to ``rr`` but for Python. @@ -142,7 +142,7 @@ equivalents. There is also ``go TIME`` to jump directly to the specified time. (Right now the debugger only stops at "line start" events, not at function entry or exit, which makes some cases a bit -surprizing: for example, a ``step`` from the return statement of +surprising: for example, a ``step`` from the return statement of function ``foo()`` will jump directly to the caller's caller, if the caller's current line was ``return foo() + 2``, because no "line start" event occurs in the caller after ``foo()`` returns to it.) @@ -401,9 +401,11 @@ because it would require systematic changes everywhere. The most obvious and successful examples are the GC and the JIT. But there have been many other experiments along the same lines, from the -so-called "stackless transformation" in the early days, to the STM +so-called `stackless transformation`_ in the early days, to the STM version of PyPy. +.. _`stackless transformation`: https://bitbucket.org/pypy/extradoc/raw/tip/eu-report/D07.1_Massive_Parallelism_and_Translation_Aspects-2007-02-28.pdf + RevPDB works in a similar way. It is a version of PyPy in which some operations are systematically replaced with other operations. From pypy.commits at gmail.com Fri Jul 8 05:57:25 2016 From: pypy.commits at gmail.com (arigo) Date: Fri, 08 Jul 2016 02:57:25 -0700 (PDT) Subject: [pypy-commit] extradoc extradoc: Clarification, and add conclusion Message-ID: <577f7905.a90bc30a.8f8fb.70ad@mx.google.com> Author: Armin Rigo Branch: extradoc Changeset: r5649:f87456e7127d Date: 2016-07-08 11:59 +0200 http://bitbucket.org/pypy/extradoc/changeset/f87456e7127d/ Log: Clarification, and add conclusion diff --git a/blog/draft/revdb.rst b/blog/draft/revdb.rst --- a/blog/draft/revdb.rst +++ b/blog/draft/revdb.rst @@ -110,9 +110,17 @@ get the failure. But once we got the failure, then we're done with this step.) -Start ``rpython/translator/revdb/revdb.py log.rdb``. We get a -pdb-style debugger. Initially, we are at the start of the program -(not at the end, like we'd get in a regular debugger):: +Start:: + + rpython/translator/revdb/revdb.py log.rdb + +We get a pdb-style debugger. This ``revdb.py`` is a normal Python +program, which you run with an unmodified Python; internally, it looks +inside the log for the path to ``pypy-revdb`` and run it as needed (as +one forking subprocess, in a special mode). + +Initially, we are at the start of the program---not at the end, like +we'd get in a regular debugger:: File "/app_main.py", line 787 in setup_bootstrap_path: (1)$ @@ -121,6 +129,7 @@ Go to the end with ``continue`` (or ``c``):: + (1)$ continue File "/tmp/x.py", line 14 in : ... lst1 = [Foo() for i in range(100)] @@ -139,7 +148,7 @@ backward with the ``bstep`` command (backward step, or ``bs``), line by line, and forward again with the ``step`` command. There are also commands ``bnext``, ``bcontinue`` and ``bfinish`` and their forward -equivalents. There is also ``go TIME`` to jump directly to the specified +equivalents. There is also "``go TIME``" to jump directly to the specified time. (Right now the debugger only stops at "line start" events, not at function entry or exit, which makes some cases a bit surprising: for example, a ``step`` from the return statement of @@ -381,8 +390,8 @@ have side-effects; these effects are discarded as soon as you move in time again. -- sometimes even ``p import foo`` will fail with ``Attempted to do - I/O``. Use instead ``p import sys; foo = sys.modules['foo']``. +- sometimes even "``p import foo``" will fail with ``Attempted to do + I/O``. Use instead "``p import sys; foo = sys.modules['foo']``". - use ``help`` to see all commands. ``backtrace`` can be useful. There is no ``up`` command; you have to move in time instead, @@ -481,3 +490,32 @@ .. _`reverse debugging`: https://en.wikipedia.org/wiki/Debugger#Reverse_debugging .. _`undodb-gdb`: http://undo.io/ .. _`rr`: http://rr-project.org/ + + +Future work +----------- + +As mentioned above, it is alpha-level, and only works on Linux with ASLR +disabled. So the plans for the immediate future are to fix the various +issues described above, and port to more operating systems and remove +the restriction that requires a non-ASLR system. The core of the system +is in the C file and headers in ``rpython/translator/revdb/src-revdb``. + +For interested people, there is also the Duhton_ interpreter and its +``reverse-debugger`` branch, which is where I prototyped the RPython +concept before moving to PyPy. The basics should work for any +interpreter written in RPython, but they require some specific code to +interface with the language; in the case of PyPy, it is in +``pypy/interpreter/reverse_debugging.py``. + +.. _Duhton: https://bitbucket.org/pypy/duhton/ + +In parallel, there are various user interface improvements that people +could be interested in, like a more "pdb++" experience. (And the script +at ``rpython/translator/revdb/revdb.py`` should be moved out into some +more "official" place, and the ``reverse-debugger`` branch should be +merged back to default.) + +I would certainly welcome any help! + +-+- Armin From pypy.commits at gmail.com Fri Jul 8 06:08:28 2016 From: pypy.commits at gmail.com (arigo) Date: Fri, 08 Jul 2016 03:08:28 -0700 (PDT) Subject: [pypy-commit] pypy reverse-debugger: Import readline Message-ID: <577f7b9c.c626c20a.a4253.6f41@mx.google.com> Author: Armin Rigo Branch: reverse-debugger Changeset: r85606:4d82621df5ed Date: 2016-07-08 08:47 +0200 http://bitbucket.org/pypy/pypy/changeset/4d82621df5ed/ Log: Import readline diff --git a/rpython/translator/revdb/interact.py b/rpython/translator/revdb/interact.py --- a/rpython/translator/revdb/interact.py +++ b/rpython/translator/revdb/interact.py @@ -2,6 +2,10 @@ import subprocess, socket import traceback from contextlib import contextmanager +try: + import readline +except ImportError: + pass from rpython.translator.revdb.process import ReplayProcessGroup from rpython.translator.revdb.process import Breakpoint @@ -43,10 +47,8 @@ self.pgroup.show_backtrace(complete=0) previous_time = last_time prompt = '(%d)$ ' % last_time - sys.stdout.write(prompt) - sys.stdout.flush() try: - cmdline = raw_input().strip() + cmdline = raw_input(prompt).strip() except EOFError: print cmdline = 'quit' From pypy.commits at gmail.com Fri Jul 8 06:28:35 2016 From: pypy.commits at gmail.com (raffael_t) Date: Fri, 08 Jul 2016 03:28:35 -0700 (PDT) Subject: [pypy-commit] pypy py3.5: Fix ast test for attributes Message-ID: <577f8053.a90bc30a.8f8fb.7d3f@mx.google.com> Author: Raffael Tfirst Branch: py3.5 Changeset: r85607:c6e0ba85ad3d Date: 2016-07-08 12:27 +0200 http://bitbucket.org/pypy/pypy/changeset/c6e0ba85ad3d/ Log: Fix ast test for attributes diff --git a/pypy/interpreter/astcompiler/test/test_ast.py b/pypy/interpreter/astcompiler/test/test_ast.py --- a/pypy/interpreter/astcompiler/test/test_ast.py +++ b/pypy/interpreter/astcompiler/test/test_ast.py @@ -45,8 +45,8 @@ w_fields = space.getattr(ast.get(space).w_arguments, space.wrap("_fields")) assert space.eq_w(w_fields, space.wrap( - ('args', 'vararg', 'varargannotation', 'kwonlyargs', 'kwarg', - 'kwargannotation', 'defaults', 'kw_defaults'))) + ('args', 'vararg', 'kwonlyargs', 'kw_defaults', + 'kwarg', 'defaults'))) def test_attributes(self, space): w_attrs = space.getattr(ast.get(space).w_FunctionDef, From pypy.commits at gmail.com Fri Jul 8 07:10:55 2016 From: pypy.commits at gmail.com (raffael_t) Date: Fri, 08 Jul 2016 04:10:55 -0700 (PDT) Subject: [pypy-commit] pypy py3.5: Fix astbuilder tests for call and matmul->matmult Message-ID: <577f8a3f.85c11c0a.3e9a4.c66f@mx.google.com> Author: Raffael Tfirst Branch: py3.5 Changeset: r85608:eb7755aa6d90 Date: 2016-07-08 13:10 +0200 http://bitbucket.org/pypy/pypy/changeset/eb7755aa6d90/ Log: Fix astbuilder tests for call and matmul->matmult diff --git a/pypy/interpreter/astcompiler/test/test_astbuilder.py b/pypy/interpreter/astcompiler/test/test_astbuilder.py --- a/pypy/interpreter/astcompiler/test/test_astbuilder.py +++ b/pypy/interpreter/astcompiler/test/test_astbuilder.py @@ -625,7 +625,7 @@ ("/=", ast.Div), ("//=", ast.FloorDiv), ("%=", ast.Mod), - ("@=", ast.MatMul), + ("@=", ast.MatMult), ("<<=", ast.LShift), (">>=", ast.RShift), ("&=", ast.BitAnd), @@ -931,7 +931,7 @@ ("*", ast.Mult), ("//", ast.FloorDiv), ("%", ast.Mod), - ("@", ast.MatMul) + ("@", ast.MatMult) ) for op, ast_type in binops: bin = self.get_first_expr("a %s b" % (op,)) @@ -987,8 +987,6 @@ assert isinstance(call, ast.Call) assert call.args is None assert call.keywords is None - assert call.starargs is None - assert call.kwargs is None assert isinstance(call.func, ast.Name) assert call.func.ctx == ast.Load call = self.get_first_expr("f(2, 3)") @@ -996,8 +994,6 @@ assert isinstance(call.args[0], ast.Num) assert isinstance(call.args[1], ast.Num) assert call.keywords is None - assert call.starargs is None - assert call.kwargs is None call = self.get_first_expr("f(a=3)") assert call.args is None assert len(call.keywords) == 1 @@ -1006,21 +1002,20 @@ assert keyword.arg == "a" assert isinstance(keyword.value, ast.Num) call = self.get_first_expr("f(*a, **b)") - assert call.args is None - assert isinstance(call.starargs, ast.Name) - assert call.starargs.id == "a" - assert call.starargs.ctx == ast.Load - assert isinstance(call.kwargs, ast.Name) - assert call.kwargs.id == "b" - assert call.kwargs.ctx == ast.Load + assert isinstance(call.args[0], ast.Starred) + assert isinstance(call.keywords[0], ast.keyword) + assert call.args[0].value.id == "a" + assert call.args[0].ctx == ast.Load + assert call.keywords[0].value.id == "b" call = self.get_first_expr("f(a, b, x=4, *m, **f)") - assert len(call.args) == 2 + assert len(call.args) == 3 assert isinstance(call.args[0], ast.Name) assert isinstance(call.args[1], ast.Name) - assert len(call.keywords) == 1 + assert isinstance(call.args[2], ast.Starred) + assert len(call.keywords) == 2 assert call.keywords[0].arg == "x" - assert call.starargs.id == "m" - assert call.kwargs.id == "f" + assert call.args[2].value.id == "m" + assert call.keywords[1].value.id == "f" call = self.get_first_expr("f(x for x in y)") assert len(call.args) == 1 assert isinstance(call.args[0], ast.GeneratorExp) @@ -1036,8 +1031,6 @@ assert exc.msg == "keyword can't be an expression" exc = py.test.raises(SyntaxError, self.get_ast, "f(a=c, a=d)").value assert exc.msg == "keyword argument repeated" - exc = py.test.raises(SyntaxError, self.get_ast, "f(x, *a, b)").value - assert exc.msg == "only named arguments may follow *expression" def test_attribute(self): attr = self.get_first_expr("x.y") @@ -1345,7 +1338,7 @@ body = mod.body assert len(body) == 1 expr = body[0].value - assert expr.op == ast.MatMul + assert expr.op == ast.MatMult assert isinstance(expr.left, ast.Name) assert isinstance(expr.right, ast.Name) # imatmul is tested earlier search for @= From pypy.commits at gmail.com Fri Jul 8 07:12:56 2016 From: pypy.commits at gmail.com (raffael_t) Date: Fri, 08 Jul 2016 04:12:56 -0700 (PDT) Subject: [pypy-commit] pypy py3.5: Fix astbuilder test for decorators Message-ID: <577f8ab8.44f6c20a.580cd.4627@mx.google.com> Author: Raffael Tfirst Branch: py3.5 Changeset: r85609:b38b0afbeafe Date: 2016-07-08 13:12 +0200 http://bitbucket.org/pypy/pypy/changeset/b38b0afbeafe/ Log: Fix astbuilder test for decorators diff --git a/pypy/interpreter/astcompiler/test/test_astbuilder.py b/pypy/interpreter/astcompiler/test/test_astbuilder.py --- a/pypy/interpreter/astcompiler/test/test_astbuilder.py +++ b/pypy/interpreter/astcompiler/test/test_astbuilder.py @@ -606,8 +606,6 @@ assert dec.func.id == "dec" assert dec.args is None assert dec.keywords is None - assert dec.starargs is None - assert dec.kwargs is None definition = self.get_first_stmt("@dec(a, b)\n%s" % (stmt,)) assert len(definition.decorator_list) == 1 dec = definition.decorator_list[0] @@ -615,8 +613,6 @@ assert dec.func.id == "dec" assert len(dec.args) == 2 assert dec.keywords is None - assert dec.starargs is None - assert dec.kwargs is None def test_augassign(self): aug_assigns = ( From pypy.commits at gmail.com Fri Jul 8 07:22:51 2016 From: pypy.commits at gmail.com (raffael_t) Date: Fri, 08 Jul 2016 04:22:51 -0700 (PDT) Subject: [pypy-commit] pypy py3.5: Fix astbuilder test for function annotations Message-ID: <577f8d0b.488e1c0a.7a4d9.cfd6@mx.google.com> Author: Raffael Tfirst Branch: py3.5 Changeset: r85610:50acbccac773 Date: 2016-07-08 13:22 +0200 http://bitbucket.org/pypy/pypy/changeset/50acbccac773/ Log: Fix astbuilder test for function annotations diff --git a/pypy/interpreter/astcompiler/test/test_astbuilder.py b/pypy/interpreter/astcompiler/test/test_astbuilder.py --- a/pypy/interpreter/astcompiler/test/test_astbuilder.py +++ b/pypy/interpreter/astcompiler/test/test_astbuilder.py @@ -545,9 +545,9 @@ assert isinstance(func.args.args[0].annotation, ast.Num) assert isinstance(func.args.defaults[0], ast.Name) func = self.get_first_stmt("def f(*x : 42): pass") - assert isinstance(func.args.varargannotation, ast.Num) + assert isinstance(func.args.vararg.annotation, ast.Num) func = self.get_first_stmt("def f(**kw : 42): pass") - assert isinstance(func.args.kwargannotation, ast.Num) + assert isinstance(func.args.kwarg.annotation, ast.Num) func = self.get_first_stmt("def f(*, kw : 42=a): pass") assert isinstance(func.args.kwonlyargs[0].annotation, ast.Num) From pypy.commits at gmail.com Fri Jul 8 07:40:03 2016 From: pypy.commits at gmail.com (raffael_t) Date: Fri, 08 Jul 2016 04:40:03 -0700 (PDT) Subject: [pypy-commit] pypy py3.5: Fix astbuilder test for function (all astbuilder tests adjusted to check new unpack functionality) Message-ID: <577f9113.22ecc20a.45542.3350@mx.google.com> Author: Raffael Tfirst Branch: py3.5 Changeset: r85611:5ef09f069fcb Date: 2016-07-08 13:39 +0200 http://bitbucket.org/pypy/pypy/changeset/5ef09f069fcb/ Log: Fix astbuilder test for function (all astbuilder tests adjusted to check new unpack functionality) diff --git a/pypy/interpreter/astcompiler/test/test_astbuilder.py b/pypy/interpreter/astcompiler/test/test_astbuilder.py --- a/pypy/interpreter/astcompiler/test/test_astbuilder.py +++ b/pypy/interpreter/astcompiler/test/test_astbuilder.py @@ -492,12 +492,12 @@ assert not args.args assert not args.defaults assert args.kwarg is None - assert args.vararg == "a" + assert args.vararg.arg == "a" args = self.get_first_stmt("def f(**a): pass").args assert not args.args assert not args.defaults assert args.vararg is None - assert args.kwarg == "a" + assert args.kwarg.arg == "a" args = self.get_first_stmt("def f(a, b, c=d, *e, **f): pass").args assert len(args.args) == 3 for arg in args.args: @@ -505,8 +505,8 @@ assert len(args.defaults) == 1 assert isinstance(args.defaults[0], ast.Name) assert args.defaults[0].ctx == ast.Load - assert args.vararg == "e" - assert args.kwarg == "f" + assert args.vararg.arg == "e" + assert args.kwarg.arg == "f" input = "def f(a=b, c): pass" exc = py.test.raises(SyntaxError, self.get_ast, input).value assert exc.msg == "non-default argument follows default argument" From pypy.commits at gmail.com Fri Jul 8 13:22:16 2016 From: pypy.commits at gmail.com (arigo) Date: Fri, 08 Jul 2016 10:22:16 -0700 (PDT) Subject: [pypy-commit] pypy default: Issue #870: ctypes.byref(.., offset) Message-ID: <577fe148.a468c20a.749cc.3845@mx.google.com> Author: Armin Rigo Branch: Changeset: r85612:36aba877dfa1 Date: 2016-07-08 14:35 +0200 http://bitbucket.org/pypy/pypy/changeset/36aba877dfa1/ Log: Issue #870: ctypes.byref(.., offset) diff --git a/lib_pypy/_ctypes/basics.py b/lib_pypy/_ctypes/basics.py --- a/lib_pypy/_ctypes/basics.py +++ b/lib_pypy/_ctypes/basics.py @@ -199,10 +199,13 @@ return tp._alignmentofinstances() @builtinify -def byref(cdata): +def byref(cdata, offset=0): # "pointer" is imported at the end of this module to avoid circular # imports - return pointer(cdata) + ptr = pointer(cdata) + if offset != 0: + ptr._buffer[0] += offset + return ptr def cdata_from_address(self, address): # fix the address: turn it into as unsigned, in case it's a negative number diff --git a/pypy/module/test_lib_pypy/ctypes_tests/test_pointers.py b/pypy/module/test_lib_pypy/ctypes_tests/test_pointers.py --- a/pypy/module/test_lib_pypy/ctypes_tests/test_pointers.py +++ b/pypy/module/test_lib_pypy/ctypes_tests/test_pointers.py @@ -265,3 +265,10 @@ class A(object): _byref = byref A._byref(c_int(5)) + + def test_byref_with_offset(self): + c = c_int() + d = byref(c) + base = cast(d, c_void_p).value + for i in [0, 1, 4, 1444, -10293]: + assert cast(byref(c, i), c_void_p).value == base + i From pypy.commits at gmail.com Fri Jul 8 13:22:18 2016 From: pypy.commits at gmail.com (arigo) Date: Fri, 08 Jul 2016 10:22:18 -0700 (PDT) Subject: [pypy-commit] pypy reverse-debugger: Use personality() with ADDR_NO_RANDOMIZE, a linux-only way to ask for Message-ID: <577fe14a.8a40c20a.935af.5eff@mx.google.com> Author: Armin Rigo Branch: reverse-debugger Changeset: r85613:ff376ccacb36 Date: 2016-07-08 18:12 +0200 http://bitbucket.org/pypy/pypy/changeset/ff376ccacb36/ Log: Use personality() with ADDR_NO_RANDOMIZE, a linux-only way to ask for fixed addresses. Thanks mjacob. diff --git a/rpython/translator/revdb/src-revdb/revdb.c b/rpython/translator/revdb/src-revdb/revdb.c --- a/rpython/translator/revdb/src-revdb/revdb.c +++ b/rpython/translator/revdb/src-revdb/revdb.c @@ -13,6 +13,13 @@ #include #include +#ifdef __linux__ +# define HAVE_PERSONALITY +#endif +#ifdef HAVE_PERSONALITY +# include +#endif + #include "structdef.h" #include "forwarddecl.h" #include "preimpl.h" @@ -54,6 +61,42 @@ static void setup_replay_mode(int *argc_p, char **argv_p[]); static void check_at_end(uint64_t stop_points); +static void ensure_fixed_address_space(char *argv[]) +{ +#ifdef HAVE_PERSONALITY + int pers = personality(0xffffffff); + if (pers == -1) { + perror("personality"); + exit(1); + } + if (!(pers & ADDR_NO_RANDOMIZE)) { + pers |= ADDR_NO_RANDOMIZE; + if (personality(pers) == -1) { + perror("personality"); + exit(1); + } + pers = personality(0xffffffff); + if (pers == -1 || !(pers & ADDR_NO_RANDOMIZE)) { + fprintf(stderr, "cannot set ADDR_NO_RANDOMIZE\n"); + exit(1); + } + /* After setting this personality(), we need to restart the + current process. It will then reload the libpypy-c.so at a + non-randomized address. + + Potentially buggy to use argv[0] here, but good enough I + suppose. For this reason ensure_fixed_address_space() is + not called when running manually without any PYPYRDB + environment variable set. + */ + execv(argv[0], argv); + + perror("execv"); + exit(1); + } +#endif +} + RPY_EXTERN void rpy_reverse_db_setup(int *argc_p, char **argv_p[]) { @@ -152,6 +195,8 @@ assert(RPY_RDB_REPLAY == 0); if (filename && *filename) { + ensure_fixed_address_space(argv); + putenv("PYPYRDB="); rpy_rev_fileno = open(filename, O_RDWR | O_CLOEXEC | O_CREAT | O_NOCTTY | O_TRUNC, 0600); @@ -651,6 +696,8 @@ argv[0]); exit(2); } + ensure_fixed_address_space(*argv_p); + rpy_rev_filename = argv[2]; reopen_revdb_file(rpy_rev_filename); rpy_rev_sockfd = atoi(argv[3]); From pypy.commits at gmail.com Sat Jul 9 04:13:11 2016 From: pypy.commits at gmail.com (arigo) Date: Sat, 09 Jul 2016 01:13:11 -0700 (PDT) Subject: [pypy-commit] pypy default: FAQ: How should I report a bug? Message-ID: <5780b217.e7a4c20a.72666.51bd@mx.google.com> Author: Armin Rigo Branch: Changeset: r85614:df780b2a454e Date: 2016-07-09 10:14 +0200 http://bitbucket.org/pypy/pypy/changeset/df780b2a454e/ Log: FAQ: How should I report a bug? diff --git a/pypy/doc/faq.rst b/pypy/doc/faq.rst --- a/pypy/doc/faq.rst +++ b/pypy/doc/faq.rst @@ -335,3 +335,57 @@ This will disable SELinux's protection and allow PyPy to configure correctly. Be sure to enable it again if you need it! + + +How should I report a bug? +-------------------------- + +Our bug tracker is here: https://bitbucket.org/pypy/pypy/issues/ + +Missing features or incompatibilities with CPython are considered +bugs, and they are welcome. (See also our list of `known +incompatibilities`__.) + +.. __: http://pypy.org/compat.html + +For bugs of the kind "I'm getting a PyPy crash or a strange +exception", please note that: **We can't do anything without +reproducing the bug ourselves**. We cannot do anything with +tracebacks from gdb, or core dumps. This is not only because the +standard PyPy is compiled without debug symbols. The real reason is +that a C-level traceback is usually of not help at all in PyPy. +Debugging PyPy can be annoying. + +In more details: + +* First, please give the exact PyPy version, and the OS. + +* It might help focus our search if we know if the bug can be + reproduced on a "``pypy --jit off``" or not. If "``pypy --jit + off``" always works, then the problem might be in the JIT. + Otherwise, we know we can ignore that part. + +* If you got the bug using only Open Source components, please give a + step-by-step guide that we can follow to reproduce the problem + ourselves. Don't assume we know anything about any program other + than PyPy. We would like a guide that we can follow point by point + on a machine similar to yours, starting from a bare PyPy, until we + see the same problem. (If you can, you can try to reduce the number + of steps and the time it needs to run, but that is not mandatory.) + +* If the bug involves Closed Source components, or just too many Open + Source components to install them all ourselves, then maybe you can + give us some temporary ssh access to a machine where the bug can be + reproduced. + +* If giving us access would require us to sign a NDA, then we can + consider a commerical support contract for a small sum of money. + +* If even that is not possible for you, then sorry, we can't help. + +Of course, you can try to debug the problem yourself, and we can help +you get started if you ask on the #pypy IRC channel, but be prepared: +debugging an annoying PyPy problem usually involves quite a lot of gdb +in auto-generated C code, and at least some knowledge about the +various components involved, from PyPy's own RPython source code to +the GC and possibly the JIT. From pypy.commits at gmail.com Sat Jul 9 04:13:51 2016 From: pypy.commits at gmail.com (arigo) Date: Sat, 09 Jul 2016 01:13:51 -0700 (PDT) Subject: [pypy-commit] pypy default: typo (thanks LarstiQ) Message-ID: <5780b23f.4ccf1c0a.b6a45.e7f9@mx.google.com> Author: Armin Rigo Branch: Changeset: r85615:54735c1e0e5d Date: 2016-07-09 10:15 +0200 http://bitbucket.org/pypy/pypy/changeset/54735c1e0e5d/ Log: typo (thanks LarstiQ) diff --git a/pypy/doc/faq.rst b/pypy/doc/faq.rst --- a/pypy/doc/faq.rst +++ b/pypy/doc/faq.rst @@ -353,7 +353,7 @@ reproducing the bug ourselves**. We cannot do anything with tracebacks from gdb, or core dumps. This is not only because the standard PyPy is compiled without debug symbols. The real reason is -that a C-level traceback is usually of not help at all in PyPy. +that a C-level traceback is usually of no help at all in PyPy. Debugging PyPy can be annoying. In more details: From pypy.commits at gmail.com Sat Jul 9 04:17:05 2016 From: pypy.commits at gmail.com (arigo) Date: Sat, 09 Jul 2016 01:17:05 -0700 (PDT) Subject: [pypy-commit] pypy default: typo (thanks LarstiQ) Message-ID: <5780b301.44f6c20a.ed356.3f12@mx.google.com> Author: Armin Rigo Branch: Changeset: r85616:ba2081b41a4f Date: 2016-07-09 10:15 +0200 http://bitbucket.org/pypy/pypy/changeset/ba2081b41a4f/ Log: typo (thanks LarstiQ) diff --git a/rpython/jit/metainterp/test/test_ajit.py b/rpython/jit/metainterp/test/test_ajit.py --- a/rpython/jit/metainterp/test/test_ajit.py +++ b/rpython/jit/metainterp/test/test_ajit.py @@ -4470,6 +4470,28 @@ self.meta_interp(f, []) + def test_issue2335_recursion(self): + # Reproduces issue #2335: same as issue #2200, but the workaround + # in c4c54cb69aba was not enough. + driver = JitDriver(greens=["level"], reds=["i"]) + def enter(level, i): + if level == 0: + f(1) # recursive call + driver.can_enter_jit(level=level, i=i) + def f(level): + i = 0 if level == 0 else 298 + while True: + driver.jit_merge_point(level=level, i=i) + i += 1 + if i >= 300: + return i + promote(i + 1) # a failing guard + enter(level, i) + def main(): + set_param(None, 'trace_eagerness', 999999) + f(0) + self.meta_interp(main, []) + def test_pending_setarrayitem_with_indirect_constant_index(self): driver = JitDriver(greens=[], reds='auto') class X: From pypy.commits at gmail.com Sat Jul 9 04:17:07 2016 From: pypy.commits at gmail.com (arigo) Date: Sat, 09 Jul 2016 01:17:07 -0700 (PDT) Subject: [pypy-commit] pypy default: Precision (LarstiQ) Message-ID: <5780b303.4d9a1c0a.6846.358d@mx.google.com> Author: Armin Rigo Branch: Changeset: r85617:5a09f6225271 Date: 2016-07-09 10:17 +0200 http://bitbucket.org/pypy/pypy/changeset/5a09f6225271/ Log: Precision (LarstiQ) diff --git a/pypy/doc/faq.rst b/pypy/doc/faq.rst --- a/pypy/doc/faq.rst +++ b/pypy/doc/faq.rst @@ -369,6 +369,7 @@ step-by-step guide that we can follow to reproduce the problem ourselves. Don't assume we know anything about any program other than PyPy. We would like a guide that we can follow point by point + (without guessing or having to figure things out) on a machine similar to yours, starting from a bare PyPy, until we see the same problem. (If you can, you can try to reduce the number of steps and the time it needs to run, but that is not mandatory.) From pypy.commits at gmail.com Sat Jul 9 05:46:14 2016 From: pypy.commits at gmail.com (arigo) Date: Sat, 09 Jul 2016 02:46:14 -0700 (PDT) Subject: [pypy-commit] pypy default: Replace workaround2200 with a proper fix: instead of calling Message-ID: <5780c7e6.244cc20a.66a0d.ffffec61@mx.google.com> Author: Armin Rigo Branch: Changeset: r85618:660e49cca2a1 Date: 2016-07-09 11:46 +0200 http://bitbucket.org/pypy/pypy/changeset/660e49cca2a1/ Log: Replace workaround2200 with a proper fix: instead of calling the assembler directly from can_enter_jit(), use an exception to get back at the level of ll_portal_runner(). diff --git a/rpython/jit/metainterp/blackhole.py b/rpython/jit/metainterp/blackhole.py --- a/rpython/jit/metainterp/blackhole.py +++ b/rpython/jit/metainterp/blackhole.py @@ -1143,45 +1143,35 @@ @arguments("cpu", "i", "R", "d", returns="i") def bhimpl_residual_call_r_i(cpu, func, args_r, calldescr): - workaround2200.active = True return cpu.bh_call_i(func, None, args_r, None, calldescr) @arguments("cpu", "i", "R", "d", returns="r") def bhimpl_residual_call_r_r(cpu, func, args_r, calldescr): - workaround2200.active = True return cpu.bh_call_r(func, None, args_r, None, calldescr) @arguments("cpu", "i", "R", "d") def bhimpl_residual_call_r_v(cpu, func, args_r, calldescr): - workaround2200.active = True return cpu.bh_call_v(func, None, args_r, None, calldescr) @arguments("cpu", "i", "I", "R", "d", returns="i") def bhimpl_residual_call_ir_i(cpu, func, args_i, args_r, calldescr): - workaround2200.active = True return cpu.bh_call_i(func, args_i, args_r, None, calldescr) @arguments("cpu", "i", "I", "R", "d", returns="r") def bhimpl_residual_call_ir_r(cpu, func, args_i, args_r, calldescr): - workaround2200.active = True return cpu.bh_call_r(func, args_i, args_r, None, calldescr) @arguments("cpu", "i", "I", "R", "d") def bhimpl_residual_call_ir_v(cpu, func, args_i, args_r, calldescr): - workaround2200.active = True return cpu.bh_call_v(func, args_i, args_r, None, calldescr) @arguments("cpu", "i", "I", "R", "F", "d", returns="i") def bhimpl_residual_call_irf_i(cpu, func, args_i,args_r,args_f,calldescr): - workaround2200.active = True return cpu.bh_call_i(func, args_i, args_r, args_f, calldescr) @arguments("cpu", "i", "I", "R", "F", "d", returns="r") def bhimpl_residual_call_irf_r(cpu, func, args_i,args_r,args_f,calldescr): - workaround2200.active = True return cpu.bh_call_r(func, args_i, args_r, args_f, calldescr) @arguments("cpu", "i", "I", "R", "F", "d", returns="f") def bhimpl_residual_call_irf_f(cpu, func, args_i,args_r,args_f,calldescr): - workaround2200.active = True return cpu.bh_call_f(func, args_i, args_r, args_f, calldescr) @arguments("cpu", "i", "I", "R", "F", "d") def bhimpl_residual_call_irf_v(cpu, func, args_i,args_r,args_f,calldescr): - workaround2200.active = True return cpu.bh_call_v(func, args_i, args_r, args_f, calldescr) # conditional calls - note that they cannot return stuff @@ -1209,54 +1199,44 @@ @arguments("cpu", "j", "R", returns="i") def bhimpl_inline_call_r_i(cpu, jitcode, args_r): - workaround2200.active = True return cpu.bh_call_i(jitcode.get_fnaddr_as_int(), None, args_r, None, jitcode.calldescr) @arguments("cpu", "j", "R", returns="r") def bhimpl_inline_call_r_r(cpu, jitcode, args_r): - workaround2200.active = True return cpu.bh_call_r(jitcode.get_fnaddr_as_int(), None, args_r, None, jitcode.calldescr) @arguments("cpu", "j", "R") def bhimpl_inline_call_r_v(cpu, jitcode, args_r): - workaround2200.active = True return cpu.bh_call_v(jitcode.get_fnaddr_as_int(), None, args_r, None, jitcode.calldescr) @arguments("cpu", "j", "I", "R", returns="i") def bhimpl_inline_call_ir_i(cpu, jitcode, args_i, args_r): - workaround2200.active = True return cpu.bh_call_i(jitcode.get_fnaddr_as_int(), args_i, args_r, None, jitcode.calldescr) @arguments("cpu", "j", "I", "R", returns="r") def bhimpl_inline_call_ir_r(cpu, jitcode, args_i, args_r): - workaround2200.active = True return cpu.bh_call_r(jitcode.get_fnaddr_as_int(), args_i, args_r, None, jitcode.calldescr) @arguments("cpu", "j", "I", "R") def bhimpl_inline_call_ir_v(cpu, jitcode, args_i, args_r): - workaround2200.active = True return cpu.bh_call_v(jitcode.get_fnaddr_as_int(), args_i, args_r, None, jitcode.calldescr) @arguments("cpu", "j", "I", "R", "F", returns="i") def bhimpl_inline_call_irf_i(cpu, jitcode, args_i, args_r, args_f): - workaround2200.active = True return cpu.bh_call_i(jitcode.get_fnaddr_as_int(), args_i, args_r, args_f, jitcode.calldescr) @arguments("cpu", "j", "I", "R", "F", returns="r") def bhimpl_inline_call_irf_r(cpu, jitcode, args_i, args_r, args_f): - workaround2200.active = True return cpu.bh_call_r(jitcode.get_fnaddr_as_int(), args_i, args_r, args_f, jitcode.calldescr) @arguments("cpu", "j", "I", "R", "F", returns="f") def bhimpl_inline_call_irf_f(cpu, jitcode, args_i, args_r, args_f): - workaround2200.active = True return cpu.bh_call_f(jitcode.get_fnaddr_as_int(), args_i, args_r, args_f, jitcode.calldescr) @arguments("cpu", "j", "I", "R", "F") def bhimpl_inline_call_irf_v(cpu, jitcode, args_i, args_r, args_f): - workaround2200.active = True return cpu.bh_call_v(jitcode.get_fnaddr_as_int(), args_i, args_r, args_f, jitcode.calldescr) @@ -1543,8 +1523,6 @@ if not self.nextblackholeinterp: self._exit_frame_with_exception(current_exc) return current_exc - finally: - workaround2200.active = False # # pass the frame's return value to the caller caller = self.nextblackholeinterp @@ -1717,10 +1695,3 @@ # _run_forever(firstbh, current_exc) convert_and_run_from_pyjitpl._dont_inline_ = True - -# ____________________________________________________________ - -class WorkaroundIssue2200(object): - pass -workaround2200 = WorkaroundIssue2200() -workaround2200.active = False diff --git a/rpython/jit/metainterp/warmspot.py b/rpython/jit/metainterp/warmspot.py --- a/rpython/jit/metainterp/warmspot.py +++ b/rpython/jit/metainterp/warmspot.py @@ -529,7 +529,7 @@ def make_enter_function(self, jd): from rpython.jit.metainterp.warmstate import WarmEnterState state = WarmEnterState(self, jd) - maybe_compile_and_run = state.make_entry_point() + maybe_compile_and_run, EnterJitAssembler = state.make_entry_point() jd.warmstate = state def crash_in_jit(e): @@ -560,6 +560,7 @@ maybe_enter_jit._always_inline_ = True jd._maybe_enter_jit_fn = maybe_enter_jit jd._maybe_compile_and_run_fn = maybe_compile_and_run + jd._EnterJitAssembler = EnterJitAssembler def make_driverhook_graphs(self): s_Str = annmodel.SomeString() @@ -833,6 +834,7 @@ # more stuff # # to that: + # XXX there are too many exceptions all around... # # def original_portal(..): # stuff @@ -841,7 +843,14 @@ # def portal_runner(*args): # while 1: # try: - # return portal(*args) + # try: + # return portal(*args) + # except EnterJitAssembler, e: + # while True: + # try: + # return e.execute() + # except EnterJitAssembler, e: + # continue # except ContinueRunningNormally, e: # *args = *e.new_args # except DoneWithThisFrame, e: @@ -887,26 +896,35 @@ ts = self.cpu.ts state = jd.warmstate maybe_compile_and_run = jd._maybe_compile_and_run_fn + EnterJitAssembler = jd._EnterJitAssembler def ll_portal_runner(*args): start = True while 1: try: - # maybe enter from the function's start. Note that the - # 'start' variable is constant-folded away because it's - # the first statement in the loop. - if start: - maybe_compile_and_run( - state.increment_function_threshold, *args) - # - # then run the normal portal function, i.e. the - # interpreter's main loop. It might enter the jit - # via maybe_enter_jit(), which typically ends with - # handle_fail() being called, which raises on the - # following exceptions --- catched here, because we - # want to interrupt the whole interpreter loop. - return support.maybe_on_top_of_llinterp(rtyper, - portal_ptr)(*args) + try: + # maybe enter from the function's start. Note that the + # 'start' variable is constant-folded away because it's + # the first statement in the loop. + if start: + maybe_compile_and_run( + state.increment_function_threshold, *args) + # + # then run the normal portal function, i.e. the + # interpreter's main loop. It might enter the jit + # via maybe_enter_jit(), which typically ends with + # handle_fail() being called, which raises on the + # following exceptions --- catched here, because we + # want to interrupt the whole interpreter loop. + return support.maybe_on_top_of_llinterp(rtyper, + portal_ptr)(*args) + except EnterJitAssembler as e: + while True: + try: + e.execute() + except EnterJitAssembler as e: + continue + assert 0 # execute() should always raise for now except jitexc.ContinueRunningNormally as e: args = () for ARGTYPE, attrname, count in portalfunc_ARGS: @@ -933,6 +951,7 @@ raise LLException(ts.get_typeptr(value), value) else: value = cast_base_ptr_to_instance(Exception, value) + assert value is not None raise value def handle_jitexception(e): diff --git a/rpython/jit/metainterp/warmstate.py b/rpython/jit/metainterp/warmstate.py --- a/rpython/jit/metainterp/warmstate.py +++ b/rpython/jit/metainterp/warmstate.py @@ -2,7 +2,7 @@ import weakref from rpython.jit.codewriter import support, heaptracker, longlong -from rpython.jit.metainterp import resoperation, history +from rpython.jit.metainterp import resoperation, history, jitexc from rpython.rlib.debug import debug_start, debug_stop, debug_print from rpython.rlib.debug import have_debug_prints_for from rpython.rlib.jit import PARAMETERS @@ -348,8 +348,8 @@ def make_entry_point(self): "NOT_RPYTHON" - if hasattr(self, 'maybe_compile_and_run'): - return self.maybe_compile_and_run + if hasattr(self, 'entry_point_fns'): + return self.entry_point_fns warmrunnerdesc = self.warmrunnerdesc metainterp_sd = warmrunnerdesc.metainterp_sd @@ -362,6 +362,8 @@ confirm_enter_jit = self.confirm_enter_jit range_red_args = unrolling_iterable( range(num_green_args, num_green_args + jitdriver_sd.num_red_args)) + name_red_args = unrolling_iterable( + [(i, 'arg%d' % i) for i in range(jitdriver_sd.num_red_args)]) # get a new specialized copy of the method ARGS = [] for kind in jitdriver_sd.red_args_types: @@ -419,7 +421,8 @@ def maybe_compile_and_run(increment_threshold, *args): """Entry point to the JIT. Called at the point with the - can_enter_jit() hint. + can_enter_jit() hint, and at the start of a function + with a different threshold. """ # Look for the cell corresponding to the current greenargs. # Search for the JitCell that is of the correct subclass of @@ -439,14 +442,6 @@ bound_reached(hash, None, *args) return - # Workaround for issue #2200, maybe temporary. This is not - # a proper fix, but only a hack that should work well enough - # for PyPy's main jitdriver... See test_issue2200_recursion - from rpython.jit.metainterp.blackhole import workaround2200 - if workaround2200.active: - workaround2200.active = False - return - # Here, we have found 'cell'. # if cell.flags & (JC_TRACING | JC_TEMPORARY): @@ -484,14 +479,27 @@ execute_args = () for i in range_red_args: execute_args += (unspecialize_value(args[i]), ) - # run it! this executes until interrupted by an exception - execute_assembler(procedure_token, *execute_args) - assert 0, "should not reach this point" + # run it, but from outside in ll_portal_runner, not from here + # (this avoids RPython-level recursion with no corresponding + # app-level recursion, as shown by issues 2200 and 2335) + raise EnterJitAssembler(procedure_token, *execute_args) + + class EnterJitAssembler(jitexc.JitException): + def __init__(self, procedure_token, *args): + self.procedure_token = procedure_token + for i, argname in name_red_args: + setattr(self, argname, args[i]) + def execute(self): + args = () + for i, argname in name_red_args: + args += (getattr(self, argname), ) + return execute_assembler(self.procedure_token, *args) maybe_compile_and_run._dont_inline_ = True - self.maybe_compile_and_run = maybe_compile_and_run self.execute_assembler = execute_assembler - return maybe_compile_and_run + self.entry_point_fns = (maybe_compile_and_run, + EnterJitAssembler) + return self.entry_point_fns # ---------- From pypy.commits at gmail.com Sat Jul 9 06:15:05 2016 From: pypy.commits at gmail.com (arigo) Date: Sat, 09 Jul 2016 03:15:05 -0700 (PDT) Subject: [pypy-commit] pypy default: Still pending some refactoring: this adds a special case to avoid Message-ID: <5780cea9.c91d1c0a.42e11.277a@mx.google.com> Author: Armin Rigo Branch: Changeset: r85619:5440c8b2d979 Date: 2016-07-09 12:16 +0200 http://bitbucket.org/pypy/pypy/changeset/5440c8b2d979/ Log: Still pending some refactoring: this adds a special case to avoid creating a DoneWithThisFrame exception in the common case diff --git a/rpython/jit/metainterp/compile.py b/rpython/jit/metainterp/compile.py --- a/rpython/jit/metainterp/compile.py +++ b/rpython/jit/metainterp/compile.py @@ -615,28 +615,35 @@ final_descr = True class DoneWithThisFrameDescrVoid(_DoneWithThisFrameDescr): + def get_result(self, cpu, deadframe): + return None def handle_fail(self, deadframe, metainterp_sd, jitdriver_sd): assert jitdriver_sd.result_type == history.VOID raise jitexc.DoneWithThisFrameVoid() class DoneWithThisFrameDescrInt(_DoneWithThisFrameDescr): + def get_result(self, cpu, deadframe): + return cpu.get_int_value(deadframe, 0) def handle_fail(self, deadframe, metainterp_sd, jitdriver_sd): assert jitdriver_sd.result_type == history.INT - result = metainterp_sd.cpu.get_int_value(deadframe, 0) - raise jitexc.DoneWithThisFrameInt(result) + cpu = metainterp_sd.cpu + raise jitexc.DoneWithThisFrameInt(self.get_result(cpu, deadframe)) class DoneWithThisFrameDescrRef(_DoneWithThisFrameDescr): + def get_result(self, cpu, deadframe): + return cpu.get_ref_value(deadframe, 0) def handle_fail(self, deadframe, metainterp_sd, jitdriver_sd): assert jitdriver_sd.result_type == history.REF cpu = metainterp_sd.cpu - result = cpu.get_ref_value(deadframe, 0) - raise jitexc.DoneWithThisFrameRef(cpu, result) + raise jitexc.DoneWithThisFrameRef(cpu, self.get_result(cpu, deadframe)) class DoneWithThisFrameDescrFloat(_DoneWithThisFrameDescr): + def get_result(self, cpu, deadframe): + return cpu.get_float_value(deadframe, 0) def handle_fail(self, deadframe, metainterp_sd, jitdriver_sd): assert jitdriver_sd.result_type == history.FLOAT - result = metainterp_sd.cpu.get_float_value(deadframe, 0) - raise jitexc.DoneWithThisFrameFloat(result) + cpu = metainterp_sd.cpu + raise jitexc.DoneWithThisFrameFloat(self.get_result(cpu, deadframe)) class ExitFrameWithExceptionDescrRef(_DoneWithThisFrameDescr): def handle_fail(self, deadframe, metainterp_sd, jitdriver_sd): diff --git a/rpython/jit/metainterp/pyjitpl.py b/rpython/jit/metainterp/pyjitpl.py --- a/rpython/jit/metainterp/pyjitpl.py +++ b/rpython/jit/metainterp/pyjitpl.py @@ -2543,7 +2543,13 @@ elif box.type == history.REF: args.append(box.getref_base()) elif box.type == history.FLOAT: args.append(box.getfloatstorage()) else: assert 0 - self.jitdriver_sd.warmstate.execute_assembler(loop_token, *args) + res = self.jitdriver_sd.warmstate.execute_assembler(loop_token, *args) + kind = getkind(lltype.typeOf(res)) + if kind == history.VOID: raise DoneWithThisFrameVoid() + if kind == history.INT: raise DoneWithThisFrameInt(res) + if kind == history.REF: raise DoneWithThisFrameRef(self.cpu, res) + if kind == history.FLOAT: raise DoneWithThisFrameFloat(res) + raise AssertionError(kind) def prepare_resume_from_failure(self, deadframe, inputargs, resumedescr): exception = self.cpu.grab_exc_value(deadframe) diff --git a/rpython/jit/metainterp/warmspot.py b/rpython/jit/metainterp/warmspot.py --- a/rpython/jit/metainterp/warmspot.py +++ b/rpython/jit/metainterp/warmspot.py @@ -893,6 +893,7 @@ rtyper = self.translator.rtyper RESULT = PORTALFUNC.RESULT result_kind = history.getkind(RESULT) + assert result_kind.startswith(jd.result_type) ts = self.cpu.ts state = jd.warmstate maybe_compile_and_run = jd._maybe_compile_and_run_fn @@ -921,10 +922,9 @@ except EnterJitAssembler as e: while True: try: - e.execute() + return e.execute() except EnterJitAssembler as e: continue - assert 0 # execute() should always raise for now except jitexc.ContinueRunningNormally as e: args = () for ARGTYPE, attrname, count in portalfunc_ARGS: diff --git a/rpython/jit/metainterp/warmstate.py b/rpython/jit/metainterp/warmstate.py --- a/rpython/jit/metainterp/warmstate.py +++ b/rpython/jit/metainterp/warmstate.py @@ -379,6 +379,18 @@ cpu = self.cpu jitcounter = self.warmrunnerdesc.jitcounter + result_type = jitdriver_sd.result_type + if result_type == history.VOID: + _DoneWithThisFrameCls = jitexc.DoneWithThisFrameVoid + elif result_type == history.INT: + _DoneWithThisFrameCls = jitexc.DoneWithThisFrameInt + elif result_type == history.REF: + _DoneWithThisFrameCls = jitexc.DoneWithThisFrameRef + elif result_type == history.FLOAT: + _DoneWithThisFrameCls = jitexc.DoneWithThisFrameFloat + else: + raise AssertionError(result_type) + def execute_assembler(loop_token, *args): # Call the backend to run the 'looptoken' with the given # input args. @@ -398,9 +410,13 @@ # # Handle the failure fail_descr = cpu.get_latest_descr(deadframe) - fail_descr.handle_fail(deadframe, metainterp_sd, jitdriver_sd) - # - assert 0, "should have raised" + if isinstance(fail_descr, _DoneWithThisFrameCls): + # A fast path to avoid raising and immediately catching + # a DoneWithThisFrame exception + return fail_descr.get_result(cpu, deadframe) + else: + fail_descr.handle_fail(deadframe, metainterp_sd, jitdriver_sd) + assert 0, "should have raised" def bound_reached(hash, cell, *args): if not confirm_enter_jit(*args): From pypy.commits at gmail.com Sat Jul 9 07:21:00 2016 From: pypy.commits at gmail.com (arigo) Date: Sat, 09 Jul 2016 04:21:00 -0700 (PDT) Subject: [pypy-commit] pypy default: Backout 660e49cca2a1 and 5440c8b2d979: the resulting PyPy Message-ID: <5780de1c.c91d1c0a.42e11.3f2e@mx.google.com> Author: Armin Rigo Branch: Changeset: r85620:1d6958f0ed94 Date: 2016-07-09 13:22 +0200 http://bitbucket.org/pypy/pypy/changeset/1d6958f0ed94/ Log: Backout 660e49cca2a1 and 5440c8b2d979: the resulting PyPy works in a slightly broken way, although all tests pass :-( diff --git a/rpython/jit/metainterp/blackhole.py b/rpython/jit/metainterp/blackhole.py --- a/rpython/jit/metainterp/blackhole.py +++ b/rpython/jit/metainterp/blackhole.py @@ -1143,35 +1143,45 @@ @arguments("cpu", "i", "R", "d", returns="i") def bhimpl_residual_call_r_i(cpu, func, args_r, calldescr): + workaround2200.active = True return cpu.bh_call_i(func, None, args_r, None, calldescr) @arguments("cpu", "i", "R", "d", returns="r") def bhimpl_residual_call_r_r(cpu, func, args_r, calldescr): + workaround2200.active = True return cpu.bh_call_r(func, None, args_r, None, calldescr) @arguments("cpu", "i", "R", "d") def bhimpl_residual_call_r_v(cpu, func, args_r, calldescr): + workaround2200.active = True return cpu.bh_call_v(func, None, args_r, None, calldescr) @arguments("cpu", "i", "I", "R", "d", returns="i") def bhimpl_residual_call_ir_i(cpu, func, args_i, args_r, calldescr): + workaround2200.active = True return cpu.bh_call_i(func, args_i, args_r, None, calldescr) @arguments("cpu", "i", "I", "R", "d", returns="r") def bhimpl_residual_call_ir_r(cpu, func, args_i, args_r, calldescr): + workaround2200.active = True return cpu.bh_call_r(func, args_i, args_r, None, calldescr) @arguments("cpu", "i", "I", "R", "d") def bhimpl_residual_call_ir_v(cpu, func, args_i, args_r, calldescr): + workaround2200.active = True return cpu.bh_call_v(func, args_i, args_r, None, calldescr) @arguments("cpu", "i", "I", "R", "F", "d", returns="i") def bhimpl_residual_call_irf_i(cpu, func, args_i,args_r,args_f,calldescr): + workaround2200.active = True return cpu.bh_call_i(func, args_i, args_r, args_f, calldescr) @arguments("cpu", "i", "I", "R", "F", "d", returns="r") def bhimpl_residual_call_irf_r(cpu, func, args_i,args_r,args_f,calldescr): + workaround2200.active = True return cpu.bh_call_r(func, args_i, args_r, args_f, calldescr) @arguments("cpu", "i", "I", "R", "F", "d", returns="f") def bhimpl_residual_call_irf_f(cpu, func, args_i,args_r,args_f,calldescr): + workaround2200.active = True return cpu.bh_call_f(func, args_i, args_r, args_f, calldescr) @arguments("cpu", "i", "I", "R", "F", "d") def bhimpl_residual_call_irf_v(cpu, func, args_i,args_r,args_f,calldescr): + workaround2200.active = True return cpu.bh_call_v(func, args_i, args_r, args_f, calldescr) # conditional calls - note that they cannot return stuff @@ -1199,44 +1209,54 @@ @arguments("cpu", "j", "R", returns="i") def bhimpl_inline_call_r_i(cpu, jitcode, args_r): + workaround2200.active = True return cpu.bh_call_i(jitcode.get_fnaddr_as_int(), None, args_r, None, jitcode.calldescr) @arguments("cpu", "j", "R", returns="r") def bhimpl_inline_call_r_r(cpu, jitcode, args_r): + workaround2200.active = True return cpu.bh_call_r(jitcode.get_fnaddr_as_int(), None, args_r, None, jitcode.calldescr) @arguments("cpu", "j", "R") def bhimpl_inline_call_r_v(cpu, jitcode, args_r): + workaround2200.active = True return cpu.bh_call_v(jitcode.get_fnaddr_as_int(), None, args_r, None, jitcode.calldescr) @arguments("cpu", "j", "I", "R", returns="i") def bhimpl_inline_call_ir_i(cpu, jitcode, args_i, args_r): + workaround2200.active = True return cpu.bh_call_i(jitcode.get_fnaddr_as_int(), args_i, args_r, None, jitcode.calldescr) @arguments("cpu", "j", "I", "R", returns="r") def bhimpl_inline_call_ir_r(cpu, jitcode, args_i, args_r): + workaround2200.active = True return cpu.bh_call_r(jitcode.get_fnaddr_as_int(), args_i, args_r, None, jitcode.calldescr) @arguments("cpu", "j", "I", "R") def bhimpl_inline_call_ir_v(cpu, jitcode, args_i, args_r): + workaround2200.active = True return cpu.bh_call_v(jitcode.get_fnaddr_as_int(), args_i, args_r, None, jitcode.calldescr) @arguments("cpu", "j", "I", "R", "F", returns="i") def bhimpl_inline_call_irf_i(cpu, jitcode, args_i, args_r, args_f): + workaround2200.active = True return cpu.bh_call_i(jitcode.get_fnaddr_as_int(), args_i, args_r, args_f, jitcode.calldescr) @arguments("cpu", "j", "I", "R", "F", returns="r") def bhimpl_inline_call_irf_r(cpu, jitcode, args_i, args_r, args_f): + workaround2200.active = True return cpu.bh_call_r(jitcode.get_fnaddr_as_int(), args_i, args_r, args_f, jitcode.calldescr) @arguments("cpu", "j", "I", "R", "F", returns="f") def bhimpl_inline_call_irf_f(cpu, jitcode, args_i, args_r, args_f): + workaround2200.active = True return cpu.bh_call_f(jitcode.get_fnaddr_as_int(), args_i, args_r, args_f, jitcode.calldescr) @arguments("cpu", "j", "I", "R", "F") def bhimpl_inline_call_irf_v(cpu, jitcode, args_i, args_r, args_f): + workaround2200.active = True return cpu.bh_call_v(jitcode.get_fnaddr_as_int(), args_i, args_r, args_f, jitcode.calldescr) @@ -1523,6 +1543,8 @@ if not self.nextblackholeinterp: self._exit_frame_with_exception(current_exc) return current_exc + finally: + workaround2200.active = False # # pass the frame's return value to the caller caller = self.nextblackholeinterp @@ -1695,3 +1717,10 @@ # _run_forever(firstbh, current_exc) convert_and_run_from_pyjitpl._dont_inline_ = True + +# ____________________________________________________________ + +class WorkaroundIssue2200(object): + pass +workaround2200 = WorkaroundIssue2200() +workaround2200.active = False diff --git a/rpython/jit/metainterp/compile.py b/rpython/jit/metainterp/compile.py --- a/rpython/jit/metainterp/compile.py +++ b/rpython/jit/metainterp/compile.py @@ -615,35 +615,28 @@ final_descr = True class DoneWithThisFrameDescrVoid(_DoneWithThisFrameDescr): - def get_result(self, cpu, deadframe): - return None def handle_fail(self, deadframe, metainterp_sd, jitdriver_sd): assert jitdriver_sd.result_type == history.VOID raise jitexc.DoneWithThisFrameVoid() class DoneWithThisFrameDescrInt(_DoneWithThisFrameDescr): - def get_result(self, cpu, deadframe): - return cpu.get_int_value(deadframe, 0) def handle_fail(self, deadframe, metainterp_sd, jitdriver_sd): assert jitdriver_sd.result_type == history.INT - cpu = metainterp_sd.cpu - raise jitexc.DoneWithThisFrameInt(self.get_result(cpu, deadframe)) + result = metainterp_sd.cpu.get_int_value(deadframe, 0) + raise jitexc.DoneWithThisFrameInt(result) class DoneWithThisFrameDescrRef(_DoneWithThisFrameDescr): - def get_result(self, cpu, deadframe): - return cpu.get_ref_value(deadframe, 0) def handle_fail(self, deadframe, metainterp_sd, jitdriver_sd): assert jitdriver_sd.result_type == history.REF cpu = metainterp_sd.cpu - raise jitexc.DoneWithThisFrameRef(cpu, self.get_result(cpu, deadframe)) + result = cpu.get_ref_value(deadframe, 0) + raise jitexc.DoneWithThisFrameRef(cpu, result) class DoneWithThisFrameDescrFloat(_DoneWithThisFrameDescr): - def get_result(self, cpu, deadframe): - return cpu.get_float_value(deadframe, 0) def handle_fail(self, deadframe, metainterp_sd, jitdriver_sd): assert jitdriver_sd.result_type == history.FLOAT - cpu = metainterp_sd.cpu - raise jitexc.DoneWithThisFrameFloat(self.get_result(cpu, deadframe)) + result = metainterp_sd.cpu.get_float_value(deadframe, 0) + raise jitexc.DoneWithThisFrameFloat(result) class ExitFrameWithExceptionDescrRef(_DoneWithThisFrameDescr): def handle_fail(self, deadframe, metainterp_sd, jitdriver_sd): diff --git a/rpython/jit/metainterp/pyjitpl.py b/rpython/jit/metainterp/pyjitpl.py --- a/rpython/jit/metainterp/pyjitpl.py +++ b/rpython/jit/metainterp/pyjitpl.py @@ -2543,13 +2543,7 @@ elif box.type == history.REF: args.append(box.getref_base()) elif box.type == history.FLOAT: args.append(box.getfloatstorage()) else: assert 0 - res = self.jitdriver_sd.warmstate.execute_assembler(loop_token, *args) - kind = getkind(lltype.typeOf(res)) - if kind == history.VOID: raise DoneWithThisFrameVoid() - if kind == history.INT: raise DoneWithThisFrameInt(res) - if kind == history.REF: raise DoneWithThisFrameRef(self.cpu, res) - if kind == history.FLOAT: raise DoneWithThisFrameFloat(res) - raise AssertionError(kind) + self.jitdriver_sd.warmstate.execute_assembler(loop_token, *args) def prepare_resume_from_failure(self, deadframe, inputargs, resumedescr): exception = self.cpu.grab_exc_value(deadframe) diff --git a/rpython/jit/metainterp/warmspot.py b/rpython/jit/metainterp/warmspot.py --- a/rpython/jit/metainterp/warmspot.py +++ b/rpython/jit/metainterp/warmspot.py @@ -529,7 +529,7 @@ def make_enter_function(self, jd): from rpython.jit.metainterp.warmstate import WarmEnterState state = WarmEnterState(self, jd) - maybe_compile_and_run, EnterJitAssembler = state.make_entry_point() + maybe_compile_and_run = state.make_entry_point() jd.warmstate = state def crash_in_jit(e): @@ -560,7 +560,6 @@ maybe_enter_jit._always_inline_ = True jd._maybe_enter_jit_fn = maybe_enter_jit jd._maybe_compile_and_run_fn = maybe_compile_and_run - jd._EnterJitAssembler = EnterJitAssembler def make_driverhook_graphs(self): s_Str = annmodel.SomeString() @@ -834,7 +833,6 @@ # more stuff # # to that: - # XXX there are too many exceptions all around... # # def original_portal(..): # stuff @@ -843,14 +841,7 @@ # def portal_runner(*args): # while 1: # try: - # try: - # return portal(*args) - # except EnterJitAssembler, e: - # while True: - # try: - # return e.execute() - # except EnterJitAssembler, e: - # continue + # return portal(*args) # except ContinueRunningNormally, e: # *args = *e.new_args # except DoneWithThisFrame, e: @@ -893,38 +884,29 @@ rtyper = self.translator.rtyper RESULT = PORTALFUNC.RESULT result_kind = history.getkind(RESULT) - assert result_kind.startswith(jd.result_type) ts = self.cpu.ts state = jd.warmstate maybe_compile_and_run = jd._maybe_compile_and_run_fn - EnterJitAssembler = jd._EnterJitAssembler def ll_portal_runner(*args): start = True while 1: try: - try: - # maybe enter from the function's start. Note that the - # 'start' variable is constant-folded away because it's - # the first statement in the loop. - if start: - maybe_compile_and_run( - state.increment_function_threshold, *args) - # - # then run the normal portal function, i.e. the - # interpreter's main loop. It might enter the jit - # via maybe_enter_jit(), which typically ends with - # handle_fail() being called, which raises on the - # following exceptions --- catched here, because we - # want to interrupt the whole interpreter loop. - return support.maybe_on_top_of_llinterp(rtyper, - portal_ptr)(*args) - except EnterJitAssembler as e: - while True: - try: - return e.execute() - except EnterJitAssembler as e: - continue + # maybe enter from the function's start. Note that the + # 'start' variable is constant-folded away because it's + # the first statement in the loop. + if start: + maybe_compile_and_run( + state.increment_function_threshold, *args) + # + # then run the normal portal function, i.e. the + # interpreter's main loop. It might enter the jit + # via maybe_enter_jit(), which typically ends with + # handle_fail() being called, which raises on the + # following exceptions --- catched here, because we + # want to interrupt the whole interpreter loop. + return support.maybe_on_top_of_llinterp(rtyper, + portal_ptr)(*args) except jitexc.ContinueRunningNormally as e: args = () for ARGTYPE, attrname, count in portalfunc_ARGS: @@ -951,7 +933,6 @@ raise LLException(ts.get_typeptr(value), value) else: value = cast_base_ptr_to_instance(Exception, value) - assert value is not None raise value def handle_jitexception(e): diff --git a/rpython/jit/metainterp/warmstate.py b/rpython/jit/metainterp/warmstate.py --- a/rpython/jit/metainterp/warmstate.py +++ b/rpython/jit/metainterp/warmstate.py @@ -2,7 +2,7 @@ import weakref from rpython.jit.codewriter import support, heaptracker, longlong -from rpython.jit.metainterp import resoperation, history, jitexc +from rpython.jit.metainterp import resoperation, history from rpython.rlib.debug import debug_start, debug_stop, debug_print from rpython.rlib.debug import have_debug_prints_for from rpython.rlib.jit import PARAMETERS @@ -348,8 +348,8 @@ def make_entry_point(self): "NOT_RPYTHON" - if hasattr(self, 'entry_point_fns'): - return self.entry_point_fns + if hasattr(self, 'maybe_compile_and_run'): + return self.maybe_compile_and_run warmrunnerdesc = self.warmrunnerdesc metainterp_sd = warmrunnerdesc.metainterp_sd @@ -362,8 +362,6 @@ confirm_enter_jit = self.confirm_enter_jit range_red_args = unrolling_iterable( range(num_green_args, num_green_args + jitdriver_sd.num_red_args)) - name_red_args = unrolling_iterable( - [(i, 'arg%d' % i) for i in range(jitdriver_sd.num_red_args)]) # get a new specialized copy of the method ARGS = [] for kind in jitdriver_sd.red_args_types: @@ -379,18 +377,6 @@ cpu = self.cpu jitcounter = self.warmrunnerdesc.jitcounter - result_type = jitdriver_sd.result_type - if result_type == history.VOID: - _DoneWithThisFrameCls = jitexc.DoneWithThisFrameVoid - elif result_type == history.INT: - _DoneWithThisFrameCls = jitexc.DoneWithThisFrameInt - elif result_type == history.REF: - _DoneWithThisFrameCls = jitexc.DoneWithThisFrameRef - elif result_type == history.FLOAT: - _DoneWithThisFrameCls = jitexc.DoneWithThisFrameFloat - else: - raise AssertionError(result_type) - def execute_assembler(loop_token, *args): # Call the backend to run the 'looptoken' with the given # input args. @@ -410,13 +396,9 @@ # # Handle the failure fail_descr = cpu.get_latest_descr(deadframe) - if isinstance(fail_descr, _DoneWithThisFrameCls): - # A fast path to avoid raising and immediately catching - # a DoneWithThisFrame exception - return fail_descr.get_result(cpu, deadframe) - else: - fail_descr.handle_fail(deadframe, metainterp_sd, jitdriver_sd) - assert 0, "should have raised" + fail_descr.handle_fail(deadframe, metainterp_sd, jitdriver_sd) + # + assert 0, "should have raised" def bound_reached(hash, cell, *args): if not confirm_enter_jit(*args): @@ -437,8 +419,7 @@ def maybe_compile_and_run(increment_threshold, *args): """Entry point to the JIT. Called at the point with the - can_enter_jit() hint, and at the start of a function - with a different threshold. + can_enter_jit() hint. """ # Look for the cell corresponding to the current greenargs. # Search for the JitCell that is of the correct subclass of @@ -458,6 +439,14 @@ bound_reached(hash, None, *args) return + # Workaround for issue #2200, maybe temporary. This is not + # a proper fix, but only a hack that should work well enough + # for PyPy's main jitdriver... See test_issue2200_recursion + from rpython.jit.metainterp.blackhole import workaround2200 + if workaround2200.active: + workaround2200.active = False + return + # Here, we have found 'cell'. # if cell.flags & (JC_TRACING | JC_TEMPORARY): @@ -495,27 +484,14 @@ execute_args = () for i in range_red_args: execute_args += (unspecialize_value(args[i]), ) - # run it, but from outside in ll_portal_runner, not from here - # (this avoids RPython-level recursion with no corresponding - # app-level recursion, as shown by issues 2200 and 2335) - raise EnterJitAssembler(procedure_token, *execute_args) - - class EnterJitAssembler(jitexc.JitException): - def __init__(self, procedure_token, *args): - self.procedure_token = procedure_token - for i, argname in name_red_args: - setattr(self, argname, args[i]) - def execute(self): - args = () - for i, argname in name_red_args: - args += (getattr(self, argname), ) - return execute_assembler(self.procedure_token, *args) + # run it! this executes until interrupted by an exception + execute_assembler(procedure_token, *execute_args) + assert 0, "should not reach this point" maybe_compile_and_run._dont_inline_ = True + self.maybe_compile_and_run = maybe_compile_and_run self.execute_assembler = execute_assembler - self.entry_point_fns = (maybe_compile_and_run, - EnterJitAssembler) - return self.entry_point_fns + return maybe_compile_and_run # ---------- From pypy.commits at gmail.com Sat Jul 9 10:07:48 2016 From: pypy.commits at gmail.com (arigo) Date: Sat, 09 Jul 2016 07:07:48 -0700 (PDT) Subject: [pypy-commit] pypy issue2335: A branch for redoing 660e49cca2a1 Message-ID: <57810534.cdcf1c0a.5e963.ad90@mx.google.com> Author: Armin Rigo Branch: issue2335 Changeset: r85621:ee3c1e707183 Date: 2016-07-09 14:37 +0200 http://bitbucket.org/pypy/pypy/changeset/ee3c1e707183/ Log: A branch for redoing 660e49cca2a1 From pypy.commits at gmail.com Sat Jul 9 10:07:51 2016 From: pypy.commits at gmail.com (arigo) Date: Sat, 09 Jul 2016 07:07:51 -0700 (PDT) Subject: [pypy-commit] pypy issue2335: Un-backout 660e49cca2a1 Message-ID: <57810537.17a71c0a.97ab0.2117@mx.google.com> Author: Armin Rigo Branch: issue2335 Changeset: r85622:3c83208ccb4a Date: 2016-07-09 14:54 +0200 http://bitbucket.org/pypy/pypy/changeset/3c83208ccb4a/ Log: Un-backout 660e49cca2a1 diff --git a/rpython/jit/metainterp/blackhole.py b/rpython/jit/metainterp/blackhole.py --- a/rpython/jit/metainterp/blackhole.py +++ b/rpython/jit/metainterp/blackhole.py @@ -1143,45 +1143,35 @@ @arguments("cpu", "i", "R", "d", returns="i") def bhimpl_residual_call_r_i(cpu, func, args_r, calldescr): - workaround2200.active = True return cpu.bh_call_i(func, None, args_r, None, calldescr) @arguments("cpu", "i", "R", "d", returns="r") def bhimpl_residual_call_r_r(cpu, func, args_r, calldescr): - workaround2200.active = True return cpu.bh_call_r(func, None, args_r, None, calldescr) @arguments("cpu", "i", "R", "d") def bhimpl_residual_call_r_v(cpu, func, args_r, calldescr): - workaround2200.active = True return cpu.bh_call_v(func, None, args_r, None, calldescr) @arguments("cpu", "i", "I", "R", "d", returns="i") def bhimpl_residual_call_ir_i(cpu, func, args_i, args_r, calldescr): - workaround2200.active = True return cpu.bh_call_i(func, args_i, args_r, None, calldescr) @arguments("cpu", "i", "I", "R", "d", returns="r") def bhimpl_residual_call_ir_r(cpu, func, args_i, args_r, calldescr): - workaround2200.active = True return cpu.bh_call_r(func, args_i, args_r, None, calldescr) @arguments("cpu", "i", "I", "R", "d") def bhimpl_residual_call_ir_v(cpu, func, args_i, args_r, calldescr): - workaround2200.active = True return cpu.bh_call_v(func, args_i, args_r, None, calldescr) @arguments("cpu", "i", "I", "R", "F", "d", returns="i") def bhimpl_residual_call_irf_i(cpu, func, args_i,args_r,args_f,calldescr): - workaround2200.active = True return cpu.bh_call_i(func, args_i, args_r, args_f, calldescr) @arguments("cpu", "i", "I", "R", "F", "d", returns="r") def bhimpl_residual_call_irf_r(cpu, func, args_i,args_r,args_f,calldescr): - workaround2200.active = True return cpu.bh_call_r(func, args_i, args_r, args_f, calldescr) @arguments("cpu", "i", "I", "R", "F", "d", returns="f") def bhimpl_residual_call_irf_f(cpu, func, args_i,args_r,args_f,calldescr): - workaround2200.active = True return cpu.bh_call_f(func, args_i, args_r, args_f, calldescr) @arguments("cpu", "i", "I", "R", "F", "d") def bhimpl_residual_call_irf_v(cpu, func, args_i,args_r,args_f,calldescr): - workaround2200.active = True return cpu.bh_call_v(func, args_i, args_r, args_f, calldescr) # conditional calls - note that they cannot return stuff @@ -1209,54 +1199,44 @@ @arguments("cpu", "j", "R", returns="i") def bhimpl_inline_call_r_i(cpu, jitcode, args_r): - workaround2200.active = True return cpu.bh_call_i(jitcode.get_fnaddr_as_int(), None, args_r, None, jitcode.calldescr) @arguments("cpu", "j", "R", returns="r") def bhimpl_inline_call_r_r(cpu, jitcode, args_r): - workaround2200.active = True return cpu.bh_call_r(jitcode.get_fnaddr_as_int(), None, args_r, None, jitcode.calldescr) @arguments("cpu", "j", "R") def bhimpl_inline_call_r_v(cpu, jitcode, args_r): - workaround2200.active = True return cpu.bh_call_v(jitcode.get_fnaddr_as_int(), None, args_r, None, jitcode.calldescr) @arguments("cpu", "j", "I", "R", returns="i") def bhimpl_inline_call_ir_i(cpu, jitcode, args_i, args_r): - workaround2200.active = True return cpu.bh_call_i(jitcode.get_fnaddr_as_int(), args_i, args_r, None, jitcode.calldescr) @arguments("cpu", "j", "I", "R", returns="r") def bhimpl_inline_call_ir_r(cpu, jitcode, args_i, args_r): - workaround2200.active = True return cpu.bh_call_r(jitcode.get_fnaddr_as_int(), args_i, args_r, None, jitcode.calldescr) @arguments("cpu", "j", "I", "R") def bhimpl_inline_call_ir_v(cpu, jitcode, args_i, args_r): - workaround2200.active = True return cpu.bh_call_v(jitcode.get_fnaddr_as_int(), args_i, args_r, None, jitcode.calldescr) @arguments("cpu", "j", "I", "R", "F", returns="i") def bhimpl_inline_call_irf_i(cpu, jitcode, args_i, args_r, args_f): - workaround2200.active = True return cpu.bh_call_i(jitcode.get_fnaddr_as_int(), args_i, args_r, args_f, jitcode.calldescr) @arguments("cpu", "j", "I", "R", "F", returns="r") def bhimpl_inline_call_irf_r(cpu, jitcode, args_i, args_r, args_f): - workaround2200.active = True return cpu.bh_call_r(jitcode.get_fnaddr_as_int(), args_i, args_r, args_f, jitcode.calldescr) @arguments("cpu", "j", "I", "R", "F", returns="f") def bhimpl_inline_call_irf_f(cpu, jitcode, args_i, args_r, args_f): - workaround2200.active = True return cpu.bh_call_f(jitcode.get_fnaddr_as_int(), args_i, args_r, args_f, jitcode.calldescr) @arguments("cpu", "j", "I", "R", "F") def bhimpl_inline_call_irf_v(cpu, jitcode, args_i, args_r, args_f): - workaround2200.active = True return cpu.bh_call_v(jitcode.get_fnaddr_as_int(), args_i, args_r, args_f, jitcode.calldescr) @@ -1543,8 +1523,6 @@ if not self.nextblackholeinterp: self._exit_frame_with_exception(current_exc) return current_exc - finally: - workaround2200.active = False # # pass the frame's return value to the caller caller = self.nextblackholeinterp @@ -1717,10 +1695,3 @@ # _run_forever(firstbh, current_exc) convert_and_run_from_pyjitpl._dont_inline_ = True - -# ____________________________________________________________ - -class WorkaroundIssue2200(object): - pass -workaround2200 = WorkaroundIssue2200() -workaround2200.active = False diff --git a/rpython/jit/metainterp/warmspot.py b/rpython/jit/metainterp/warmspot.py --- a/rpython/jit/metainterp/warmspot.py +++ b/rpython/jit/metainterp/warmspot.py @@ -529,7 +529,7 @@ def make_enter_function(self, jd): from rpython.jit.metainterp.warmstate import WarmEnterState state = WarmEnterState(self, jd) - maybe_compile_and_run = state.make_entry_point() + maybe_compile_and_run, EnterJitAssembler = state.make_entry_point() jd.warmstate = state def crash_in_jit(e): @@ -560,6 +560,7 @@ maybe_enter_jit._always_inline_ = True jd._maybe_enter_jit_fn = maybe_enter_jit jd._maybe_compile_and_run_fn = maybe_compile_and_run + jd._EnterJitAssembler = EnterJitAssembler def make_driverhook_graphs(self): s_Str = annmodel.SomeString() @@ -833,6 +834,7 @@ # more stuff # # to that: + # XXX there are too many exceptions all around... # # def original_portal(..): # stuff @@ -841,7 +843,14 @@ # def portal_runner(*args): # while 1: # try: - # return portal(*args) + # try: + # return portal(*args) + # except EnterJitAssembler, e: + # while True: + # try: + # return e.execute() + # except EnterJitAssembler, e: + # continue # except ContinueRunningNormally, e: # *args = *e.new_args # except DoneWithThisFrame, e: @@ -887,26 +896,35 @@ ts = self.cpu.ts state = jd.warmstate maybe_compile_and_run = jd._maybe_compile_and_run_fn + EnterJitAssembler = jd._EnterJitAssembler def ll_portal_runner(*args): start = True while 1: try: - # maybe enter from the function's start. Note that the - # 'start' variable is constant-folded away because it's - # the first statement in the loop. - if start: - maybe_compile_and_run( - state.increment_function_threshold, *args) - # - # then run the normal portal function, i.e. the - # interpreter's main loop. It might enter the jit - # via maybe_enter_jit(), which typically ends with - # handle_fail() being called, which raises on the - # following exceptions --- catched here, because we - # want to interrupt the whole interpreter loop. - return support.maybe_on_top_of_llinterp(rtyper, - portal_ptr)(*args) + try: + # maybe enter from the function's start. Note that the + # 'start' variable is constant-folded away because it's + # the first statement in the loop. + if start: + maybe_compile_and_run( + state.increment_function_threshold, *args) + # + # then run the normal portal function, i.e. the + # interpreter's main loop. It might enter the jit + # via maybe_enter_jit(), which typically ends with + # handle_fail() being called, which raises on the + # following exceptions --- catched here, because we + # want to interrupt the whole interpreter loop. + return support.maybe_on_top_of_llinterp(rtyper, + portal_ptr)(*args) + except EnterJitAssembler as e: + while True: + try: + e.execute() + except EnterJitAssembler as e: + continue + assert 0 # execute() should always raise for now except jitexc.ContinueRunningNormally as e: args = () for ARGTYPE, attrname, count in portalfunc_ARGS: @@ -933,6 +951,7 @@ raise LLException(ts.get_typeptr(value), value) else: value = cast_base_ptr_to_instance(Exception, value) + assert value is not None raise value def handle_jitexception(e): diff --git a/rpython/jit/metainterp/warmstate.py b/rpython/jit/metainterp/warmstate.py --- a/rpython/jit/metainterp/warmstate.py +++ b/rpython/jit/metainterp/warmstate.py @@ -2,7 +2,7 @@ import weakref from rpython.jit.codewriter import support, heaptracker, longlong -from rpython.jit.metainterp import resoperation, history +from rpython.jit.metainterp import resoperation, history, jitexc from rpython.rlib.debug import debug_start, debug_stop, debug_print from rpython.rlib.debug import have_debug_prints_for from rpython.rlib.jit import PARAMETERS @@ -348,8 +348,8 @@ def make_entry_point(self): "NOT_RPYTHON" - if hasattr(self, 'maybe_compile_and_run'): - return self.maybe_compile_and_run + if hasattr(self, 'entry_point_fns'): + return self.entry_point_fns warmrunnerdesc = self.warmrunnerdesc metainterp_sd = warmrunnerdesc.metainterp_sd @@ -362,6 +362,8 @@ confirm_enter_jit = self.confirm_enter_jit range_red_args = unrolling_iterable( range(num_green_args, num_green_args + jitdriver_sd.num_red_args)) + name_red_args = unrolling_iterable( + [(i, 'arg%d' % i) for i in range(jitdriver_sd.num_red_args)]) # get a new specialized copy of the method ARGS = [] for kind in jitdriver_sd.red_args_types: @@ -419,7 +421,8 @@ def maybe_compile_and_run(increment_threshold, *args): """Entry point to the JIT. Called at the point with the - can_enter_jit() hint. + can_enter_jit() hint, and at the start of a function + with a different threshold. """ # Look for the cell corresponding to the current greenargs. # Search for the JitCell that is of the correct subclass of @@ -439,14 +442,6 @@ bound_reached(hash, None, *args) return - # Workaround for issue #2200, maybe temporary. This is not - # a proper fix, but only a hack that should work well enough - # for PyPy's main jitdriver... See test_issue2200_recursion - from rpython.jit.metainterp.blackhole import workaround2200 - if workaround2200.active: - workaround2200.active = False - return - # Here, we have found 'cell'. # if cell.flags & (JC_TRACING | JC_TEMPORARY): @@ -484,14 +479,27 @@ execute_args = () for i in range_red_args: execute_args += (unspecialize_value(args[i]), ) - # run it! this executes until interrupted by an exception - execute_assembler(procedure_token, *execute_args) - assert 0, "should not reach this point" + # run it, but from outside in ll_portal_runner, not from here + # (this avoids RPython-level recursion with no corresponding + # app-level recursion, as shown by issues 2200 and 2335) + raise EnterJitAssembler(procedure_token, *execute_args) + + class EnterJitAssembler(jitexc.JitException): + def __init__(self, procedure_token, *args): + self.procedure_token = procedure_token + for i, argname in name_red_args: + setattr(self, argname, args[i]) + def execute(self): + args = () + for i, argname in name_red_args: + args += (getattr(self, argname), ) + return execute_assembler(self.procedure_token, *args) maybe_compile_and_run._dont_inline_ = True - self.maybe_compile_and_run = maybe_compile_and_run self.execute_assembler = execute_assembler - return maybe_compile_and_run + self.entry_point_fns = (maybe_compile_and_run, + EnterJitAssembler) + return self.entry_point_fns # ---------- From pypy.commits at gmail.com Sat Jul 9 10:07:53 2016 From: pypy.commits at gmail.com (arigo) Date: Sat, 09 Jul 2016 07:07:53 -0700 (PDT) Subject: [pypy-commit] pypy issue2335: Un-backout 5440c8b2d979 Message-ID: <57810539.e152c20a.9ca18.ffffb3c7@mx.google.com> Author: Armin Rigo Branch: issue2335 Changeset: r85623:7667ac88c004 Date: 2016-07-09 14:59 +0200 http://bitbucket.org/pypy/pypy/changeset/7667ac88c004/ Log: Un-backout 5440c8b2d979 diff --git a/rpython/jit/metainterp/compile.py b/rpython/jit/metainterp/compile.py --- a/rpython/jit/metainterp/compile.py +++ b/rpython/jit/metainterp/compile.py @@ -615,28 +615,35 @@ final_descr = True class DoneWithThisFrameDescrVoid(_DoneWithThisFrameDescr): + def get_result(self, cpu, deadframe): + return None def handle_fail(self, deadframe, metainterp_sd, jitdriver_sd): assert jitdriver_sd.result_type == history.VOID raise jitexc.DoneWithThisFrameVoid() class DoneWithThisFrameDescrInt(_DoneWithThisFrameDescr): + def get_result(self, cpu, deadframe): + return cpu.get_int_value(deadframe, 0) def handle_fail(self, deadframe, metainterp_sd, jitdriver_sd): assert jitdriver_sd.result_type == history.INT - result = metainterp_sd.cpu.get_int_value(deadframe, 0) - raise jitexc.DoneWithThisFrameInt(result) + cpu = metainterp_sd.cpu + raise jitexc.DoneWithThisFrameInt(self.get_result(cpu, deadframe)) class DoneWithThisFrameDescrRef(_DoneWithThisFrameDescr): + def get_result(self, cpu, deadframe): + return cpu.get_ref_value(deadframe, 0) def handle_fail(self, deadframe, metainterp_sd, jitdriver_sd): assert jitdriver_sd.result_type == history.REF cpu = metainterp_sd.cpu - result = cpu.get_ref_value(deadframe, 0) - raise jitexc.DoneWithThisFrameRef(cpu, result) + raise jitexc.DoneWithThisFrameRef(cpu, self.get_result(cpu, deadframe)) class DoneWithThisFrameDescrFloat(_DoneWithThisFrameDescr): + def get_result(self, cpu, deadframe): + return cpu.get_float_value(deadframe, 0) def handle_fail(self, deadframe, metainterp_sd, jitdriver_sd): assert jitdriver_sd.result_type == history.FLOAT - result = metainterp_sd.cpu.get_float_value(deadframe, 0) - raise jitexc.DoneWithThisFrameFloat(result) + cpu = metainterp_sd.cpu + raise jitexc.DoneWithThisFrameFloat(self.get_result(cpu, deadframe)) class ExitFrameWithExceptionDescrRef(_DoneWithThisFrameDescr): def handle_fail(self, deadframe, metainterp_sd, jitdriver_sd): diff --git a/rpython/jit/metainterp/pyjitpl.py b/rpython/jit/metainterp/pyjitpl.py --- a/rpython/jit/metainterp/pyjitpl.py +++ b/rpython/jit/metainterp/pyjitpl.py @@ -2543,7 +2543,13 @@ elif box.type == history.REF: args.append(box.getref_base()) elif box.type == history.FLOAT: args.append(box.getfloatstorage()) else: assert 0 - self.jitdriver_sd.warmstate.execute_assembler(loop_token, *args) + res = self.jitdriver_sd.warmstate.execute_assembler(loop_token, *args) + kind = getkind(lltype.typeOf(res)) + if kind == history.VOID: raise DoneWithThisFrameVoid() + if kind == history.INT: raise DoneWithThisFrameInt(res) + if kind == history.REF: raise DoneWithThisFrameRef(self.cpu, res) + if kind == history.FLOAT: raise DoneWithThisFrameFloat(res) + raise AssertionError(kind) def prepare_resume_from_failure(self, deadframe, inputargs, resumedescr): exception = self.cpu.grab_exc_value(deadframe) diff --git a/rpython/jit/metainterp/warmspot.py b/rpython/jit/metainterp/warmspot.py --- a/rpython/jit/metainterp/warmspot.py +++ b/rpython/jit/metainterp/warmspot.py @@ -893,6 +893,7 @@ rtyper = self.translator.rtyper RESULT = PORTALFUNC.RESULT result_kind = history.getkind(RESULT) + assert result_kind.startswith(jd.result_type) ts = self.cpu.ts state = jd.warmstate maybe_compile_and_run = jd._maybe_compile_and_run_fn @@ -921,10 +922,9 @@ except EnterJitAssembler as e: while True: try: - e.execute() + return e.execute() except EnterJitAssembler as e: continue - assert 0 # execute() should always raise for now except jitexc.ContinueRunningNormally as e: args = () for ARGTYPE, attrname, count in portalfunc_ARGS: diff --git a/rpython/jit/metainterp/warmstate.py b/rpython/jit/metainterp/warmstate.py --- a/rpython/jit/metainterp/warmstate.py +++ b/rpython/jit/metainterp/warmstate.py @@ -379,6 +379,18 @@ cpu = self.cpu jitcounter = self.warmrunnerdesc.jitcounter + result_type = jitdriver_sd.result_type + if result_type == history.VOID: + _DoneWithThisFrameCls = jitexc.DoneWithThisFrameVoid + elif result_type == history.INT: + _DoneWithThisFrameCls = jitexc.DoneWithThisFrameInt + elif result_type == history.REF: + _DoneWithThisFrameCls = jitexc.DoneWithThisFrameRef + elif result_type == history.FLOAT: + _DoneWithThisFrameCls = jitexc.DoneWithThisFrameFloat + else: + raise AssertionError(result_type) + def execute_assembler(loop_token, *args): # Call the backend to run the 'looptoken' with the given # input args. @@ -398,9 +410,13 @@ # # Handle the failure fail_descr = cpu.get_latest_descr(deadframe) - fail_descr.handle_fail(deadframe, metainterp_sd, jitdriver_sd) - # - assert 0, "should have raised" + if isinstance(fail_descr, _DoneWithThisFrameCls): + # A fast path to avoid raising and immediately catching + # a DoneWithThisFrame exception + return fail_descr.get_result(cpu, deadframe) + else: + fail_descr.handle_fail(deadframe, metainterp_sd, jitdriver_sd) + assert 0, "should have raised" def bound_reached(hash, cell, *args): if not confirm_enter_jit(*args): From pypy.commits at gmail.com Sat Jul 9 10:07:56 2016 From: pypy.commits at gmail.com (arigo) Date: Sat, 09 Jul 2016 07:07:56 -0700 (PDT) Subject: [pypy-commit] pypy issue2335: oops, nonsense (checked the wrong class) Message-ID: <5781053c.e457c20a.55c3c.ffffe6bf@mx.google.com> Author: Armin Rigo Branch: issue2335 Changeset: r85624:a03bc876b025 Date: 2016-07-09 15:24 +0200 http://bitbucket.org/pypy/pypy/changeset/a03bc876b025/ Log: oops, nonsense (checked the wrong class) diff --git a/rpython/jit/metainterp/warmstate.py b/rpython/jit/metainterp/warmstate.py --- a/rpython/jit/metainterp/warmstate.py +++ b/rpython/jit/metainterp/warmstate.py @@ -348,6 +348,7 @@ def make_entry_point(self): "NOT_RPYTHON" + from rpython.jit.metainterp import compile if hasattr(self, 'entry_point_fns'): return self.entry_point_fns @@ -378,18 +379,7 @@ func_execute_token = self.cpu.make_execute_token(*ARGS) cpu = self.cpu jitcounter = self.warmrunnerdesc.jitcounter - result_type = jitdriver_sd.result_type - if result_type == history.VOID: - _DoneWithThisFrameCls = jitexc.DoneWithThisFrameVoid - elif result_type == history.INT: - _DoneWithThisFrameCls = jitexc.DoneWithThisFrameInt - elif result_type == history.REF: - _DoneWithThisFrameCls = jitexc.DoneWithThisFrameRef - elif result_type == history.FLOAT: - _DoneWithThisFrameCls = jitexc.DoneWithThisFrameFloat - else: - raise AssertionError(result_type) def execute_assembler(loop_token, *args): # Call the backend to run the 'looptoken' with the given @@ -410,13 +400,24 @@ # # Handle the failure fail_descr = cpu.get_latest_descr(deadframe) - if isinstance(fail_descr, _DoneWithThisFrameCls): - # A fast path to avoid raising and immediately catching - # a DoneWithThisFrame exception - return fail_descr.get_result(cpu, deadframe) - else: - fail_descr.handle_fail(deadframe, metainterp_sd, jitdriver_sd) - assert 0, "should have raised" + # First, a fast path to avoid raising and immediately catching + # a DoneWithThisFrame exception + if result_type == history.VOID: + if isinstance(fail_descr, compile.DoneWithThisFrameDescrVoid): + return None + if result_type == history.INT: + if isinstance(fail_descr, compile.DoneWithThisFrameDescrInt): + return fail_descr.get_result() + if result_type == history.REF: + if isinstance(fail_descr, compile.DoneWithThisFrameDescrRef): + return fail_descr.get_result() + if result_type == history.FLOAT: + if isinstance(fail_descr, compile.DoneWithThisFrameDescrFloat): + return fail_descr.get_result() + # + # General case + fail_descr.handle_fail(deadframe, metainterp_sd, jitdriver_sd) + assert 0, "should have raised" def bound_reached(hash, cell, *args): if not confirm_enter_jit(*args): From pypy.commits at gmail.com Sat Jul 9 10:07:58 2016 From: pypy.commits at gmail.com (arigo) Date: Sat, 09 Jul 2016 07:07:58 -0700 (PDT) Subject: [pypy-commit] pypy issue2335: Merge the two copies of the big "except:" chain Message-ID: <5781053e.2457c20a.b9df2.ffffd30a@mx.google.com> Author: Armin Rigo Branch: issue2335 Changeset: r85625:175cf7b5a040 Date: 2016-07-09 15:24 +0200 http://bitbucket.org/pypy/pypy/changeset/175cf7b5a040/ Log: Merge the two copies of the big "except:" chain diff --git a/rpython/jit/metainterp/compile.py b/rpython/jit/metainterp/compile.py --- a/rpython/jit/metainterp/compile.py +++ b/rpython/jit/metainterp/compile.py @@ -615,8 +615,6 @@ final_descr = True class DoneWithThisFrameDescrVoid(_DoneWithThisFrameDescr): - def get_result(self, cpu, deadframe): - return None def handle_fail(self, deadframe, metainterp_sd, jitdriver_sd): assert jitdriver_sd.result_type == history.VOID raise jitexc.DoneWithThisFrameVoid() diff --git a/rpython/jit/metainterp/warmspot.py b/rpython/jit/metainterp/warmspot.py --- a/rpython/jit/metainterp/warmspot.py +++ b/rpython/jit/metainterp/warmspot.py @@ -900,52 +900,62 @@ EnterJitAssembler = jd._EnterJitAssembler def ll_portal_runner(*args): - start = True - while 1: - try: + try: + # maybe enter from the function's start. + maybe_compile_and_run( + state.increment_function_threshold, *args) + # + # then run the normal portal function, i.e. the + # interpreter's main loop. It might enter the jit + # via maybe_enter_jit(), which typically ends with + # handle_fail() being called, which raises on the + # following exceptions --- catched here, because we + # want to interrupt the whole interpreter loop. + return support.maybe_on_top_of_llinterp(rtyper, + portal_ptr)(*args) + except jitexc.JitException as e: + result = handle_jitexception(e) + if result_kind != 'void': + result = specialize_value(RESULT, result) + return result + + def handle_jitexception(e): + while True: + if isinstance(e, EnterJitAssembler): try: - # maybe enter from the function's start. Note that the - # 'start' variable is constant-folded away because it's - # the first statement in the loop. - if start: - maybe_compile_and_run( - state.increment_function_threshold, *args) - # - # then run the normal portal function, i.e. the - # interpreter's main loop. It might enter the jit - # via maybe_enter_jit(), which typically ends with - # handle_fail() being called, which raises on the - # following exceptions --- catched here, because we - # want to interrupt the whole interpreter loop. - return support.maybe_on_top_of_llinterp(rtyper, - portal_ptr)(*args) - except EnterJitAssembler as e: - while True: - try: - return e.execute() - except EnterJitAssembler as e: - continue - except jitexc.ContinueRunningNormally as e: + return e.execute() + except jitexc.JitException as e: + continue + # + if isinstance(e, jitexc.ContinueRunningNormally): args = () for ARGTYPE, attrname, count in portalfunc_ARGS: x = getattr(e, attrname)[count] x = specialize_value(ARGTYPE, x) args = args + (x,) - start = False - continue - except jitexc.DoneWithThisFrameVoid: - assert result_kind == 'void' - return - except jitexc.DoneWithThisFrameInt as e: - assert result_kind == 'int' - return specialize_value(RESULT, e.result) - except jitexc.DoneWithThisFrameRef as e: - assert result_kind == 'ref' - return specialize_value(RESULT, e.result) - except jitexc.DoneWithThisFrameFloat as e: - assert result_kind == 'float' - return specialize_value(RESULT, e.result) - except jitexc.ExitFrameWithExceptionRef as e: + try: + result = support.maybe_on_top_of_llinterp(rtyper, + portal_ptr)(*args) + except jitexc.JitException as e: + continue + if result_kind != 'void': + result = unspecialize_value(result) + return result + # + if result_kind == 'void': + if isinstance(e, jitexc.DoneWithThisFrameVoid): + return None + if result_kind == 'int': + if isinstance(e, jitexc.DoneWithThisFrameInt): + return e.result + if result_kind == 'ref': + if isinstance(e, jitexc.DoneWithThisFrameRef): + return e.result + if result_kind == 'float': + if isinstance(e, jitexc.DoneWithThisFrameFloat): + return e.result + # + if isinstance(e, jitexc.ExitFrameWithExceptionRef): value = ts.cast_to_baseclass(e.value) if not we_are_translated(): raise LLException(ts.get_typeptr(value), value) @@ -953,40 +963,8 @@ value = cast_base_ptr_to_instance(Exception, value) assert value is not None raise value - - def handle_jitexception(e): - # XXX the bulk of this function is mostly a copy-paste from above - try: - raise e - except jitexc.ContinueRunningNormally as e: - args = () - for ARGTYPE, attrname, count in portalfunc_ARGS: - x = getattr(e, attrname)[count] - x = specialize_value(ARGTYPE, x) - args = args + (x,) - result = ll_portal_runner(*args) - if result_kind != 'void': - result = unspecialize_value(result) - return result - except jitexc.DoneWithThisFrameVoid: - assert result_kind == 'void' - return - except jitexc.DoneWithThisFrameInt as e: - assert result_kind == 'int' - return e.result - except jitexc.DoneWithThisFrameRef as e: - assert result_kind == 'ref' - return e.result - except jitexc.DoneWithThisFrameFloat as e: - assert result_kind == 'float' - return e.result - except jitexc.ExitFrameWithExceptionRef as e: - value = ts.cast_to_baseclass(e.value) - if not we_are_translated(): - raise LLException(ts.get_typeptr(value), value) - else: - value = cast_base_ptr_to_instance(Exception, value) - raise value + # + raise AssertionError("all cases should have been handled") jd._ll_portal_runner = ll_portal_runner # for debugging jd.portal_runner_ptr = self.helper_func(jd._PTR_PORTAL_FUNCTYPE, From pypy.commits at gmail.com Sat Jul 9 10:08:00 2016 From: pypy.commits at gmail.com (arigo) Date: Sat, 09 Jul 2016 07:08:00 -0700 (PDT) Subject: [pypy-commit] pypy issue2335: Fixes, comments Message-ID: <57810540.cf981c0a.2afd4.a6c9@mx.google.com> Author: Armin Rigo Branch: issue2335 Changeset: r85626:790e1806bcc0 Date: 2016-07-09 15:31 +0200 http://bitbucket.org/pypy/pypy/changeset/790e1806bcc0/ Log: Fixes, comments diff --git a/rpython/jit/metainterp/pyjitpl.py b/rpython/jit/metainterp/pyjitpl.py --- a/rpython/jit/metainterp/pyjitpl.py +++ b/rpython/jit/metainterp/pyjitpl.py @@ -2544,11 +2544,11 @@ elif box.type == history.FLOAT: args.append(box.getfloatstorage()) else: assert 0 res = self.jitdriver_sd.warmstate.execute_assembler(loop_token, *args) - kind = getkind(lltype.typeOf(res)) - if kind == history.VOID: raise DoneWithThisFrameVoid() - if kind == history.INT: raise DoneWithThisFrameInt(res) - if kind == history.REF: raise DoneWithThisFrameRef(self.cpu, res) - if kind == history.FLOAT: raise DoneWithThisFrameFloat(res) + kind = history.getkind(lltype.typeOf(res)) + if kind == 'void': raise jitexc.DoneWithThisFrameVoid() + if kind == 'int': raise jitexc.DoneWithThisFrameInt(res) + if kind == 'ref': raise jitexc.DoneWithThisFrameRef(self.cpu, res) + if kind == 'float': raise jitexc.DoneWithThisFrameFloat(res) raise AssertionError(kind) def prepare_resume_from_failure(self, deadframe, inputargs, resumedescr): diff --git a/rpython/jit/metainterp/warmspot.py b/rpython/jit/metainterp/warmspot.py --- a/rpython/jit/metainterp/warmspot.py +++ b/rpython/jit/metainterp/warmspot.py @@ -834,7 +834,6 @@ # more stuff # # to that: - # XXX there are too many exceptions all around... # # def original_portal(..): # stuff @@ -843,20 +842,9 @@ # def portal_runner(*args): # while 1: # try: - # try: - # return portal(*args) - # except EnterJitAssembler, e: - # while True: - # try: - # return e.execute() - # except EnterJitAssembler, e: - # continue - # except ContinueRunningNormally, e: - # *args = *e.new_args - # except DoneWithThisFrame, e: - # return e.return - # except ExitFrameWithException, e: - # raise Exception, e.value + # return portal(*args) + # except JitException, e: + # return handle_jitexception(e) # # def portal(*args): # while 1: @@ -920,6 +908,7 @@ return result def handle_jitexception(e): + # XXX there are too many exceptions all around... while True: if isinstance(e, EnterJitAssembler): try: diff --git a/rpython/jit/metainterp/warmstate.py b/rpython/jit/metainterp/warmstate.py --- a/rpython/jit/metainterp/warmstate.py +++ b/rpython/jit/metainterp/warmstate.py @@ -407,13 +407,13 @@ return None if result_type == history.INT: if isinstance(fail_descr, compile.DoneWithThisFrameDescrInt): - return fail_descr.get_result() + return fail_descr.get_result(cpu, deadframe) if result_type == history.REF: if isinstance(fail_descr, compile.DoneWithThisFrameDescrRef): - return fail_descr.get_result() + return fail_descr.get_result(cpu, deadframe) if result_type == history.FLOAT: if isinstance(fail_descr, compile.DoneWithThisFrameDescrFloat): - return fail_descr.get_result() + return fail_descr.get_result(cpu, deadframe) # # General case fail_descr.handle_fail(deadframe, metainterp_sd, jitdriver_sd) From pypy.commits at gmail.com Sat Jul 9 10:42:56 2016 From: pypy.commits at gmail.com (raffael_t) Date: Sat, 09 Jul 2016 07:42:56 -0700 (PDT) Subject: [pypy-commit] pypy py3.5-async: Add unfinished astbuilder tests for async Message-ID: <57810d70.67c0c20a.4abef.ffff95f5@mx.google.com> Author: Raffael Tfirst Branch: py3.5-async Changeset: r85627:9eeb13a22e8a Date: 2016-07-09 16:42 +0200 http://bitbucket.org/pypy/pypy/changeset/9eeb13a22e8a/ Log: Add unfinished astbuilder tests for async diff --git a/pypy/interpreter/astcompiler/test/test_astbuilder.py b/pypy/interpreter/astcompiler/test/test_astbuilder.py --- a/pypy/interpreter/astcompiler/test/test_astbuilder.py +++ b/pypy/interpreter/astcompiler/test/test_astbuilder.py @@ -1349,3 +1349,33 @@ assert isinstance(expr.left, ast.Name) assert isinstance(expr.right, ast.Name) # imatmul is tested earlier search for @= + + def test_asyncFunctionDef(self): + mod = self.get_ast("async def f():\n await something()") + assert isinstance(mod, ast.Module) + body = mod.body + assert len(body) == 1 + expr = body[0].value + assert expr.op == ast.MatMul + assert isinstance(expr.left, ast.Name) + assert isinstance(expr.right, ast.Name) + + def test_asyncAsyncFor(self): + mod = self.get_ast("async def f():\n async for e in i: 1\n else: 2") + assert isinstance(mod, ast.Module) + body = mod.body + assert len(body) == 1 + expr = body[0].value + assert expr.op == ast.MatMul + assert isinstance(expr.left, ast.Name) + assert isinstance(expr.right, ast.Name) + + def test_asyncAsyncWith(self): + mod = self.get_ast("async def f():\n async with a as b: 1") + assert isinstance(mod, ast.Module) + body = mod.body + assert len(body) == 1 + expr = body[0].value + assert expr.op == ast.MatMul + assert isinstance(expr.left, ast.Name) + assert isinstance(expr.right, ast.Name) From pypy.commits at gmail.com Sat Jul 9 10:45:18 2016 From: pypy.commits at gmail.com (raffael_t) Date: Sat, 09 Jul 2016 07:45:18 -0700 (PDT) Subject: [pypy-commit] pypy py3.5-async: Add test_astbuilder changes of branch py3.5 Message-ID: <57810dfe.c92bc20a.88d4f.0029@mx.google.com> Author: Raffael Tfirst Branch: py3.5-async Changeset: r85628:fe1d33bb7ba2 Date: 2016-07-09 16:44 +0200 http://bitbucket.org/pypy/pypy/changeset/fe1d33bb7ba2/ Log: Add test_astbuilder changes of branch py3.5 diff --git a/pypy/interpreter/astcompiler/test/test_astbuilder.py b/pypy/interpreter/astcompiler/test/test_astbuilder.py --- a/pypy/interpreter/astcompiler/test/test_astbuilder.py +++ b/pypy/interpreter/astcompiler/test/test_astbuilder.py @@ -492,12 +492,12 @@ assert not args.args assert not args.defaults assert args.kwarg is None - assert args.vararg == "a" + assert args.vararg.arg == "a" args = self.get_first_stmt("def f(**a): pass").args assert not args.args assert not args.defaults assert args.vararg is None - assert args.kwarg == "a" + assert args.kwarg.arg == "a" args = self.get_first_stmt("def f(a, b, c=d, *e, **f): pass").args assert len(args.args) == 3 for arg in args.args: @@ -505,8 +505,8 @@ assert len(args.defaults) == 1 assert isinstance(args.defaults[0], ast.Name) assert args.defaults[0].ctx == ast.Load - assert args.vararg == "e" - assert args.kwarg == "f" + assert args.vararg.arg == "e" + assert args.kwarg.arg == "f" input = "def f(a=b, c): pass" exc = py.test.raises(SyntaxError, self.get_ast, input).value assert exc.msg == "non-default argument follows default argument" @@ -545,9 +545,9 @@ assert isinstance(func.args.args[0].annotation, ast.Num) assert isinstance(func.args.defaults[0], ast.Name) func = self.get_first_stmt("def f(*x : 42): pass") - assert isinstance(func.args.varargannotation, ast.Num) + assert isinstance(func.args.vararg.annotation, ast.Num) func = self.get_first_stmt("def f(**kw : 42): pass") - assert isinstance(func.args.kwargannotation, ast.Num) + assert isinstance(func.args.kwarg.annotation, ast.Num) func = self.get_first_stmt("def f(*, kw : 42=a): pass") assert isinstance(func.args.kwonlyargs[0].annotation, ast.Num) @@ -606,8 +606,6 @@ assert dec.func.id == "dec" assert dec.args is None assert dec.keywords is None - assert dec.starargs is None - assert dec.kwargs is None definition = self.get_first_stmt("@dec(a, b)\n%s" % (stmt,)) assert len(definition.decorator_list) == 1 dec = definition.decorator_list[0] @@ -615,8 +613,6 @@ assert dec.func.id == "dec" assert len(dec.args) == 2 assert dec.keywords is None - assert dec.starargs is None - assert dec.kwargs is None def test_augassign(self): aug_assigns = ( @@ -625,7 +621,7 @@ ("/=", ast.Div), ("//=", ast.FloorDiv), ("%=", ast.Mod), - ("@=", ast.MatMul), + ("@=", ast.MatMult), ("<<=", ast.LShift), (">>=", ast.RShift), ("&=", ast.BitAnd), @@ -931,7 +927,7 @@ ("*", ast.Mult), ("//", ast.FloorDiv), ("%", ast.Mod), - ("@", ast.MatMul) + ("@", ast.MatMult) ) for op, ast_type in binops: bin = self.get_first_expr("a %s b" % (op,)) @@ -987,8 +983,6 @@ assert isinstance(call, ast.Call) assert call.args is None assert call.keywords is None - assert call.starargs is None - assert call.kwargs is None assert isinstance(call.func, ast.Name) assert call.func.ctx == ast.Load call = self.get_first_expr("f(2, 3)") @@ -996,8 +990,6 @@ assert isinstance(call.args[0], ast.Num) assert isinstance(call.args[1], ast.Num) assert call.keywords is None - assert call.starargs is None - assert call.kwargs is None call = self.get_first_expr("f(a=3)") assert call.args is None assert len(call.keywords) == 1 @@ -1006,21 +998,20 @@ assert keyword.arg == "a" assert isinstance(keyword.value, ast.Num) call = self.get_first_expr("f(*a, **b)") - assert call.args is None - assert isinstance(call.starargs, ast.Name) - assert call.starargs.id == "a" - assert call.starargs.ctx == ast.Load - assert isinstance(call.kwargs, ast.Name) - assert call.kwargs.id == "b" - assert call.kwargs.ctx == ast.Load + assert isinstance(call.args[0], ast.Starred) + assert isinstance(call.keywords[0], ast.keyword) + assert call.args[0].value.id == "a" + assert call.args[0].ctx == ast.Load + assert call.keywords[0].value.id == "b" call = self.get_first_expr("f(a, b, x=4, *m, **f)") - assert len(call.args) == 2 + assert len(call.args) == 3 assert isinstance(call.args[0], ast.Name) assert isinstance(call.args[1], ast.Name) - assert len(call.keywords) == 1 + assert isinstance(call.args[2], ast.Starred) + assert len(call.keywords) == 2 assert call.keywords[0].arg == "x" - assert call.starargs.id == "m" - assert call.kwargs.id == "f" + assert call.args[2].value.id == "m" + assert call.keywords[1].value.id == "f" call = self.get_first_expr("f(x for x in y)") assert len(call.args) == 1 assert isinstance(call.args[0], ast.GeneratorExp) @@ -1036,8 +1027,6 @@ assert exc.msg == "keyword can't be an expression" exc = py.test.raises(SyntaxError, self.get_ast, "f(a=c, a=d)").value assert exc.msg == "keyword argument repeated" - exc = py.test.raises(SyntaxError, self.get_ast, "f(x, *a, b)").value - assert exc.msg == "only named arguments may follow *expression" def test_attribute(self): attr = self.get_first_expr("x.y") @@ -1345,7 +1334,7 @@ body = mod.body assert len(body) == 1 expr = body[0].value - assert expr.op == ast.MatMul + assert expr.op == ast.MatMult assert isinstance(expr.left, ast.Name) assert isinstance(expr.right, ast.Name) # imatmul is tested earlier search for @= From pypy.commits at gmail.com Sat Jul 9 11:11:29 2016 From: pypy.commits at gmail.com (raffael_t) Date: Sat, 09 Jul 2016 08:11:29 -0700 (PDT) Subject: [pypy-commit] pypy py3.5-async: Add missing async opcodes in python lib Message-ID: <57811421.8a40c20a.624f4.ffffd3e4@mx.google.com> Author: Raffael Tfirst Branch: py3.5-async Changeset: r85629:062ebc10a032 Date: 2016-07-09 17:10 +0200 http://bitbucket.org/pypy/pypy/changeset/062ebc10a032/ Log: Add missing async opcodes in python lib diff --git a/lib-python/3/opcode.py b/lib-python/3/opcode.py --- a/lib-python/3/opcode.py +++ b/lib-python/3/opcode.py @@ -85,7 +85,10 @@ def_op('INPLACE_FLOOR_DIVIDE', 28) def_op('INPLACE_TRUE_DIVIDE', 29) -def_op('STORE_MAP', 54) +def_op('GET_AITER', 50) +def_op('GET_ANEXT', 51) +def_op('BEFORE_ASYNC_WITH', 52) + def_op('INPLACE_ADD', 55) def_op('INPLACE_SUBTRACT', 56) def_op('INPLACE_MULTIPLY', 57) @@ -100,11 +103,12 @@ def_op('BINARY_OR', 66) def_op('INPLACE_POWER', 67) def_op('GET_ITER', 68) -def_op('STORE_LOCALS', 69) +def_op('GET_YIELD_FROM_ITER', 69) def_op('PRINT_EXPR', 70) def_op('LOAD_BUILD_CLASS', 71) def_op('YIELD_FROM', 72) +def_op('GET_AWAITABLE', 73) def_op('INPLACE_LSHIFT', 75) def_op('INPLACE_RSHIFT', 76) @@ -196,6 +200,11 @@ def_op('SET_ADD', 146) def_op('MAP_ADD', 147) +def_op('LOAD_CLASSDEREF', 148) +hasfree.append(148) + +jrel_op('SETUP_ASYNC_WITH', 154) + def_op('EXTENDED_ARG', 144) EXTENDED_ARG = 144 diff --git a/pypy/interpreter/astcompiler/test/test_astbuilder.py b/pypy/interpreter/astcompiler/test/test_astbuilder.py --- a/pypy/interpreter/astcompiler/test/test_astbuilder.py +++ b/pypy/interpreter/astcompiler/test/test_astbuilder.py @@ -1340,8 +1340,8 @@ # imatmul is tested earlier search for @= def test_asyncFunctionDef(self): - mod = self.get_ast("async def f():\n await something()") - assert isinstance(mod, ast.Module) + asyncdef = self.get_ast("async def f():\n await something()") + assert isinstance(asyncdef, ast.AsyncFunctionDef) body = mod.body assert len(body) == 1 expr = body[0].value diff --git a/pypy/interpreter/pyopcode.py b/pypy/interpreter/pyopcode.py --- a/pypy/interpreter/pyopcode.py +++ b/pypy/interpreter/pyopcode.py @@ -1402,7 +1402,21 @@ self.popvalue() itemcount -= 1 self.pushvalue(w_dict) - + + def GET_AWAITABLE(self): + pass + + def SETUP_ASYNC_WITH(self): + pass + + def BEFORE_ASYNC_WITH(self): + pass + + def GET_AITER(self): + pass + + def GET_ANEXT(self): + pass ### ____________________________________________________________ ### From pypy.commits at gmail.com Sat Jul 9 11:46:26 2016 From: pypy.commits at gmail.com (raffael_t) Date: Sat, 09 Jul 2016 08:46:26 -0700 (PDT) Subject: [pypy-commit] pypy py3.5-async: Change stack effect for with_cleanup_start back to original value (Stackdeptherror fix) Message-ID: <57811c52.85c11c0a.3e9a4.c74c@mx.google.com> Author: Raffael Tfirst Branch: py3.5-async Changeset: r85630:cf8246dbc687 Date: 2016-07-09 17:45 +0200 http://bitbucket.org/pypy/pypy/changeset/cf8246dbc687/ Log: Change stack effect for with_cleanup_start back to original value (Stackdeptherror fix) diff --git a/pypy/interpreter/astcompiler/assemble.py b/pypy/interpreter/astcompiler/assemble.py --- a/pypy/interpreter/astcompiler/assemble.py +++ b/pypy/interpreter/astcompiler/assemble.py @@ -599,7 +599,7 @@ ops.PRINT_EXPR: -1, - ops.WITH_CLEANUP_START: 1, + ops.WITH_CLEANUP_START: -1, ops.WITH_CLEANUP_FINISH: -1, # XXX Sometimes more ops.LOAD_BUILD_CLASS: 1, ops.POP_BLOCK: 0, From pypy.commits at gmail.com Sat Jul 9 12:13:31 2016 From: pypy.commits at gmail.com (arigo) Date: Sat, 09 Jul 2016 09:13:31 -0700 (PDT) Subject: [pypy-commit] pypy default: hg merge issue2335 Message-ID: <578122ab.e655c20a.2be74.411a@mx.google.com> Author: Armin Rigo Branch: Changeset: r85631:9927e7ba72e9 Date: 2016-07-09 18:14 +0200 http://bitbucket.org/pypy/pypy/changeset/9927e7ba72e9/ Log: hg merge issue2335 Hopefully, this is a real fix for the issue shown in #2200 and #2235. It should avoid a remaining case in the JIT, where successive guard failures in the same Python function end up as successive levels of RPython functions, eventually exhausting the stack, while at app-level the traceback is very short. diff --git a/rpython/jit/metainterp/blackhole.py b/rpython/jit/metainterp/blackhole.py --- a/rpython/jit/metainterp/blackhole.py +++ b/rpython/jit/metainterp/blackhole.py @@ -1143,45 +1143,35 @@ @arguments("cpu", "i", "R", "d", returns="i") def bhimpl_residual_call_r_i(cpu, func, args_r, calldescr): - workaround2200.active = True return cpu.bh_call_i(func, None, args_r, None, calldescr) @arguments("cpu", "i", "R", "d", returns="r") def bhimpl_residual_call_r_r(cpu, func, args_r, calldescr): - workaround2200.active = True return cpu.bh_call_r(func, None, args_r, None, calldescr) @arguments("cpu", "i", "R", "d") def bhimpl_residual_call_r_v(cpu, func, args_r, calldescr): - workaround2200.active = True return cpu.bh_call_v(func, None, args_r, None, calldescr) @arguments("cpu", "i", "I", "R", "d", returns="i") def bhimpl_residual_call_ir_i(cpu, func, args_i, args_r, calldescr): - workaround2200.active = True return cpu.bh_call_i(func, args_i, args_r, None, calldescr) @arguments("cpu", "i", "I", "R", "d", returns="r") def bhimpl_residual_call_ir_r(cpu, func, args_i, args_r, calldescr): - workaround2200.active = True return cpu.bh_call_r(func, args_i, args_r, None, calldescr) @arguments("cpu", "i", "I", "R", "d") def bhimpl_residual_call_ir_v(cpu, func, args_i, args_r, calldescr): - workaround2200.active = True return cpu.bh_call_v(func, args_i, args_r, None, calldescr) @arguments("cpu", "i", "I", "R", "F", "d", returns="i") def bhimpl_residual_call_irf_i(cpu, func, args_i,args_r,args_f,calldescr): - workaround2200.active = True return cpu.bh_call_i(func, args_i, args_r, args_f, calldescr) @arguments("cpu", "i", "I", "R", "F", "d", returns="r") def bhimpl_residual_call_irf_r(cpu, func, args_i,args_r,args_f,calldescr): - workaround2200.active = True return cpu.bh_call_r(func, args_i, args_r, args_f, calldescr) @arguments("cpu", "i", "I", "R", "F", "d", returns="f") def bhimpl_residual_call_irf_f(cpu, func, args_i,args_r,args_f,calldescr): - workaround2200.active = True return cpu.bh_call_f(func, args_i, args_r, args_f, calldescr) @arguments("cpu", "i", "I", "R", "F", "d") def bhimpl_residual_call_irf_v(cpu, func, args_i,args_r,args_f,calldescr): - workaround2200.active = True return cpu.bh_call_v(func, args_i, args_r, args_f, calldescr) # conditional calls - note that they cannot return stuff @@ -1209,54 +1199,44 @@ @arguments("cpu", "j", "R", returns="i") def bhimpl_inline_call_r_i(cpu, jitcode, args_r): - workaround2200.active = True return cpu.bh_call_i(jitcode.get_fnaddr_as_int(), None, args_r, None, jitcode.calldescr) @arguments("cpu", "j", "R", returns="r") def bhimpl_inline_call_r_r(cpu, jitcode, args_r): - workaround2200.active = True return cpu.bh_call_r(jitcode.get_fnaddr_as_int(), None, args_r, None, jitcode.calldescr) @arguments("cpu", "j", "R") def bhimpl_inline_call_r_v(cpu, jitcode, args_r): - workaround2200.active = True return cpu.bh_call_v(jitcode.get_fnaddr_as_int(), None, args_r, None, jitcode.calldescr) @arguments("cpu", "j", "I", "R", returns="i") def bhimpl_inline_call_ir_i(cpu, jitcode, args_i, args_r): - workaround2200.active = True return cpu.bh_call_i(jitcode.get_fnaddr_as_int(), args_i, args_r, None, jitcode.calldescr) @arguments("cpu", "j", "I", "R", returns="r") def bhimpl_inline_call_ir_r(cpu, jitcode, args_i, args_r): - workaround2200.active = True return cpu.bh_call_r(jitcode.get_fnaddr_as_int(), args_i, args_r, None, jitcode.calldescr) @arguments("cpu", "j", "I", "R") def bhimpl_inline_call_ir_v(cpu, jitcode, args_i, args_r): - workaround2200.active = True return cpu.bh_call_v(jitcode.get_fnaddr_as_int(), args_i, args_r, None, jitcode.calldescr) @arguments("cpu", "j", "I", "R", "F", returns="i") def bhimpl_inline_call_irf_i(cpu, jitcode, args_i, args_r, args_f): - workaround2200.active = True return cpu.bh_call_i(jitcode.get_fnaddr_as_int(), args_i, args_r, args_f, jitcode.calldescr) @arguments("cpu", "j", "I", "R", "F", returns="r") def bhimpl_inline_call_irf_r(cpu, jitcode, args_i, args_r, args_f): - workaround2200.active = True return cpu.bh_call_r(jitcode.get_fnaddr_as_int(), args_i, args_r, args_f, jitcode.calldescr) @arguments("cpu", "j", "I", "R", "F", returns="f") def bhimpl_inline_call_irf_f(cpu, jitcode, args_i, args_r, args_f): - workaround2200.active = True return cpu.bh_call_f(jitcode.get_fnaddr_as_int(), args_i, args_r, args_f, jitcode.calldescr) @arguments("cpu", "j", "I", "R", "F") def bhimpl_inline_call_irf_v(cpu, jitcode, args_i, args_r, args_f): - workaround2200.active = True return cpu.bh_call_v(jitcode.get_fnaddr_as_int(), args_i, args_r, args_f, jitcode.calldescr) @@ -1543,8 +1523,6 @@ if not self.nextblackholeinterp: self._exit_frame_with_exception(current_exc) return current_exc - finally: - workaround2200.active = False # # pass the frame's return value to the caller caller = self.nextblackholeinterp @@ -1717,10 +1695,3 @@ # _run_forever(firstbh, current_exc) convert_and_run_from_pyjitpl._dont_inline_ = True - -# ____________________________________________________________ - -class WorkaroundIssue2200(object): - pass -workaround2200 = WorkaroundIssue2200() -workaround2200.active = False diff --git a/rpython/jit/metainterp/compile.py b/rpython/jit/metainterp/compile.py --- a/rpython/jit/metainterp/compile.py +++ b/rpython/jit/metainterp/compile.py @@ -620,23 +620,28 @@ raise jitexc.DoneWithThisFrameVoid() class DoneWithThisFrameDescrInt(_DoneWithThisFrameDescr): + def get_result(self, cpu, deadframe): + return cpu.get_int_value(deadframe, 0) def handle_fail(self, deadframe, metainterp_sd, jitdriver_sd): assert jitdriver_sd.result_type == history.INT - result = metainterp_sd.cpu.get_int_value(deadframe, 0) - raise jitexc.DoneWithThisFrameInt(result) + cpu = metainterp_sd.cpu + raise jitexc.DoneWithThisFrameInt(self.get_result(cpu, deadframe)) class DoneWithThisFrameDescrRef(_DoneWithThisFrameDescr): + def get_result(self, cpu, deadframe): + return cpu.get_ref_value(deadframe, 0) def handle_fail(self, deadframe, metainterp_sd, jitdriver_sd): assert jitdriver_sd.result_type == history.REF cpu = metainterp_sd.cpu - result = cpu.get_ref_value(deadframe, 0) - raise jitexc.DoneWithThisFrameRef(cpu, result) + raise jitexc.DoneWithThisFrameRef(cpu, self.get_result(cpu, deadframe)) class DoneWithThisFrameDescrFloat(_DoneWithThisFrameDescr): + def get_result(self, cpu, deadframe): + return cpu.get_float_value(deadframe, 0) def handle_fail(self, deadframe, metainterp_sd, jitdriver_sd): assert jitdriver_sd.result_type == history.FLOAT - result = metainterp_sd.cpu.get_float_value(deadframe, 0) - raise jitexc.DoneWithThisFrameFloat(result) + cpu = metainterp_sd.cpu + raise jitexc.DoneWithThisFrameFloat(self.get_result(cpu, deadframe)) class ExitFrameWithExceptionDescrRef(_DoneWithThisFrameDescr): def handle_fail(self, deadframe, metainterp_sd, jitdriver_sd): diff --git a/rpython/jit/metainterp/pyjitpl.py b/rpython/jit/metainterp/pyjitpl.py --- a/rpython/jit/metainterp/pyjitpl.py +++ b/rpython/jit/metainterp/pyjitpl.py @@ -2543,7 +2543,13 @@ elif box.type == history.REF: args.append(box.getref_base()) elif box.type == history.FLOAT: args.append(box.getfloatstorage()) else: assert 0 - self.jitdriver_sd.warmstate.execute_assembler(loop_token, *args) + res = self.jitdriver_sd.warmstate.execute_assembler(loop_token, *args) + kind = history.getkind(lltype.typeOf(res)) + if kind == 'void': raise jitexc.DoneWithThisFrameVoid() + if kind == 'int': raise jitexc.DoneWithThisFrameInt(res) + if kind == 'ref': raise jitexc.DoneWithThisFrameRef(self.cpu, res) + if kind == 'float': raise jitexc.DoneWithThisFrameFloat(res) + raise AssertionError(kind) def prepare_resume_from_failure(self, deadframe, inputargs, resumedescr): exception = self.cpu.grab_exc_value(deadframe) diff --git a/rpython/jit/metainterp/warmspot.py b/rpython/jit/metainterp/warmspot.py --- a/rpython/jit/metainterp/warmspot.py +++ b/rpython/jit/metainterp/warmspot.py @@ -529,7 +529,7 @@ def make_enter_function(self, jd): from rpython.jit.metainterp.warmstate import WarmEnterState state = WarmEnterState(self, jd) - maybe_compile_and_run = state.make_entry_point() + maybe_compile_and_run, EnterJitAssembler = state.make_entry_point() jd.warmstate = state def crash_in_jit(e): @@ -560,6 +560,7 @@ maybe_enter_jit._always_inline_ = True jd._maybe_enter_jit_fn = maybe_enter_jit jd._maybe_compile_and_run_fn = maybe_compile_and_run + jd._EnterJitAssembler = EnterJitAssembler def make_driverhook_graphs(self): s_Str = annmodel.SomeString() @@ -842,12 +843,8 @@ # while 1: # try: # return portal(*args) - # except ContinueRunningNormally, e: - # *args = *e.new_args - # except DoneWithThisFrame, e: - # return e.return - # except ExitFrameWithException, e: - # raise Exception, e.value + # except JitException, e: + # return handle_jitexception(e) # # def portal(*args): # while 1: @@ -884,90 +881,79 @@ rtyper = self.translator.rtyper RESULT = PORTALFUNC.RESULT result_kind = history.getkind(RESULT) + assert result_kind.startswith(jd.result_type) ts = self.cpu.ts state = jd.warmstate maybe_compile_and_run = jd._maybe_compile_and_run_fn + EnterJitAssembler = jd._EnterJitAssembler def ll_portal_runner(*args): - start = True - while 1: - try: - # maybe enter from the function's start. Note that the - # 'start' variable is constant-folded away because it's - # the first statement in the loop. - if start: - maybe_compile_and_run( - state.increment_function_threshold, *args) - # - # then run the normal portal function, i.e. the - # interpreter's main loop. It might enter the jit - # via maybe_enter_jit(), which typically ends with - # handle_fail() being called, which raises on the - # following exceptions --- catched here, because we - # want to interrupt the whole interpreter loop. - return support.maybe_on_top_of_llinterp(rtyper, - portal_ptr)(*args) - except jitexc.ContinueRunningNormally as e: + try: + # maybe enter from the function's start. + maybe_compile_and_run( + state.increment_function_threshold, *args) + # + # then run the normal portal function, i.e. the + # interpreter's main loop. It might enter the jit + # via maybe_enter_jit(), which typically ends with + # handle_fail() being called, which raises on the + # following exceptions --- catched here, because we + # want to interrupt the whole interpreter loop. + return support.maybe_on_top_of_llinterp(rtyper, + portal_ptr)(*args) + except jitexc.JitException as e: + result = handle_jitexception(e) + if result_kind != 'void': + result = specialize_value(RESULT, result) + return result + + def handle_jitexception(e): + # XXX there are too many exceptions all around... + while True: + if isinstance(e, EnterJitAssembler): + try: + return e.execute() + except jitexc.JitException as e: + continue + # + if isinstance(e, jitexc.ContinueRunningNormally): args = () for ARGTYPE, attrname, count in portalfunc_ARGS: x = getattr(e, attrname)[count] x = specialize_value(ARGTYPE, x) args = args + (x,) - start = False - continue - except jitexc.DoneWithThisFrameVoid: - assert result_kind == 'void' - return - except jitexc.DoneWithThisFrameInt as e: - assert result_kind == 'int' - return specialize_value(RESULT, e.result) - except jitexc.DoneWithThisFrameRef as e: - assert result_kind == 'ref' - return specialize_value(RESULT, e.result) - except jitexc.DoneWithThisFrameFloat as e: - assert result_kind == 'float' - return specialize_value(RESULT, e.result) - except jitexc.ExitFrameWithExceptionRef as e: + try: + result = support.maybe_on_top_of_llinterp(rtyper, + portal_ptr)(*args) + except jitexc.JitException as e: + continue + if result_kind != 'void': + result = unspecialize_value(result) + return result + # + if result_kind == 'void': + if isinstance(e, jitexc.DoneWithThisFrameVoid): + return None + if result_kind == 'int': + if isinstance(e, jitexc.DoneWithThisFrameInt): + return e.result + if result_kind == 'ref': + if isinstance(e, jitexc.DoneWithThisFrameRef): + return e.result + if result_kind == 'float': + if isinstance(e, jitexc.DoneWithThisFrameFloat): + return e.result + # + if isinstance(e, jitexc.ExitFrameWithExceptionRef): value = ts.cast_to_baseclass(e.value) if not we_are_translated(): raise LLException(ts.get_typeptr(value), value) else: value = cast_base_ptr_to_instance(Exception, value) + assert value is not None raise value - - def handle_jitexception(e): - # XXX the bulk of this function is mostly a copy-paste from above - try: - raise e - except jitexc.ContinueRunningNormally as e: - args = () - for ARGTYPE, attrname, count in portalfunc_ARGS: - x = getattr(e, attrname)[count] - x = specialize_value(ARGTYPE, x) - args = args + (x,) - result = ll_portal_runner(*args) - if result_kind != 'void': - result = unspecialize_value(result) - return result - except jitexc.DoneWithThisFrameVoid: - assert result_kind == 'void' - return - except jitexc.DoneWithThisFrameInt as e: - assert result_kind == 'int' - return e.result - except jitexc.DoneWithThisFrameRef as e: - assert result_kind == 'ref' - return e.result - except jitexc.DoneWithThisFrameFloat as e: - assert result_kind == 'float' - return e.result - except jitexc.ExitFrameWithExceptionRef as e: - value = ts.cast_to_baseclass(e.value) - if not we_are_translated(): - raise LLException(ts.get_typeptr(value), value) - else: - value = cast_base_ptr_to_instance(Exception, value) - raise value + # + raise AssertionError("all cases should have been handled") jd._ll_portal_runner = ll_portal_runner # for debugging jd.portal_runner_ptr = self.helper_func(jd._PTR_PORTAL_FUNCTYPE, diff --git a/rpython/jit/metainterp/warmstate.py b/rpython/jit/metainterp/warmstate.py --- a/rpython/jit/metainterp/warmstate.py +++ b/rpython/jit/metainterp/warmstate.py @@ -2,7 +2,7 @@ import weakref from rpython.jit.codewriter import support, heaptracker, longlong -from rpython.jit.metainterp import resoperation, history +from rpython.jit.metainterp import resoperation, history, jitexc from rpython.rlib.debug import debug_start, debug_stop, debug_print from rpython.rlib.debug import have_debug_prints_for from rpython.rlib.jit import PARAMETERS @@ -348,8 +348,9 @@ def make_entry_point(self): "NOT_RPYTHON" - if hasattr(self, 'maybe_compile_and_run'): - return self.maybe_compile_and_run + from rpython.jit.metainterp import compile + if hasattr(self, 'entry_point_fns'): + return self.entry_point_fns warmrunnerdesc = self.warmrunnerdesc metainterp_sd = warmrunnerdesc.metainterp_sd @@ -362,6 +363,8 @@ confirm_enter_jit = self.confirm_enter_jit range_red_args = unrolling_iterable( range(num_green_args, num_green_args + jitdriver_sd.num_red_args)) + name_red_args = unrolling_iterable( + [(i, 'arg%d' % i) for i in range(jitdriver_sd.num_red_args)]) # get a new specialized copy of the method ARGS = [] for kind in jitdriver_sd.red_args_types: @@ -376,6 +379,7 @@ func_execute_token = self.cpu.make_execute_token(*ARGS) cpu = self.cpu jitcounter = self.warmrunnerdesc.jitcounter + result_type = jitdriver_sd.result_type def execute_assembler(loop_token, *args): # Call the backend to run the 'looptoken' with the given @@ -396,8 +400,23 @@ # # Handle the failure fail_descr = cpu.get_latest_descr(deadframe) + # First, a fast path to avoid raising and immediately catching + # a DoneWithThisFrame exception + if result_type == history.VOID: + if isinstance(fail_descr, compile.DoneWithThisFrameDescrVoid): + return None + if result_type == history.INT: + if isinstance(fail_descr, compile.DoneWithThisFrameDescrInt): + return fail_descr.get_result(cpu, deadframe) + if result_type == history.REF: + if isinstance(fail_descr, compile.DoneWithThisFrameDescrRef): + return fail_descr.get_result(cpu, deadframe) + if result_type == history.FLOAT: + if isinstance(fail_descr, compile.DoneWithThisFrameDescrFloat): + return fail_descr.get_result(cpu, deadframe) + # + # General case fail_descr.handle_fail(deadframe, metainterp_sd, jitdriver_sd) - # assert 0, "should have raised" def bound_reached(hash, cell, *args): @@ -419,7 +438,8 @@ def maybe_compile_and_run(increment_threshold, *args): """Entry point to the JIT. Called at the point with the - can_enter_jit() hint. + can_enter_jit() hint, and at the start of a function + with a different threshold. """ # Look for the cell corresponding to the current greenargs. # Search for the JitCell that is of the correct subclass of @@ -439,14 +459,6 @@ bound_reached(hash, None, *args) return - # Workaround for issue #2200, maybe temporary. This is not - # a proper fix, but only a hack that should work well enough - # for PyPy's main jitdriver... See test_issue2200_recursion - from rpython.jit.metainterp.blackhole import workaround2200 - if workaround2200.active: - workaround2200.active = False - return - # Here, we have found 'cell'. # if cell.flags & (JC_TRACING | JC_TEMPORARY): @@ -484,14 +496,27 @@ execute_args = () for i in range_red_args: execute_args += (unspecialize_value(args[i]), ) - # run it! this executes until interrupted by an exception - execute_assembler(procedure_token, *execute_args) - assert 0, "should not reach this point" + # run it, but from outside in ll_portal_runner, not from here + # (this avoids RPython-level recursion with no corresponding + # app-level recursion, as shown by issues 2200 and 2335) + raise EnterJitAssembler(procedure_token, *execute_args) + + class EnterJitAssembler(jitexc.JitException): + def __init__(self, procedure_token, *args): + self.procedure_token = procedure_token + for i, argname in name_red_args: + setattr(self, argname, args[i]) + def execute(self): + args = () + for i, argname in name_red_args: + args += (getattr(self, argname), ) + return execute_assembler(self.procedure_token, *args) maybe_compile_and_run._dont_inline_ = True - self.maybe_compile_and_run = maybe_compile_and_run self.execute_assembler = execute_assembler - return maybe_compile_and_run + self.entry_point_fns = (maybe_compile_and_run, + EnterJitAssembler) + return self.entry_point_fns # ---------- From pypy.commits at gmail.com Sat Jul 9 14:22:48 2016 From: pypy.commits at gmail.com (raffael_t) Date: Sat, 09 Jul 2016 11:22:48 -0700 (PDT) Subject: [pypy-commit] pypy py3.5-async: Complete astbuilder test for AsyncFunctionDef Message-ID: <578140f8.46461c0a.ad758.0d24@mx.google.com> Author: Raffael Tfirst Branch: py3.5-async Changeset: r85632:77e67b87c3c5 Date: 2016-07-09 20:22 +0200 http://bitbucket.org/pypy/pypy/changeset/77e67b87c3c5/ Log: Complete astbuilder test for AsyncFunctionDef diff --git a/pypy/interpreter/astcompiler/test/test_astbuilder.py b/pypy/interpreter/astcompiler/test/test_astbuilder.py --- a/pypy/interpreter/astcompiler/test/test_astbuilder.py +++ b/pypy/interpreter/astcompiler/test/test_astbuilder.py @@ -1340,14 +1340,24 @@ # imatmul is tested earlier search for @= def test_asyncFunctionDef(self): - asyncdef = self.get_ast("async def f():\n await something()") + mod = self.get_ast("async def f():\n await something()") + assert isinstance(mod, ast.Module) + assert len(mod.body) == 1 + asyncdef = mod.body[0] assert isinstance(asyncdef, ast.AsyncFunctionDef) - body = mod.body - assert len(body) == 1 - expr = body[0].value - assert expr.op == ast.MatMul - assert isinstance(expr.left, ast.Name) - assert isinstance(expr.right, ast.Name) + assert asyncdef.name == 'f' + assert asyncdef.args.args == None + assert len(asyncdef.body) == 1 + expr = asyncdef.body[0] + assert isinstance(expr, ast.Expr) + exprvalue = expr.value + assert isinstance(exprvalue, ast.Await) + awaitvalue = exprvalue.value + assert isinstance(awaitvalue, ast.Call) + func = awaitvalue.func + assert isinstance(func, ast.Name) + assert func.id == 'something' + assert func.ctx == ast.Load def test_asyncAsyncFor(self): mod = self.get_ast("async def f():\n async for e in i: 1\n else: 2") From pypy.commits at gmail.com Sat Jul 9 15:22:02 2016 From: pypy.commits at gmail.com (raffael_t) Date: Sat, 09 Jul 2016 12:22:02 -0700 (PDT) Subject: [pypy-commit] pypy py3.5-async: Complete astbuilder test for AsyncFor Message-ID: <57814eda.e457c20a.55c3c.4125@mx.google.com> Author: Raffael Tfirst Branch: py3.5-async Changeset: r85633:2b649eed6eb4 Date: 2016-07-09 21:21 +0200 http://bitbucket.org/pypy/pypy/changeset/2b649eed6eb4/ Log: Complete astbuilder test for AsyncFor diff --git a/pypy/interpreter/astcompiler/test/test_astbuilder.py b/pypy/interpreter/astcompiler/test/test_astbuilder.py --- a/pypy/interpreter/astcompiler/test/test_astbuilder.py +++ b/pypy/interpreter/astcompiler/test/test_astbuilder.py @@ -1359,22 +1359,34 @@ assert func.id == 'something' assert func.ctx == ast.Load - def test_asyncAsyncFor(self): + def test_asyncFor(self): mod = self.get_ast("async def f():\n async for e in i: 1\n else: 2") assert isinstance(mod, ast.Module) - body = mod.body - assert len(body) == 1 - expr = body[0].value - assert expr.op == ast.MatMul - assert isinstance(expr.left, ast.Name) - assert isinstance(expr.right, ast.Name) + assert len(mod.body) == 1 + asyncdef = mod.body[0] + assert isinstance(asyncdef, ast.AsyncFunctionDef) + assert asyncdef.name == 'f' + assert asyncdef.args.args == None + assert len(asyncdef.body) == 1 + asyncfor = asyncdef.body[0] + assert isinstance(asyncfor, ast.AsyncFor) + assert isinstance(asyncfor.target, ast.Name) + assert isinstance(asyncfor.iter, ast.Name) + assert len(asyncfor.body) == 1 + assert isinstance(asyncfor.body[0], ast.Expr) + assert isinstance(asyncfor.body[0].value, ast.Num) + assert len(asyncfor.orelse) == 1 + assert isinstance(asyncfor.orelse[0], ast.Expr) + assert isinstance(asyncfor.orelse[0].value, ast.Num) - def test_asyncAsyncWith(self): + def test_asyncWith(self): mod = self.get_ast("async def f():\n async with a as b: 1") assert isinstance(mod, ast.Module) - body = mod.body - assert len(body) == 1 - expr = body[0].value - assert expr.op == ast.MatMul - assert isinstance(expr.left, ast.Name) - assert isinstance(expr.right, ast.Name) + assert len(mod.body) == 1 + asyncdef = mod.body[0] + assert isinstance(asyncdef, ast.AsyncFunctionDef) + assert asyncdef.name == 'f' + assert asyncdef.args.args == None + assert len(asyncdef.body) == 1 + expr = asyncdef.body[0] + assert isinstance(expr, ast.AsyncWith) From pypy.commits at gmail.com Sat Jul 9 15:50:52 2016 From: pypy.commits at gmail.com (raffael_t) Date: Sat, 09 Jul 2016 12:50:52 -0700 (PDT) Subject: [pypy-commit] pypy py3.5-async: Complete astbuilder test for AsyncWith Message-ID: <5781559c.10371c0a.39c8d.0932@mx.google.com> Author: Raffael Tfirst Branch: py3.5-async Changeset: r85634:b1818584cc50 Date: 2016-07-09 21:50 +0200 http://bitbucket.org/pypy/pypy/changeset/b1818584cc50/ Log: Complete astbuilder test for AsyncWith diff --git a/pypy/interpreter/astcompiler/test/test_astbuilder.py b/pypy/interpreter/astcompiler/test/test_astbuilder.py --- a/pypy/interpreter/astcompiler/test/test_astbuilder.py +++ b/pypy/interpreter/astcompiler/test/test_astbuilder.py @@ -1388,5 +1388,13 @@ assert asyncdef.name == 'f' assert asyncdef.args.args == None assert len(asyncdef.body) == 1 - expr = asyncdef.body[0] - assert isinstance(expr, ast.AsyncWith) + asyncwith = asyncdef.body[0] + assert isinstance(asyncwith, ast.AsyncWith) + assert len(asyncwith.items) == 1 + asyncitem = asyncwith.items[0] + assert isinstance(asyncitem, ast.withitem) + assert isinstance(asyncitem.context_expr, ast.Name) + assert isinstance(asyncitem.optional_vars, ast.Name) + assert len(asyncwith.body) == 1 + assert isinstance(asyncwith.body[0], ast.Expr) + assert isinstance(asyncwith.body[0].value, ast.Num) From pypy.commits at gmail.com Sat Jul 9 16:41:54 2016 From: pypy.commits at gmail.com (plan_rich) Date: Sat, 09 Jul 2016 13:41:54 -0700 (PDT) Subject: [pypy-commit] pypy new-jit-log: change the loop numbering (token.number) to use jitlog's trace id, Message-ID: <57816192.cdcf1c0a.5e963.2112@mx.google.com> Author: Richard Plangger Branch: new-jit-log Changeset: r85635:4f7beec17bd8 Date: 2016-07-09 22:41 +0200 http://bitbucket.org/pypy/pypy/changeset/4f7beec17bd8/ Log: change the loop numbering (token.number) to use jitlog's trace id, change test in test_compile, modify the emitted jitlog format when logging jitlog counters diff --git a/pypy/module/_vmprof/interp_vmprof.py b/pypy/module/_vmprof/interp_vmprof.py --- a/pypy/module/_vmprof/interp_vmprof.py +++ b/pypy/module/_vmprof/interp_vmprof.py @@ -3,7 +3,7 @@ from pypy.interpreter.pyframe import PyFrame from pypy.interpreter.pycode import PyCode from pypy.interpreter.baseobjspace import W_Root -from rpython.rlib import rvmprof +from rpython.rlib import rvmprof, jit # ____________________________________________________________ @@ -77,6 +77,7 @@ except rvmprof.VMProfError, e: raise VMProfError(space, e) + at jit.dont_look_inside def disable_jitlog(space): """ Disable PyPy's logging facility. """ rvmprof.disable_jitlog() diff --git a/rpython/jit/backend/llsupport/assembler.py b/rpython/jit/backend/llsupport/assembler.py --- a/rpython/jit/backend/llsupport/assembler.py +++ b/rpython/jit/backend/llsupport/assembler.py @@ -13,6 +13,7 @@ from rpython.rlib.jitlog import _log_jit_counter from rpython.rtyper.annlowlevel import cast_instance_to_gcref, llhelper from rpython.rtyper.lltypesystem import rffi, lltype +from rpython.rlib.rvmprof.rvmprof import _get_vmprof DEBUG_COUNTER = lltype.Struct('DEBUG_COUNTER', # 'b'ridge, 'l'abel or # 'e'ntry point @@ -134,7 +135,9 @@ # if self._debug is already set it means that someone called # set_debug by hand before initializing the assembler. Leave it # as it is - self.set_debug(have_debug_prints_for('jit-backend-counts')) + should_debug = have_debug_prints_for('jit-backend-counts') or \ + _get_vmprof().cintf.jitlog_enabled() + self.set_debug(should_debug) # when finishing, we only have one value at [0], the rest dies self.gcmap_for_finish = lltype.malloc(jitframe.GCMAP, 1, flavor='raw', diff --git a/rpython/jit/backend/llsupport/test/ztranslation_test.py b/rpython/jit/backend/llsupport/test/ztranslation_test.py --- a/rpython/jit/backend/llsupport/test/ztranslation_test.py +++ b/rpython/jit/backend/llsupport/test/ztranslation_test.py @@ -12,6 +12,7 @@ from rpython.config.config import ConfigError from rpython.translator.tool.cbuild import ExternalCompilationInfo from rpython.rtyper.lltypesystem import lltype, rffi +from rpython.rlib import jitlog class TranslationTest(CCompiledMixin): @@ -240,6 +241,23 @@ assert res == 2 # one for loop and one for the prologue, no unrolling + def test_flush_trace_counts(self): + driver = JitDriver(greens = [], reds = ['i']) + + def f(): + i = 0 + while i < 100000: + driver.jit_merge_point(i=i) + i += 1 + + def main(): + jit_hooks.stats_set_debug(None, True) + f() + jitlog.stats_flush_trace_counts(None) + return 0 + + res = self.meta_interp(main, []) + assert res == 0 class TranslationRemoveTypePtrTest(CCompiledMixin): CPUClass = getcpuclass() diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py --- a/rpython/jit/backend/x86/assembler.py +++ b/rpython/jit/backend/x86/assembler.py @@ -493,8 +493,9 @@ clt.frame_info.clear() # for now if log: + number = looptoken.number operations = self._inject_debugging_code(looptoken, operations, - 'e', looptoken.number) + 'e', number) regalloc = RegAlloc(self, self.cpu.translate_support_code) # diff --git a/rpython/jit/backend/x86/runner.py b/rpython/jit/backend/x86/runner.py --- a/rpython/jit/backend/x86/runner.py +++ b/rpython/jit/backend/x86/runner.py @@ -2,7 +2,6 @@ from rpython.rtyper.lltypesystem import lltype, llmemory, rffi from rpython.rlib.jit_hooks import LOOP_RUN_CONTAINER from rpython.rlib import rgc -from rpython.jit.metainterp.debug import debug_sd from rpython.jit.backend.x86.assembler import Assembler386 from rpython.jit.backend.x86.regalloc import gpr_reg_mgr_cls, xmm_reg_mgr_cls from rpython.jit.backend.x86.profagent import ProfileAgent @@ -115,9 +114,10 @@ looptoken.compiled_loop_token.invalidate_positions = [] def get_all_loop_runs(self): + asm = self.assembler l = lltype.malloc(LOOP_RUN_CONTAINER, - len(debug_sd.loop_run_counters)) - for i, ll_s in enumerate(debug_sd.loop_run_counters): + len(asm.loop_run_counters)) + for i, ll_s in enumerate(asm.loop_run_counters): l[i].type = ll_s.type l[i].number = ll_s.number l[i].counter = ll_s.i diff --git a/rpython/jit/metainterp/compile.py b/rpython/jit/metainterp/compile.py --- a/rpython/jit/metainterp/compile.py +++ b/rpython/jit/metainterp/compile.py @@ -526,9 +526,7 @@ patch_new_loop_to_load_virtualizable_fields(loop, jitdriver_sd, vable) original_jitcell_token = loop.original_jitcell_token - globaldata = metainterp_sd.globaldata - original_jitcell_token.number = n = globaldata.loopnumbering - globaldata.loopnumbering += 1 + original_jitcell_token.number = n = metainterp_sd.jitlog.trace_id if not we_are_translated(): show_procedures(metainterp_sd, loop) diff --git a/rpython/jit/metainterp/pyjitpl.py b/rpython/jit/metainterp/pyjitpl.py --- a/rpython/jit/metainterp/pyjitpl.py +++ b/rpython/jit/metainterp/pyjitpl.py @@ -1931,7 +1931,6 @@ self.initialized = False self.indirectcall_dict = None self.addr2name = None - self.loopnumbering = 0 # ____________________________________________________________ diff --git a/rpython/jit/metainterp/test/test_compile.py b/rpython/jit/metainterp/test/test_compile.py --- a/rpython/jit/metainterp/test/test_compile.py +++ b/rpython/jit/metainterp/test/test_compile.py @@ -52,7 +52,7 @@ return 'location' class FakeGlobalData(object): - loopnumbering = 0 + pass class FakeMetaInterpStaticData(object): all_descrs = [] @@ -80,8 +80,8 @@ cpu = FakeCPU() staticdata = FakeMetaInterpStaticData() staticdata.cpu = cpu - staticdata.globaldata = FakeGlobalData() - staticdata.globaldata.loopnumbering = 1 + staticdata.jitlog = jl.VMProfJitLogger(cpu) + staticdata.jitlog.trace_id = 1 # loop = parse(''' [p1] @@ -108,8 +108,7 @@ jitcell_token = target_token.targeting_jitcell_token assert jitcell_token == target_token.original_jitcell_token assert jitcell_token.target_tokens == [target_token] - assert jitcell_token.number == 1 - assert staticdata.globaldata.loopnumbering == 2 + assert jitcell_token.number == 2 # assert len(cpu.seen) == 1 assert cpu.seen[0][2] == jitcell_token diff --git a/rpython/rlib/jitlog.py b/rpython/rlib/jitlog.py --- a/rpython/rlib/jitlog.py +++ b/rpython/rlib/jitlog.py @@ -17,9 +17,7 @@ @register_helper(None) def stats_flush_trace_counts(warmrunnerdesc): - print("hello") warmrunnerdesc.metainterp_sd.cpu.assembler.flush_trace_counters() - return True def commonprefix(a,b): "Given a list of pathnames, returns the longest common leading component" @@ -235,11 +233,13 @@ cintf = _get_vmprof().cintf if not cintf.jitlog_enabled(): return - le_addr = encode_le_addr(struct.number) - # not an address (but a number) but it is a machine word - le_count = encode_le_addr(struct.i) - out = le_addr + le_count - cintf.jitlog_write_marked(MARK_JITLOG_COUNTER + out, len(out) + 1) + # addr is either a number (trace_id), or the address + # of the descriptor. for entries it is a the trace_id, + # for any label/bridge entry the addr is the address + list = [MARK_JITLOG_COUNTER, encode_le_addr(struct.number), + struct.type, encode_le_64bit(struct.i)] + content = ''.join(list) + cintf.jitlog_write_marked(content, len(content)) class VMProfJitLogger(object): def __init__(self, cpu=None): @@ -265,10 +265,11 @@ self.cintf.jitlog_teardown() def start_new_trace(self, metainterp_sd, faildescr=None, entry_bridge=False): + # even if the logger is not enabled, increment the trace id + self.trace_id += 1 if not self.cintf.jitlog_enabled(): return self.metainterp_sd = metainterp_sd - self.trace_id += 1 content = [encode_le_addr(self.trace_id)] if faildescr: content.append(encode_str('bridge')) diff --git a/rpython/rlib/rvmprof/rvmprof.py b/rpython/rlib/rvmprof/rvmprof.py --- a/rpython/rlib/rvmprof/rvmprof.py +++ b/rpython/rlib/rvmprof/rvmprof.py @@ -122,6 +122,7 @@ raise VMProfError(os.strerror(rposix.get_saved_errno())) self.is_enabled = True + @jit.dont_look_inside def enable_jitlog(self, fileno): # initialize the jit log from rpython.rlib import jitlog as jl From pypy.commits at gmail.com Sat Jul 9 17:34:17 2016 From: pypy.commits at gmail.com (mattip) Date: Sat, 09 Jul 2016 14:34:17 -0700 (PDT) Subject: [pypy-commit] pypy call-via-pyobj: backout 3c9d24675b27; wrong code, wasn't supposed to be pushed Message-ID: <57816dd9.d11b1c0a.1be01.7027@mx.google.com> Author: Matti Picus Branch: call-via-pyobj Changeset: r85636:a2f929e92587 Date: 2016-07-08 21:40 -0400 http://bitbucket.org/pypy/pypy/changeset/a2f929e92587/ Log: backout 3c9d24675b27; wrong code, wasn't supposed to be pushed diff --git a/pypy/module/cpyext/typeobject.py b/pypy/module/cpyext/typeobject.py --- a/pypy/module/cpyext/typeobject.py +++ b/pypy/module/cpyext/typeobject.py @@ -794,27 +794,15 @@ # While this is a hack, cpython does it as well. w_metatype = space.w_type - base = py_type.c_tp_base - bases_set = False - if base and not space.is_w(from_ref(space, base), space.w_int): - # inheriting tp_as_* slots - # set these now so the functions in add_operators() will have offsets - # XXX why is int special (inifinite recursion if these are set on - # a subclass of int, i.e. Enum in test_int_subtype in test_intobject.py) - if not py_type.c_tp_as_number: py_type.c_tp_as_number = base.c_tp_as_number - if not py_type.c_tp_as_sequence: py_type.c_tp_as_sequence = base.c_tp_as_sequence - if not py_type.c_tp_as_mapping: py_type.c_tp_as_mapping = base.c_tp_as_mapping - if not py_type.c_tp_as_buffer: py_type.c_tp_as_buffer = base.c_tp_as_buffer - bases_set = True - w_obj = space.allocate_instance(W_PyCTypeObject, w_metatype) track_reference(space, py_obj, w_obj) w_obj.__init__(space, py_type) w_obj.ready() finish_type_2(space, py_type, w_obj) - if base and not bases_set: - # inheriting tp_as_* slots + # inheriting tp_as_* slots + base = py_type.c_tp_base + if base: if not py_type.c_tp_as_number: py_type.c_tp_as_number = base.c_tp_as_number if not py_type.c_tp_as_sequence: py_type.c_tp_as_sequence = base.c_tp_as_sequence if not py_type.c_tp_as_mapping: py_type.c_tp_as_mapping = base.c_tp_as_mapping From pypy.commits at gmail.com Sat Jul 9 17:34:21 2016 From: pypy.commits at gmail.com (mattip) Date: Sat, 09 Jul 2016 14:34:21 -0700 (PDT) Subject: [pypy-commit] pypy call-via-pyobj: rename, expand passing test to show how subclass can affect base class's nb_multiply Message-ID: <57816ddd.a1c8c20a.c2594.ffffe0f0@mx.google.com> Author: Matti Picus Branch: call-via-pyobj Changeset: r85638:d87275ed7128 Date: 2016-07-09 07:54 -0400 http://bitbucket.org/pypy/pypy/changeset/d87275ed7128/ Log: rename, expand passing test to show how subclass can affect base class's nb_multiply passes w/-A and untranslated diff --git a/pypy/module/cpyext/test/test_arraymodule.py b/pypy/module/cpyext/test/test_arraymodule.py --- a/pypy/module/cpyext/test/test_arraymodule.py +++ b/pypy/module/cpyext/test/test_arraymodule.py @@ -78,13 +78,26 @@ rra = pickle.loads(s) # rra is arr backwards #assert arr.tolist() == rra.tolist() - def test_binop_mul_impl(self): + def test_subclass(self): # check that rmul is called module = self.import_module(name='array') arr = module.array('i', [2]) res = [1, 2, 3] * arr assert res == [1, 2, 3, 1, 2, 3] + + arr = module.arraybase('i', [2]) + res = [1, 2, 3] * arr + assert res == [1, 2, 3, 1, 2, 3] + + # switch nb_multiply on Arraytype, + # switches BaseArraytpye as well since tp_as_number is a reference module.switch_multiply() + + arr = module.array('i', [2]) + res = [1, 2, 3] * arr + assert res == [2, 4, 6] + + arr = module.arraybase('i', [2]) res = [1, 2, 3] * arr assert res == [2, 4, 6] From pypy.commits at gmail.com Sat Jul 9 17:34:23 2016 From: pypy.commits at gmail.com (mattip) Date: Sat, 09 Jul 2016 14:34:23 -0700 (PDT) Subject: [pypy-commit] pypy call-via-pyobj: cleanup Message-ID: <57816ddf.a7ddc20a.eeb23.ffffca20@mx.google.com> Author: Matti Picus Branch: call-via-pyobj Changeset: r85639:64b0facf01f3 Date: 2016-07-09 16:16 -0400 http://bitbucket.org/pypy/pypy/changeset/64b0facf01f3/ Log: cleanup diff --git a/pypy/module/cpyext/test/array.c b/pypy/module/cpyext/test/array.c --- a/pypy/module/cpyext/test/array.c +++ b/pypy/module/cpyext/test/array.c @@ -2147,8 +2147,6 @@ static PyObject * switch_multiply(void) { - fprintf(stdout, "switching nb_multiply from %p to %p\n", - Arraytype.tp_as_number->nb_multiply, array_base_multiply); Arraytype.tp_as_number->nb_multiply = array_base_multiply; Py_RETURN_NONE; }; diff --git a/pypy/module/cpyext/typeobject.py b/pypy/module/cpyext/typeobject.py --- a/pypy/module/cpyext/typeobject.py +++ b/pypy/module/cpyext/typeobject.py @@ -717,8 +717,6 @@ def py_type_ready(space, pto): if pto.c_tp_flags & Py_TPFLAGS_READY: - name = rffi.charp2str(pto.c_tp_name) - print 'py_type_ready',name, 'but PyTP_FLAGS_READY is set' return type_realize(space, rffi.cast(PyObject, pto)) From pypy.commits at gmail.com Sat Jul 9 17:34:19 2016 From: pypy.commits at gmail.com (mattip) Date: Sat, 09 Jul 2016 14:34:19 -0700 (PDT) Subject: [pypy-commit] pypy call-via-pyobj: tp_as_sequence provides a __mul__ method, was making descriptor inheritance test fail Message-ID: <57816ddb.06a81c0a.93f93.ffff9e51@mx.google.com> Author: Matti Picus Branch: call-via-pyobj Changeset: r85637:a06b2c93707b Date: 2016-07-08 21:44 -0400 http://bitbucket.org/pypy/pypy/changeset/a06b2c93707b/ Log: tp_as_sequence provides a __mul__ method, was making descriptor inheritance test fail diff --git a/pypy/module/cpyext/test/array.c b/pypy/module/cpyext/test/array.c --- a/pypy/module/cpyext/test/array.c +++ b/pypy/module/cpyext/test/array.c @@ -2267,7 +2267,7 @@ 0, /* tp_compare */ (reprfunc)array_repr, /* tp_repr */ 0, /* tp_as_number*/ - &array_as_sequence, /* tp_as_sequence*/ + 0, /* tp_as_sequence*/ &array_as_mapping, /* tp_as_mapping*/ 0, /* tp_hash */ 0, /* tp_call */ From pypy.commits at gmail.com Sat Jul 9 17:34:26 2016 From: pypy.commits at gmail.com (mattip) Date: Sat, 09 Jul 2016 14:34:26 -0700 (PDT) Subject: [pypy-commit] pypy call-via-pyobj: close branch to be merged Message-ID: <57816de2.c92bc20a.88d4f.7664@mx.google.com> Author: Matti Picus Branch: call-via-pyobj Changeset: r85641:0cf4e1c5f460 Date: 2016-07-09 16:19 -0400 http://bitbucket.org/pypy/pypy/changeset/0cf4e1c5f460/ Log: close branch to be merged From pypy.commits at gmail.com Sat Jul 9 17:34:24 2016 From: pypy.commits at gmail.com (mattip) Date: Sat, 09 Jul 2016 14:34:24 -0700 (PDT) Subject: [pypy-commit] pypy call-via-pyobj: abandon last set of changes which only made method substitution worse Message-ID: <57816de0.6237c20a.351a1.ffffc6c0@mx.google.com> Author: Matti Picus Branch: call-via-pyobj Changeset: r85640:0f7983336c59 Date: 2016-07-09 16:18 -0400 http://bitbucket.org/pypy/pypy/changeset/0f7983336c59/ Log: abandon last set of changes which only made method substitution worse From pypy.commits at gmail.com Sat Jul 9 17:34:30 2016 From: pypy.commits at gmail.com (mattip) Date: Sat, 09 Jul 2016 14:34:30 -0700 (PDT) Subject: [pypy-commit] pypy default: document merged branches Message-ID: <57816de6.94a51c0a.fbcf0.e545@mx.google.com> Author: Matti Picus Branch: Changeset: r85643:42ee7e5a0650 Date: 2016-07-09 16:31 -0400 http://bitbucket.org/pypy/pypy/changeset/42ee7e5a0650/ Log: document merged branches diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -53,3 +53,14 @@ Refactor PyTupleObject to look like cpython's and allow subclassing PyTuple_Type + +.. branch: call-via-pyobj + +Use offsets from PyTypeObject to find actual c function to call rather than +fixed functions, allows function override after PyType_Ready is called + +.. branch: issue2335 + +Avoid exhausting the stack in the JIT due to successive guard +failures in the same Python function ending up as successive levels of +RPython functions, while at app-level the traceback is very short From pypy.commits at gmail.com Sat Jul 9 17:34:28 2016 From: pypy.commits at gmail.com (mattip) Date: Sat, 09 Jul 2016 14:34:28 -0700 (PDT) Subject: [pypy-commit] pypy default: merge call-via-pyobj which uses offset-from-PyTypeObject when calling functions Message-ID: <57816de4.8f1d1c0a.d6f90.30fe@mx.google.com> Author: Matti Picus Branch: Changeset: r85642:221017a4d9f7 Date: 2016-07-09 16:20 -0400 http://bitbucket.org/pypy/pypy/changeset/221017a4d9f7/ Log: merge call-via-pyobj which uses offset-from-PyTypeObject when calling functions diff --git a/pypy/module/cpyext/methodobject.py b/pypy/module/cpyext/methodobject.py --- a/pypy/module/cpyext/methodobject.py +++ b/pypy/module/cpyext/methodobject.py @@ -1,4 +1,4 @@ -from rpython.rtyper.lltypesystem import lltype, rffi +from rpython.rtyper.lltypesystem import lltype, rffi, llmemory from pypy.interpreter.baseobjspace import W_Root from pypy.interpreter.error import OperationError, oefmt @@ -10,9 +10,10 @@ from pypy.module.cpyext.api import ( CONST_STRING, METH_CLASS, METH_COEXIST, METH_KEYWORDS, METH_NOARGS, METH_O, METH_STATIC, METH_VARARGS, PyObject, PyObjectFields, bootstrap_function, - build_type_checkers, cpython_api, cpython_struct, generic_cpy_call) + build_type_checkers, cpython_api, cpython_struct, generic_cpy_call, + PyTypeObjectPtr) from pypy.module.cpyext.pyobject import ( - Py_DecRef, from_ref, make_ref, make_typedescr) + Py_DecRef, from_ref, make_ref, as_pyobj, make_typedescr) PyCFunction_typedef = rffi.COpaquePtr(typedef='PyCFunction') PyCFunction = lltype.Ptr(lltype.FuncType([PyObject, PyObject], PyObject)) @@ -151,28 +152,45 @@ class W_PyCWrapperObject(W_Root): def __init__(self, space, pto, method_name, wrapper_func, - wrapper_func_kwds, doc, func): + wrapper_func_kwds, doc, func, offset=None): self.space = space self.method_name = method_name self.wrapper_func = wrapper_func self.wrapper_func_kwds = wrapper_func_kwds self.doc = doc self.func = func + self.offset = offset pyo = rffi.cast(PyObject, pto) w_type = from_ref(space, pyo) assert isinstance(w_type, W_TypeObject) self.w_objclass = w_type def call(self, space, w_self, w_args, w_kw): + func_to_call = self.func + if self.offset: + pto = as_pyobj(space, self.w_objclass) + # make ptr the equivalent of this, using the offsets + #func_to_call = rffi.cast(rffi.VOIDP, ptr.c_tp_as_number.c_nb_multiply) + if pto: + cptr = rffi.cast(rffi.CCHARP, pto) + for o in self.offset: + ptr = rffi.cast(rffi.VOIDPP, rffi.ptradd(cptr, o))[0] + cptr = rffi.cast(rffi.CCHARP, ptr) + func_to_call = rffi.cast(rffi.VOIDP, cptr) + else: + # Should never happen, assert to get a traceback + assert False, "failed to convert w_type %s to PyObject" % str( + self.w_objclass) + assert func_to_call if self.wrapper_func is None: assert self.wrapper_func_kwds is not None - return self.wrapper_func_kwds(space, w_self, w_args, self.func, + return self.wrapper_func_kwds(space, w_self, w_args, func_to_call, w_kw) if space.is_true(w_kw): raise oefmt(space.w_TypeError, "wrapper %s doesn't take any keyword arguments", self.method_name) - return self.wrapper_func(space, w_self, w_args, self.func) + return self.wrapper_func(space, w_self, w_args, func_to_call) def descr_method_repr(self): return self.space.wrap("" % @@ -301,12 +319,6 @@ def PyDescr_NewClassMethod(space, w_type, method): return space.wrap(W_PyCClassMethodObject(space, method, w_type)) -def PyDescr_NewWrapper(space, pto, method_name, wrapper_func, - wrapper_func_kwds, doc, func): - # not exactly the API sig - return space.wrap(W_PyCWrapperObject(space, pto, method_name, - wrapper_func, wrapper_func_kwds, doc, func)) - @cpython_api([lltype.Ptr(PyMethodDef), PyObject, CONST_STRING], PyObject) def Py_FindMethod(space, table, w_obj, name_ptr): """Return a bound method object for an extension type implemented in diff --git a/pypy/module/cpyext/test/array.c b/pypy/module/cpyext/test/array.c --- a/pypy/module/cpyext/test/array.c +++ b/pypy/module/cpyext/test/array.c @@ -2144,6 +2144,13 @@ return array_new(type, args, NULL); } +static PyObject * +switch_multiply(void) +{ + Arraytype.tp_as_number->nb_multiply = array_base_multiply; + Py_RETURN_NONE; +}; + PyDoc_STRVAR(module_doc, "This module defines an object type which can efficiently represent\n\ an array of basic values: characters, integers, floating point\n\ @@ -2394,6 +2401,7 @@ /* No functions in array module. */ static PyMethodDef a_methods[] = { {"_reconstruct", (PyCFunction)_reconstruct, METH_VARARGS, NULL}, + {"switch_multiply", (PyCFunction)switch_multiply, METH_NOARGS, NULL}, {NULL, NULL, 0, NULL} /* Sentinel */ }; diff --git a/pypy/module/cpyext/test/test_arraymodule.py b/pypy/module/cpyext/test/test_arraymodule.py --- a/pypy/module/cpyext/test/test_arraymodule.py +++ b/pypy/module/cpyext/test/test_arraymodule.py @@ -84,3 +84,7 @@ arr = module.array('i', [2]) res = [1, 2, 3] * arr assert res == [1, 2, 3, 1, 2, 3] + module.switch_multiply() + res = [1, 2, 3] * arr + assert res == [2, 4, 6] + diff --git a/pypy/module/cpyext/typeobject.py b/pypy/module/cpyext/typeobject.py --- a/pypy/module/cpyext/typeobject.py +++ b/pypy/module/cpyext/typeobject.py @@ -17,9 +17,9 @@ generic_cpy_call, Py_TPFLAGS_READY, Py_TPFLAGS_READYING, Py_TPFLAGS_HEAPTYPE, METH_VARARGS, METH_KEYWORDS, CANNOT_FAIL, Py_TPFLAGS_HAVE_GETCHARBUFFER, build_type_checkers, StaticObjectBuilder, - PyObjectFields, Py_TPFLAGS_BASETYPE) + PyObjectFields, Py_TPFLAGS_BASETYPE, PyTypeObject, PyTypeObjectPtr) from pypy.module.cpyext.methodobject import (W_PyCClassMethodObject, - PyDescr_NewWrapper, PyCFunction_NewEx, PyCFunction_typedef, PyMethodDef, + W_PyCWrapperObject, PyCFunction_NewEx, PyCFunction_typedef, PyMethodDef, W_PyCMethodObject, W_PyCFunctionObject) from pypy.module.cpyext.modsupport import convert_method_defs from pypy.module.cpyext.pyobject import ( @@ -30,7 +30,7 @@ from pypy.module.cpyext.state import State from pypy.module.cpyext.structmember import PyMember_GetOne, PyMember_SetOne from pypy.module.cpyext.typeobjectdefs import ( - PyTypeObjectPtr, PyTypeObject, PyGetSetDef, PyMemberDef, newfunc, + PyGetSetDef, PyMemberDef, newfunc, PyNumberMethods, PyMappingMethods, PySequenceMethods, PyBufferProcs) from pypy.objspace.std.typeobject import W_TypeObject, find_best_base @@ -296,6 +296,7 @@ for method_name, slot_names, wrapper_func, wrapper_func_kwds, doc in slotdefs_for_wrappers: if method_name in dict_w: continue + offset = [rffi.offsetof(lltype.typeOf(pto).TO, slot_names[0])] if len(slot_names) == 1: func = getattr(pto, slot_names[0]) else: @@ -303,14 +304,16 @@ struct = getattr(pto, slot_names[0]) if not struct: continue + offset.append(rffi.offsetof(lltype.typeOf(struct).TO, slot_names[1])) func = getattr(struct, slot_names[1]) func_voidp = rffi.cast(rffi.VOIDP, func) if not func: continue if wrapper_func is None and wrapper_func_kwds is None: continue - dict_w[method_name] = PyDescr_NewWrapper(space, pto, method_name, wrapper_func, - wrapper_func_kwds, doc, func_voidp) + w_obj = W_PyCWrapperObject(space, pto, method_name, wrapper_func, + wrapper_func_kwds, doc, func_voidp, offset=offset) + dict_w[method_name] = space.wrap(w_obj) if pto.c_tp_new: add_tp_new_wrapper(space, dict_w, pto) @@ -730,6 +733,7 @@ try: w_obj = _type_realize(space, py_obj) finally: + name = rffi.charp2str(pto.c_tp_name) pto.c_tp_flags &= ~Py_TPFLAGS_READYING pto.c_tp_flags |= Py_TPFLAGS_READY return w_obj @@ -811,7 +815,8 @@ base = pto.c_tp_base base_pyo = rffi.cast(PyObject, pto.c_tp_base) if base and not base.c_tp_flags & Py_TPFLAGS_READY: - type_realize(space, rffi.cast(PyObject, base_pyo)) + name = rffi.charp2str(base.c_tp_name) + type_realize(space, base_pyo) if base and not pto.c_ob_type: # will be filled later pto.c_ob_type = base.c_ob_type if not pto.c_tp_bases: From pypy.commits at gmail.com Sat Jul 9 17:34:32 2016 From: pypy.commits at gmail.com (mattip) Date: Sat, 09 Jul 2016 14:34:32 -0700 (PDT) Subject: [pypy-commit] pypy default: c89 syntax for msvc Message-ID: <57816de8.42431c0a.3a046.29ff@mx.google.com> Author: Matti Picus Branch: Changeset: r85644:bb6ab1bbb0e3 Date: 2016-07-09 16:42 -0400 http://bitbucket.org/pypy/pypy/changeset/bb6ab1bbb0e3/ Log: c89 syntax for msvc diff --git a/pypy/module/cpyext/test/test_bytearrayobject.py b/pypy/module/cpyext/test/test_bytearrayobject.py --- a/pypy/module/cpyext/test/test_bytearrayobject.py +++ b/pypy/module/cpyext/test/test_bytearrayobject.py @@ -87,11 +87,12 @@ module = self.import_extension('foo', [ ("getbytearray", "METH_NOARGS", """ - PyObject* s1 = PyByteArray_FromStringAndSize("test", 4); + const char *c; + PyObject *s2, *s1 = PyByteArray_FromStringAndSize("test", 4); if (s1 == NULL) return NULL; - const char* c = PyByteArray_AsString(s1); - PyObject* s2 = PyByteArray_FromStringAndSize(c, 4); + c = PyByteArray_AsString(s1); + s2 = PyByteArray_FromStringAndSize(c, 4); Py_DECREF(s1); return s2; """), diff --git a/pypy/module/cpyext/test/test_datetime.py b/pypy/module/cpyext/test/test_datetime.py --- a/pypy/module/cpyext/test/test_datetime.py +++ b/pypy/module/cpyext/test/test_datetime.py @@ -176,13 +176,15 @@ """), ("test_datetime_macros", "METH_NOARGS", """ + PyObject* obj; + PyDateTime_DateTime *dt; PyDateTime_IMPORT; if (!PyDateTimeAPI) { PyErr_SetString(PyExc_RuntimeError, "No PyDateTimeAPI"); return NULL; } - PyObject* obj = PyDateTime_FromDateAndTime(2000, 6, 6, 6, 6, 6, 6); - PyDateTime_DateTime* dt = (PyDateTime_DateTime*)obj; + obj = PyDateTime_FromDateAndTime(2000, 6, 6, 6, 6, 6, 6); + dt = (PyDateTime_DateTime*)obj; PyDateTime_GET_YEAR(obj); PyDateTime_GET_YEAR(dt); @@ -209,13 +211,15 @@ """), ("test_time_macros", "METH_NOARGS", """ + PyObject* obj; + PyDateTime_Time* t; PyDateTime_IMPORT; if (!PyDateTimeAPI) { PyErr_SetString(PyExc_RuntimeError, "No PyDateTimeAPI"); return NULL; } - PyObject* obj = PyTime_FromTime(6, 6, 6, 6); - PyDateTime_Time* t = (PyDateTime_Time*)obj; + obj = PyTime_FromTime(6, 6, 6, 6); + t = (PyDateTime_Time*)obj; PyDateTime_TIME_GET_HOUR(obj); PyDateTime_TIME_GET_HOUR(t); @@ -233,13 +237,15 @@ """), ("test_delta_macros", "METH_NOARGS", """ + PyObject* obj; + PyDateTime_Delta* delta; PyDateTime_IMPORT; if (!PyDateTimeAPI) { PyErr_SetString(PyExc_RuntimeError, "No PyDateTimeAPI"); return NULL; } - PyObject* obj = PyDelta_FromDSU(6, 6, 6); - PyDateTime_Delta* delta = (PyDateTime_Delta*)obj; + obj = PyDelta_FromDSU(6, 6, 6); + delta = (PyDateTime_Delta*)obj; #if defined(PYPY_VERSION) || PY_VERSION_HEX >= 0x03030000 // These macros are only defined in CPython 3.x and PyPy. From pypy.commits at gmail.com Sun Jul 10 05:07:23 2016 From: pypy.commits at gmail.com (arigo) Date: Sun, 10 Jul 2016 02:07:23 -0700 (PDT) Subject: [pypy-commit] pypy reverse-debugger: Add --color=dark or --color=light options to ask for colorized source code Message-ID: <5782104b.d11b1c0a.1be01.0ba5@mx.google.com> Author: Armin Rigo Branch: reverse-debugger Changeset: r85645:6d9691593717 Date: 2016-07-10 11:08 +0200 http://bitbucket.org/pypy/pypy/changeset/6d9691593717/ Log: Add --color=dark or --color=light options to ask for colorized source code diff --git a/rpython/translator/revdb/interact.py b/rpython/translator/revdb/interact.py --- a/rpython/translator/revdb/interact.py +++ b/rpython/translator/revdb/interact.py @@ -1,6 +1,6 @@ import sys, os, re import subprocess, socket -import traceback +import traceback, linecache from contextlib import contextmanager try: import readline @@ -16,7 +16,8 @@ class RevDebugControl(object): - def __init__(self, revdb_log_filename, executable=None): + def __init__(self, revdb_log_filename, executable=None, + pygments_background=None): with open(revdb_log_filename, 'rb') as f: header = f.readline() assert header.endswith('\n') @@ -28,7 +29,9 @@ executable = fields[1] if not os.path.isfile(executable): raise ValueError("executable %r not found" % (executable,)) - self.pgroup = ReplayProcessGroup(executable, revdb_log_filename) + linecacheoutput = self.getlinecacheoutput(pygments_background) + self.pgroup = ReplayProcessGroup(executable, revdb_log_filename, + linecacheoutput) self.print_extra_pending_info = None def interact(self): @@ -365,3 +368,21 @@ # self._bp_new(argument, 'W', compiled_code, nids=nids) self.pgroup.update_watch_values() + + def getlinecacheoutput(self, pygments_background): + if not pygments_background or pygments_background == 'off': + return + try: + from pygments import highlight + from pygments.lexers import PythonLexer + from pygments.formatters import TerminalFormatter + except ImportError: + return None + # + lexer = PythonLexer() + fmt = TerminalFormatter(bg=pygments_background) + # + def linecacheoutput(filename, lineno): + line = linecache.getline(filename, lineno) + return highlight(line, lexer, fmt) + return linecacheoutput diff --git a/rpython/translator/revdb/process.py b/rpython/translator/revdb/process.py --- a/rpython/translator/revdb/process.py +++ b/rpython/translator/revdb/process.py @@ -65,7 +65,8 @@ def __init__(self, pid, control_socket, breakpoints_cache=AllBreakpoints(), - printed_objects=frozenset()): + printed_objects=frozenset(), + linecacheoutput=None): self.pid = pid self.control_socket = control_socket self.tainted = False @@ -75,6 +76,7 @@ # either already discovered in this child # (if uid < currently_created_objects), or that will # automatically be discovered when we move forward + self.linecacheoutput = linecacheoutput or linecache.getline def _recv_all(self, size): pieces = [] @@ -134,7 +136,8 @@ self.expect_ready() other = ReplayProcess(child_pid, s1, breakpoints_cache=self.breakpoints_cache, - printed_objects=self.printed_objects) + printed_objects=self.printed_objects, + linecacheoutput=self.linecacheoutput) other.expect_ready() #print >> sys.stderr, 'CLONED', self.current_time return other @@ -184,7 +187,7 @@ self.update_times(msg) break elif msg.cmd == ANSWER_LINECACHE: - line = linecache.getline(msg.extra, msg.arg1) + line = self.linecacheoutput(msg.extra, msg.arg1) if line == '': line = '?' if msg.arg2: # strip? @@ -215,13 +218,14 @@ STEP_RATIO = 0.25 # subprocess n is between subprocess n-1 # and the end, at this fraction of interval - def __init__(self, executable, revdb_log_filename): + def __init__(self, executable, revdb_log_filename, linecacheoutput=None): s1, s2 = socket.socketpair() initial_subproc = subprocess.Popen( [executable, '--revdb-replay', revdb_log_filename, str(s2.fileno())]) s2.close() - child = ReplayProcess(initial_subproc.pid, s1) + child = ReplayProcess(initial_subproc.pid, s1, + linecacheoutput=linecacheoutput) msg = child.expect(ANSWER_INIT, INIT_VERSION_NUMBER, Ellipsis) self.total_stop_points = msg.arg2 if self.total_stop_points == 0: diff --git a/rpython/translator/revdb/revdb.py b/rpython/translator/revdb/revdb.py --- a/rpython/translator/revdb/revdb.py +++ b/rpython/translator/revdb/revdb.py @@ -10,11 +10,14 @@ parser.add_argument('-x', '--executable', dest='executable', help='name of the executable file ' 'that recorded the log') + parser.add_argument('-c', '--color', dest='color', + help='colorize source code (dark,light,off)') options = parser.parse_args() sys.path.insert(0, os.path.abspath( os.path.join(__file__, '..', '..', '..', '..'))) from rpython.translator.revdb.interact import RevDebugControl - ctrl = RevDebugControl(options.log, executable=options.executable) + ctrl = RevDebugControl(options.log, executable=options.executable, + pygments_background=options.color) ctrl.interact() From pypy.commits at gmail.com Sun Jul 10 05:11:19 2016 From: pypy.commits at gmail.com (arigo) Date: Sun, 10 Jul 2016 02:11:19 -0700 (PDT) Subject: [pypy-commit] pypy reverse-debugger: Print the ImportError at least, don't silently ignore the option Message-ID: <57821137.05b01c0a.8fbb6.bae9@mx.google.com> Author: Armin Rigo Branch: reverse-debugger Changeset: r85646:c5e982ac074f Date: 2016-07-10 11:12 +0200 http://bitbucket.org/pypy/pypy/changeset/c5e982ac074f/ Log: Print the ImportError at least, don't silently ignore the option diff --git a/rpython/translator/revdb/interact.py b/rpython/translator/revdb/interact.py --- a/rpython/translator/revdb/interact.py +++ b/rpython/translator/revdb/interact.py @@ -376,7 +376,8 @@ from pygments import highlight from pygments.lexers import PythonLexer from pygments.formatters import TerminalFormatter - except ImportError: + except ImportError as e: + print >> sys.stderr, 'ImportError: %s' % (e,) return None # lexer = PythonLexer() From pypy.commits at gmail.com Sun Jul 10 08:31:45 2016 From: pypy.commits at gmail.com (mattip) Date: Sun, 10 Jul 2016 05:31:45 -0700 (PDT) Subject: [pypy-commit] pypy override-tp_as-methods: refactor code and add a failing test that passes with -A Message-ID: <57824031.488e1c0a.7a4d9.4f4d@mx.google.com> Author: Matti Picus Branch: override-tp_as-methods Changeset: r85647:1ebdd0291415 Date: 2016-07-10 08:30 -0400 http://bitbucket.org/pypy/pypy/changeset/1ebdd0291415/ Log: refactor code and add a failing test that passes with -A diff --git a/pypy/module/cpyext/test/test_typeobject.py b/pypy/module/cpyext/test/test_typeobject.py --- a/pypy/module/cpyext/test/test_typeobject.py +++ b/pypy/module/cpyext/test/test_typeobject.py @@ -793,11 +793,6 @@ if (!PyArg_ParseTuple(args, "l", &intval)) return NULL; - IntLike_Type.tp_as_number = &intlike_as_number; - IntLike_Type.tp_flags |= Py_TPFLAGS_DEFAULT | Py_TPFLAGS_CHECKTYPES; - intlike_as_number.nb_add = intlike_nb_add; - intlike_as_number.nb_power = intlike_nb_pow; - if (PyType_Ready(&IntLike_Type) < 0) return NULL; intObj = PyObject_New(IntLikeObject, &IntLike_Type); if (!intObj) { return NULL; @@ -814,8 +809,6 @@ if (!PyArg_ParseTuple(args, "l", &intval)) return NULL; - IntLike_Type_NoOp.tp_flags |= Py_TPFLAGS_DEFAULT | Py_TPFLAGS_CHECKTYPES; - if (PyType_Ready(&IntLike_Type_NoOp) < 0) return NULL; intObjNoOp = PyObject_New(IntLikeObjectNoOp, &IntLike_Type_NoOp); if (!intObjNoOp) { return NULL; @@ -823,6 +816,12 @@ intObjNoOp->ival = intval; return (PyObject *)intObjNoOp; + """), + ("switch_as_number", 'METH_NOARGS', + """ + intlike_as_number2.nb_subtract = &intlike_nb_subtract; + IntLike_Type.tp_as_number = &intlike_as_number2; + Py_RETURN_NONE; """)], prologue= """ #include @@ -856,7 +855,19 @@ val2 = ((IntLikeObject *)(other))->ival; return PyInt_FromLong((int)pow(val1,val2)); - } + } + static PyObject * + intlike_nb_subtract(PyObject *self, PyObject *other) + { + long val2, val1 = ((IntLikeObject *)(self))->ival; + if (PyInt_Check(other)) { + long val2 = PyInt_AsLong(other); + return PyInt_FromLong(val1-val2); + } + + val2 = ((IntLikeObject *)(other))->ival; + return PyInt_FromLong(val1-val2); + } PyTypeObject IntLike_Type = { PyObject_HEAD_INIT(0) @@ -865,6 +876,7 @@ /*tp_basicsize*/ sizeof(IntLikeObject), }; static PyNumberMethods intlike_as_number; + static PyNumberMethods intlike_as_number2; typedef struct { @@ -878,6 +890,15 @@ /*tp_name*/ "IntLikeNoOp", /*tp_basicsize*/ sizeof(IntLikeObjectNoOp), }; + """, more_init= + """ + IntLike_Type.tp_as_number = &intlike_as_number; + IntLike_Type.tp_flags |= Py_TPFLAGS_DEFAULT | Py_TPFLAGS_CHECKTYPES; + intlike_as_number.nb_add = intlike_nb_add; + intlike_as_number.nb_power = intlike_nb_pow; + if (PyType_Ready(&IntLike_Type) < 0) return; + IntLike_Type_NoOp.tp_flags |= Py_TPFLAGS_DEFAULT | Py_TPFLAGS_CHECKTYPES; + if (PyType_Ready(&IntLike_Type_NoOp) < 0) return; """) a = module.newInt(1) b = module.newInt(2) @@ -888,6 +909,13 @@ assert (d + a) == 5 assert pow(d,b) == 16 + # Make sure tp_as_number slots can be assigned to after PyType_Ready, + # even if the slot was NULL at PyType_Ready + module.switch_as_number() + a = module.newInt(4) + b = module.newInt(2) + assert (a - b) == 2 + def test_tp_new_in_subclass_of_type(self): module = self.import_module(name='foo3') #print('calling module.footype()...') From pypy.commits at gmail.com Sun Jul 10 12:58:25 2016 From: pypy.commits at gmail.com (arigo) Date: Sun, 10 Jul 2016 09:58:25 -0700 (PDT) Subject: [pypy-commit] pypy reverse-debugger: Improve the test. Now it correctly fails if we comment out Message-ID: <57827eb1.4dd11c0a.ee9bc.464c@mx.google.com> Author: Armin Rigo Branch: reverse-debugger Changeset: r85648:f516429f49f3 Date: 2016-07-10 18:59 +0200 http://bitbucket.org/pypy/pypy/changeset/f516429f49f3/ Log: Improve the test. Now it correctly fails if we comment out "#define HAVE_PERSONALITY". diff --git a/rpython/translator/revdb/test/test_basic.py b/rpython/translator/revdb/test/test_basic.py --- a/rpython/translator/revdb/test/test_basic.py +++ b/rpython/translator/revdb/test/test_basic.py @@ -97,12 +97,13 @@ def compile(self, entry_point, backendopt=True, - withsmallfuncsets=None): + withsmallfuncsets=None, shared=False): t = Translation(entry_point, None, gc="boehm") self.t = t t.set_backend_extra_options(c_debug_defines=True) t.config.translation.reverse_debugger = True t.config.translation.lldebug0 = True + t.config.translation.shared = shared if withsmallfuncsets is not None: t.config.translation.withsmallfuncsets = withsmallfuncsets if not backendopt: diff --git a/rpython/translator/revdb/test/test_raw.py b/rpython/translator/revdb/test/test_raw.py --- a/rpython/translator/revdb/test/test_raw.py +++ b/rpython/translator/revdb/test/test_raw.py @@ -65,7 +65,7 @@ revdb.stop_point() return 9 - compile(cls, main, backendopt=False) + compile(cls, main, backendopt=False, shared=True) run(cls, '') rdb = fetch_rdb(cls, [cls.exename]) #assert len(rdb.rdb_struct) >= 4 From pypy.commits at gmail.com Sun Jul 10 14:49:36 2016 From: pypy.commits at gmail.com (raffael_t) Date: Sun, 10 Jul 2016 11:49:36 -0700 (PDT) Subject: [pypy-commit] pypy py3.5-async: Create missing visit_AsyncFunctionDef with codegenerator for asyncfunctions in codegen (fixes startup errors) Message-ID: <578298c0.85c11c0a.3e9a4.393e@mx.google.com> Author: Raffael Tfirst Branch: py3.5-async Changeset: r85649:0ebb9c362c57 Date: 2016-07-10 20:48 +0200 http://bitbucket.org/pypy/pypy/changeset/0ebb9c362c57/ Log: Create missing visit_AsyncFunctionDef with codegenerator for asyncfunctions in codegen (fixes startup errors) diff --git a/pypy/interpreter/astcompiler/codegen.py b/pypy/interpreter/astcompiler/codegen.py --- a/pypy/interpreter/astcompiler/codegen.py +++ b/pypy/interpreter/astcompiler/codegen.py @@ -387,6 +387,30 @@ for i in range(len(func.decorator_list)): self.emit_op_arg(ops.CALL_FUNCTION, 1) self.name_op(func.name, ast.Store) + + def visit_AsyncFunctionDef(self, func): + self.update_position(func.lineno, True) + # Load decorators first, but apply them after the function is created. + self.visit_sequence(func.decorator_list) + args = func.args + assert isinstance(args, ast.arguments) + kw_default_count = 0 + if args.kwonlyargs: + kw_default_count = self._visit_kwonlydefaults(args) + self.visit_sequence(args.defaults) + num_annotations = self._visit_annotations(func, args, func.returns) + num_defaults = len(args.defaults) if args.defaults is not None else 0 + oparg = num_defaults + oparg |= kw_default_count << 8 + oparg |= num_annotations << 16 + code, qualname = self.sub_scope(AsyncFunctionCodeGenerator, func.name, func, + func.lineno) + self._make_function(code, oparg, qualname=qualname) + # Apply decorators. + if func.decorator_list: + for i in range(len(func.decorator_list)): + self.emit_op_arg(ops.CALL_FUNCTION, 1) + self.name_op(func.name, ast.Store) def visit_Lambda(self, lam): self.update_position(lam.lineno) @@ -1539,6 +1563,21 @@ for i in range(start, len(func.body)): func.body[i].walkabout(self) +class AsyncFunctionCodeGenerator(AbstractFunctionCodeGenerator): + + def _compile(self, func): + assert isinstance(func, ast.AsyncFunctionDef) + has_docstring = self.ensure_docstring_constant(func.body) + start = 1 if has_docstring else 0 + args = func.args + assert isinstance(args, ast.arguments) + if args.args: + self.argcount = len(args.args) + if args.kwonlyargs: + self.kwonlyargcount = len(args.kwonlyargs) + if func.body: + for i in range(start, len(func.body)): + func.body[i].walkabout(self) class LambdaCodeGenerator(AbstractFunctionCodeGenerator): diff --git a/pypy/interpreter/astcompiler/symtable.py b/pypy/interpreter/astcompiler/symtable.py --- a/pypy/interpreter/astcompiler/symtable.py +++ b/pypy/interpreter/astcompiler/symtable.py @@ -388,7 +388,6 @@ func.args.walkabout(self) self.visit_sequence(func.body) self.pop_scope() - def visit_Return(self, ret): self.scope.note_return(ret) From pypy.commits at gmail.com Mon Jul 11 03:10:36 2016 From: pypy.commits at gmail.com (Raemi) Date: Mon, 11 Jul 2016 00:10:36 -0700 (PDT) Subject: [pypy-commit] pypy stmgc-c8: protect setting the execution context threadlocal with inevitable Message-ID: <5783466c.88cb1c0a.7f51b.a154@mx.google.com> Author: Remi Meier Branch: stmgc-c8 Changeset: r85650:8c9fbb5b5f31 Date: 2016-07-11 09:09 +0200 http://bitbucket.org/pypy/pypy/changeset/8c9fbb5b5f31/ Log: protect setting the execution context threadlocal with inevitable Seems to fix a crash where the threadlocal was corrupted. If not setting the TL in an inevitable TX, the superclass' try_enter_thread would not re-set the EC after an abort. Instead, it would detect that the TL already has a value and probably reuse it in multiple threads. diff --git a/pypy/module/pypystm/threadlocals.py b/pypy/module/pypystm/threadlocals.py --- a/pypy/module/pypystm/threadlocals.py +++ b/pypy/module/pypystm/threadlocals.py @@ -42,6 +42,11 @@ interval = space.actionflag.getcheckinterval() rstm.set_transaction_length(interval / 10000.0) + def _set_ec(self, ec, register_in_valuedict=True): + # must turn inevitable, for raw_thread_local.set(ec) + rstm.become_inevitable() + OSThreadLocals._set_ec(self, ec, register_in_valuedict) + def leave_thread(self, space): # must turn inevitable, for raw_thread_local.set(None) rstm.become_inevitable() From pypy.commits at gmail.com Mon Jul 11 05:24:55 2016 From: pypy.commits at gmail.com (raffael_t) Date: Mon, 11 Jul 2016 02:24:55 -0700 (PDT) Subject: [pypy-commit] pypy py3.5-async: Add syntax tests for invalid async await entries Message-ID: <578365e7.05261c0a.b28bd.eb96@mx.google.com> Author: Raffael Tfirst Branch: py3.5-async Changeset: r85651:5df7aae1f80a Date: 2016-07-11 11:24 +0200 http://bitbucket.org/pypy/pypy/changeset/5df7aae1f80a/ Log: Add syntax tests for invalid async await entries diff --git a/pypy/interpreter/test/test_syntax.py b/pypy/interpreter/test/test_syntax.py --- a/pypy/interpreter/test/test_syntax.py +++ b/pypy/interpreter/test/test_syntax.py @@ -78,6 +78,27 @@ def f(): (i for i in x) = 10 + + async def foo(a=await something()): + pass + + async def foo(): + [i async for i in els] + + async def foo(): + await + + def foo(): + await something() + + async def foo(): + yield + + async def foo(): + yield from [] + + async def foo(): + await await fut """) From pypy.commits at gmail.com Mon Jul 11 05:35:52 2016 From: pypy.commits at gmail.com (Raemi) Date: Mon, 11 Jul 2016 02:35:52 -0700 (PDT) Subject: [pypy-commit] pypy stmgc-c8: add very direct test for the previous commit Message-ID: <57836878.d4e41c0a.8c41.6c5e@mx.google.com> Author: Remi Meier Branch: stmgc-c8 Changeset: r85652:e5f4066bc640 Date: 2016-07-11 11:34 +0200 http://bitbucket.org/pypy/pypy/changeset/e5f4066bc640/ Log: add very direct test for the previous commit diff --git a/pypy/module/pypystm/test/test_local.py b/pypy/module/pypystm/test/test_local.py --- a/pypy/module/pypystm/test/test_local.py +++ b/pypy/module/pypystm/test/test_local.py @@ -11,3 +11,37 @@ import pypystm return pypystm.local """) + + +def test_direct_call_to_become_inevitable(): + # this test directly checks if we call rstm.become_inevitable() in the + # right places (before modifying the real threadlocal). Could possibly be + # tested in a better way... + from pypy.module.pypystm.threadlocals import STMThreadLocals + from rpython.rlib import rstm + from pypy.interpreter.executioncontext import ExecutionContext + + class FakeEC(ExecutionContext): + def __init__(self): pass + class FakeConfig: + class translation: + rweakref = False + class FakeSpace: + config = FakeConfig() + def createexecutioncontext(self): + return FakeEC() + call_counter = [0] + def fake_become_inevitable(): + call_counter[0] += 1 + rstm.become_inevitable = fake_become_inevitable + + space = FakeSpace() + + l = STMThreadLocals(space) + assert call_counter[0] == 0 + l.try_enter_thread(space) + assert call_counter[0] == 1 + l.enter_thread(space) + assert call_counter[0] == 2 + l.leave_thread(space) + assert call_counter[0] == 3 From pypy.commits at gmail.com Mon Jul 11 07:04:18 2016 From: pypy.commits at gmail.com (arigo) Date: Mon, 11 Jul 2016 04:04:18 -0700 (PDT) Subject: [pypy-commit] pypy use-mmap: comment Message-ID: <57837d32.021a1c0a.b958f.fdf8@mx.google.com> Author: Armin Rigo Branch: use-mmap Changeset: r85653:b5611bc3d970 Date: 2016-07-11 13:00 +0200 http://bitbucket.org/pypy/pypy/changeset/b5611bc3d970/ Log: comment diff --git a/rpython/memory/gc/minimarkpage.py b/rpython/memory/gc/minimarkpage.py --- a/rpython/memory/gc/minimarkpage.py +++ b/rpython/memory/gc/minimarkpage.py @@ -131,6 +131,11 @@ # arenas that were already in the 'dead_arenas' list at that # point (from the previous major collection) are really # returned to the OS. + # Memory usage goes down quickly during the incremental + # major collection and up slowly the rest of the time. The + # point of these two lists is to avoid constantly freeing and + # re-allocating arenas: we return to the OS the arenas that + # have been unused for a complete cycle already. self.dying_arenas = ARENA_NULL self.dead_arenas = ARENA_NULL # From pypy.commits at gmail.com Mon Jul 11 08:32:53 2016 From: pypy.commits at gmail.com (arigo) Date: Mon, 11 Jul 2016 05:32:53 -0700 (PDT) Subject: [pypy-commit] pypy use-mmap: Windows support Message-ID: <578391f5.d23f1c0a.2c640.5774@mx.google.com> Author: Armin Rigo Branch: use-mmap Changeset: r85654:7b5ec14900f0 Date: 2016-07-11 14:34 +0200 http://bitbucket.org/pypy/pypy/changeset/7b5ec14900f0/ Log: Windows support diff --git a/rpython/rlib/rmmap.py b/rpython/rlib/rmmap.py --- a/rpython/rlib/rmmap.py +++ b/rpython/rlib/rmmap.py @@ -926,4 +926,12 @@ def free(ptr, map_size): VirtualFree_safe(ptr, 0, MEM_RELEASE) -# register_external here? + def allocate_memory_chunk(map_size): + # used by the memory allocator (in llarena.py, from minimarkpage.py) + null = lltype.nullptr(rffi.VOIDP.TO) + return VirtualAlloc_safe(null, map_size, MEM_COMMIT | MEM_RESERVE, + PAGE_READWRITE) + + def free_memory_chunk(addr, map_size): + # used by the memory allocator (in llarena.py, from minimarkpage.py) + VirtualFree_safe(addr, 0, MEM_RELEASE) From pypy.commits at gmail.com Mon Jul 11 09:17:27 2016 From: pypy.commits at gmail.com (arigo) Date: Mon, 11 Jul 2016 06:17:27 -0700 (PDT) Subject: [pypy-commit] cffi default: Use Py_ssize_t, not ssize_t directly Message-ID: <57839c67.9a4a1c0a.37639.2980@mx.google.com> Author: Armin Rigo Branch: Changeset: r2723:4479cc40cfbb Date: 2016-07-11 15:19 +0200 http://bitbucket.org/cffi/cffi/changeset/4479cc40cfbb/ Log: Use Py_ssize_t, not ssize_t directly diff --git a/c/_cffi_backend.c b/c/_cffi_backend.c --- a/c/_cffi_backend.c +++ b/c/_cffi_backend.c @@ -5472,7 +5472,7 @@ *offset = cf->cf_offset; } else { - ssize_t index = PyInt_AsSsize_t(fieldname); + Py_ssize_t index = PyInt_AsSsize_t(fieldname); if (index < 0 && PyErr_Occurred()) { PyErr_SetString(PyExc_TypeError, "field name or array index expected"); From pypy.commits at gmail.com Mon Jul 11 09:18:04 2016 From: pypy.commits at gmail.com (arigo) Date: Mon, 11 Jul 2016 06:18:04 -0700 (PDT) Subject: [pypy-commit] cffi default: Same here Message-ID: <57839c8c.cbc71c0a.152f3.7ecb@mx.google.com> Author: Armin Rigo Branch: Changeset: r2724:a3708c91be7b Date: 2016-07-11 15:19 +0200 http://bitbucket.org/cffi/cffi/changeset/a3708c91be7b/ Log: Same here diff --git a/c/_cffi_backend.c b/c/_cffi_backend.c --- a/c/_cffi_backend.c +++ b/c/_cffi_backend.c @@ -3778,7 +3778,7 @@ EPTYPE(um, uintmax_t, CT_PRIMITIVE_UNSIGNED) \ EPTYPE(pd, ptrdiff_t, CT_PRIMITIVE_SIGNED) \ EPTYPE(sz, size_t, CT_PRIMITIVE_UNSIGNED) \ - EPTYPE(ssz, ssize_t, CT_PRIMITIVE_SIGNED) + EPTYPE(ssz, Py_ssize_t, CT_PRIMITIVE_SIGNED) #ifdef HAVE_WCHAR_H # define ENUM_PRIMITIVE_TYPES_WCHAR \ From pypy.commits at gmail.com Mon Jul 11 09:19:44 2016 From: pypy.commits at gmail.com (arigo) Date: Mon, 11 Jul 2016 06:19:44 -0700 (PDT) Subject: [pypy-commit] pypy use-mmap: translation fix Message-ID: <57839cf0.83261c0a.851cf.7804@mx.google.com> Author: Armin Rigo Branch: use-mmap Changeset: r85655:849bfe24c677 Date: 2016-07-11 15:21 +0200 http://bitbucket.org/pypy/pypy/changeset/849bfe24c677/ Log: translation fix diff --git a/rpython/rlib/rmmap.py b/rpython/rlib/rmmap.py --- a/rpython/rlib/rmmap.py +++ b/rpython/rlib/rmmap.py @@ -912,8 +912,10 @@ case of a sandboxed process """ null = lltype.nullptr(rffi.VOIDP.TO) - res = VirtualAlloc_safe(null, map_size, MEM_COMMIT | MEM_RESERVE, - PAGE_EXECUTE_READWRITE) + prot = PAGE_EXECUTE_READWRITE + if we_are_translated(): + prot = NonConstant(prot) + res = VirtualAlloc_safe(null, map_size, MEM_COMMIT | MEM_RESERVE, prot) if not res: raise MemoryError arg = lltype.malloc(LPDWORD.TO, 1, zero=True, flavor='raw') @@ -929,8 +931,11 @@ def allocate_memory_chunk(map_size): # used by the memory allocator (in llarena.py, from minimarkpage.py) null = lltype.nullptr(rffi.VOIDP.TO) - return VirtualAlloc_safe(null, map_size, MEM_COMMIT | MEM_RESERVE, - PAGE_READWRITE) + prot = PAGE_READWRITE + if we_are_translated(): + prot = NonConstant(prot) + res = VirtualAlloc_safe(null, map_size, MEM_COMMIT | MEM_RESERVE, prot) + return res def free_memory_chunk(addr, map_size): # used by the memory allocator (in llarena.py, from minimarkpage.py) From pypy.commits at gmail.com Mon Jul 11 09:37:23 2016 From: pypy.commits at gmail.com (arigo) Date: Mon, 11 Jul 2016 06:37:23 -0700 (PDT) Subject: [pypy-commit] pypy default: Issue #2330: at least pygobject compiles like that Message-ID: <5783a113.4ccf1c0a.b6a45.51f1@mx.google.com> Author: Armin Rigo Branch: Changeset: r85656:f8b2350c1afe Date: 2016-07-11 15:38 +0200 http://bitbucket.org/pypy/pypy/changeset/f8b2350c1afe/ Log: Issue #2330: at least pygobject compiles like that diff --git a/pypy/module/cpyext/include/object.h b/pypy/module/cpyext/include/object.h --- a/pypy/module/cpyext/include/object.h +++ b/pypy/module/cpyext/include/object.h @@ -561,8 +561,10 @@ #define PyObject_TypeCheck(ob, tp) \ ((ob)->ob_type == (tp) || PyType_IsSubtype((ob)->ob_type, (tp))) -#define Py_TRASHCAN_SAFE_BEGIN(pyObj) -#define Py_TRASHCAN_SAFE_END(pyObj) +#define Py_TRASHCAN_SAFE_BEGIN(pyObj) do { +#define Py_TRASHCAN_SAFE_END(pyObj) ; } while(0); +/* note: the ";" at the start of Py_TRASHCAN_SAFE_END is needed + if the code has a label in front of the macro call */ /* Copied from CPython ----------------------------- */ PyAPI_FUNC(int) PyObject_AsReadBuffer(PyObject *, const void **, Py_ssize_t *); From pypy.commits at gmail.com Mon Jul 11 10:46:19 2016 From: pypy.commits at gmail.com (plan_rich) Date: Mon, 11 Jul 2016 07:46:19 -0700 (PDT) Subject: [pypy-commit] extradoc extradoc: slides for the compiler workshop (reveal.js) Message-ID: <5783b13b.45061c0a.8f3f1.99b3@mx.google.com> Author: Richard Plangger Branch: extradoc Changeset: r5650:25b5647471e6 Date: 2016-07-10 19:22 -0500 http://bitbucket.org/pypy/extradoc/changeset/25b5647471e6/ Log: slides for the compiler workshop (reveal.js) diff too long, truncating to 2000 out of 20939 lines diff --git a/talk/compiler-workshop-2016/slides/CONTRIBUTING.md b/talk/compiler-workshop-2016/slides/CONTRIBUTING.md new file mode 100644 --- /dev/null +++ b/talk/compiler-workshop-2016/slides/CONTRIBUTING.md @@ -0,0 +1,23 @@ +## Contributing + +Please keep the [issue tracker](http://github.com/hakimel/reveal.js/issues) limited to **bug reports**, **feature requests** and **pull requests**. + + +### Personal Support +If you have personal support or setup questions the best place to ask those are [StackOverflow](http://stackoverflow.com/questions/tagged/reveal.js). + + +### Bug Reports +When reporting a bug make sure to include information about which browser and operating system you are on as well as the necessary steps to reproduce the issue. If possible please include a link to a sample presentation where the bug can be tested. + + +### Pull Requests +- Should follow the coding style of the file you work in, most importantly: + - Tabs to indent + - Single-quoted strings +- Should be made towards the **dev branch** +- Should be submitted from a feature/topic branch (not your master) + + +### Plugins +Please do not submit plugins as pull requests. They should be maintained in their own separate repository. More information here: https://github.com/hakimel/reveal.js/wiki/Plugin-Guidelines diff --git a/talk/compiler-workshop-2016/slides/Gruntfile.js b/talk/compiler-workshop-2016/slides/Gruntfile.js new file mode 100644 --- /dev/null +++ b/talk/compiler-workshop-2016/slides/Gruntfile.js @@ -0,0 +1,176 @@ +/* global module:false */ +module.exports = function(grunt) { + var port = grunt.option('port') || 8000; + var base = grunt.option('base') || '.'; + + // Project configuration + grunt.initConfig({ + pkg: grunt.file.readJSON('package.json'), + meta: { + banner: + '/*!\n' + + ' * reveal.js <%= pkg.version %> (<%= grunt.template.today("yyyy-mm-dd, HH:MM") %>)\n' + + ' * http://lab.hakim.se/reveal-js\n' + + ' * MIT licensed\n' + + ' *\n' + + ' * Copyright (C) 2016 Hakim El Hattab, http://hakim.se\n' + + ' */' + }, + + qunit: { + files: [ 'test/*.html' ] + }, + + uglify: { + options: { + banner: '<%= meta.banner %>\n' + }, + build: { + src: 'js/reveal.js', + dest: 'js/reveal.min.js' + } + }, + + sass: { + core: { + files: { + 'css/reveal.css': 'css/reveal.scss', + } + }, + themes: { + files: [ + { + expand: true, + cwd: 'css/theme/source', + src: ['*.scss'], + dest: 'css/theme', + ext: '.css' + } + ] + } + }, + + autoprefixer: { + dist: { + src: 'css/reveal.css' + } + }, + + cssmin: { + compress: { + files: { + 'css/reveal.min.css': [ 'css/reveal.css' ] + } + } + }, + + jshint: { + options: { + curly: false, + eqeqeq: true, + immed: true, + latedef: true, + newcap: true, + noarg: true, + sub: true, + undef: true, + eqnull: true, + browser: true, + expr: true, + globals: { + head: false, + module: false, + console: false, + unescape: false, + define: false, + exports: false + } + }, + files: [ 'Gruntfile.js', 'js/reveal.js' ] + }, + + connect: { + server: { + options: { + port: port, + base: base, + livereload: true, + open: true + } + } + }, + + zip: { + 'reveal-js-presentation.zip': [ + 'index.html', + 'css/**', + 'js/**', + 'lib/**', + 'images/**', + 'plugin/**', + '**.md' + ] + }, + + watch: { + js: { + files: [ 'Gruntfile.js', 'js/reveal.js' ], + tasks: 'js' + }, + theme: { + files: [ 'css/theme/source/*.scss', 'css/theme/template/*.scss' ], + tasks: 'css-themes' + }, + css: { + files: [ 'css/reveal.scss' ], + tasks: 'css-core' + }, + html: { + files: [ '*.html'] + }, + markdown: { + files: [ '*.md' ] + }, + options: { + livereload: true + } + } + + }); + + // Dependencies + grunt.loadNpmTasks( 'grunt-contrib-qunit' ); + grunt.loadNpmTasks( 'grunt-contrib-jshint' ); + grunt.loadNpmTasks( 'grunt-contrib-cssmin' ); + grunt.loadNpmTasks( 'grunt-contrib-uglify' ); + grunt.loadNpmTasks( 'grunt-contrib-watch' ); + grunt.loadNpmTasks( 'grunt-sass' ); + grunt.loadNpmTasks( 'grunt-contrib-connect' ); + grunt.loadNpmTasks( 'grunt-autoprefixer' ); + grunt.loadNpmTasks( 'grunt-zip' ); + + // Default task + grunt.registerTask( 'default', [ 'css', 'js' ] ); + + // JS task + grunt.registerTask( 'js', [ 'jshint', 'uglify', 'qunit' ] ); + + // Theme CSS + grunt.registerTask( 'css-themes', [ 'sass:themes' ] ); + + // Core framework CSS + grunt.registerTask( 'css-core', [ 'sass:core', 'autoprefixer', 'cssmin' ] ); + + // All CSS + grunt.registerTask( 'css', [ 'sass', 'autoprefixer', 'cssmin' ] ); + + // Package presentation to archive + grunt.registerTask( 'package', [ 'default', 'zip' ] ); + + // Serve presentation locally + grunt.registerTask( 'serve', [ 'connect', 'watch' ] ); + + // Run tests + grunt.registerTask( 'test', [ 'jshint', 'qunit' ] ); + +}; diff --git a/talk/compiler-workshop-2016/slides/LICENSE b/talk/compiler-workshop-2016/slides/LICENSE new file mode 100644 --- /dev/null +++ b/talk/compiler-workshop-2016/slides/LICENSE @@ -0,0 +1,19 @@ +Copyright (C) 2016 Hakim El Hattab, http://hakim.se + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. \ No newline at end of file diff --git a/talk/compiler-workshop-2016/slides/README.md b/talk/compiler-workshop-2016/slides/README.md new file mode 100644 --- /dev/null +++ b/talk/compiler-workshop-2016/slides/README.md @@ -0,0 +1,1104 @@ +# reveal.js [![Build Status](https://travis-ci.org/hakimel/reveal.js.svg?branch=master)](https://travis-ci.org/hakimel/reveal.js) + +A framework for easily creating beautiful presentations using HTML. [Check out the live demo](http://lab.hakim.se/reveal-js/). + +reveal.js comes with a broad range of features including [nested slides](https://github.com/hakimel/reveal.js#markup), [Markdown contents](https://github.com/hakimel/reveal.js#markdown), [PDF export](https://github.com/hakimel/reveal.js#pdf-export), [speaker notes](https://github.com/hakimel/reveal.js#speaker-notes) and a [JavaScript API](https://github.com/hakimel/reveal.js#api). There's also a fully featured visual editor and platform for sharing reveal.js presentations at [slides.com](https://slides.com). + +## Table of contents +- [Online Editor](#online-editor) +- [Instructions](#instructions) + - [Markup](#markup) + - [Markdown](#markdown) + - [Element Attributes](#element-attributes) + - [Slide Attributes](#slide-attributes) +- [Configuration](#configuration) +- [Presentation Size](#presentation-size) +- [Dependencies](#dependencies) +- [Ready Event](#ready-event) +- [Auto-sliding](#auto-sliding) +- [Keyboard Bindings](#keyboard-bindings) +- [Touch Navigation](#touch-navigation) +- [Lazy Loading](#lazy-loading) +- [API](#api) + - [Slide Changed Event](#slide-changed-event) + - [Presentation State](#presentation-state) + - [Slide States](#slide-states) + - [Slide Backgrounds](#slide-backgrounds) + - [Parallax Background](#parallax-background) + - [Slide Transitions](#slide-transitions) + - [Internal links](#internal-links) + - [Fragments](#fragments) + - [Fragment events](#fragment-events) + - [Code syntax highlighting](#code-syntax-highlighting) + - [Slide number](#slide-number) + - [Overview mode](#overview-mode) + - [Fullscreen mode](#fullscreen-mode) + - [Embedded media](#embedded-media) + - [Stretching elements](#stretching-elements) + - [postMessage API](#postmessage-api) +- [PDF Export](#pdf-export) +- [Theming](#theming) +- [Speaker Notes](#speaker-notes) + - [Share and Print Speaker Notes](#share-and-print-speaker-notes) + - [Server Side Speaker Notes](#server-side-speaker-notes) +- [Multiplexing](#multiplexing) + - [Master presentation](#master-presentation) + - [Client presentation](#client-presentation) + - [Socket.io server](#socketio-server) +- [MathJax](#mathjax) +- [Installation](#installation) + - [Basic setup](#basic-setup) + - [Full setup](#full-setup) + - [Folder Structure](#folder-structure) +- [License](#license) + +#### More reading +- [Changelog](https://github.com/hakimel/reveal.js/releases): Up-to-date version history. +- [Examples](https://github.com/hakimel/reveal.js/wiki/Example-Presentations): Presentations created with reveal.js, add your own! +- [Browser Support](https://github.com/hakimel/reveal.js/wiki/Browser-Support): Explanation of browser support and fallbacks. +- [Plugins](https://github.com/hakimel/reveal.js/wiki/Plugins,-Tools-and-Hardware): A list of plugins that can be used to extend reveal.js. + +## Online Editor + +Presentations are written using HTML or Markdown but there's also an online editor for those of you who prefer a graphical interface. Give it a try at [http://slides.com](http://slides.com?ref=github). + + +## Instructions + +### Markup + +Here's a barebones example of a fully working reveal.js presentation: +```html + + + + + + +
    +
    +
    Slide 1
    +
    Slide 2
    +
    +
    + + + + +``` + +The presentation markup hierarchy needs to be `.reveal > .slides > section` where the `section` represents one slide and can be repeated indefinitely. If you place multiple `section` elements inside of another `section` they will be shown as vertical slides. The first of the vertical slides is the "root" of the others (at the top), and will be included in the horizontal sequence. For example: + +```html +
    +
    +
    Single Horizontal Slide
    +
    +
    Vertical Slide 1
    +
    Vertical Slide 2
    +
    +
    +
    +``` + +### Markdown + +It's possible to write your slides using Markdown. To enable Markdown, add the ```data-markdown``` attribute to your ```
    ``` elements and wrap the contents in a ``` +
    +``` + +#### External Markdown + +You can write your content as a separate file and have reveal.js load it at runtime. Note the separator arguments which determine how slides are delimited in the external file. The ```data-charset``` attribute is optional and specifies which charset to use when loading the external file. + +When used locally, this feature requires that reveal.js [runs from a local web server](#full-setup). + +```html +
    +
    +``` + +#### Element Attributes + +Special syntax (in html comment) is available for adding attributes to Markdown elements. This is useful for fragments, amongst other things. + +```html +
    + +
    +``` + +#### Slide Attributes + +Special syntax (in html comment) is available for adding attributes to the slide `
    ` elements generated by your Markdown. + +```html +
    + +
    +``` + + +### Configuration + +At the end of your page you need to initialize reveal by running the following code. Note that all config values are optional and will default as specified below. + +```javascript +Reveal.initialize({ + + // Display controls in the bottom right corner + controls: true, + + // Display a presentation progress bar + progress: true, + + // Display the page number of the current slide + slideNumber: false, + + // Push each slide change to the browser history + history: false, + + // Enable keyboard shortcuts for navigation + keyboard: true, + + // Enable the slide overview mode + overview: true, + + // Vertical centering of slides + center: true, + + // Enables touch navigation on devices with touch input + touch: true, + + // Loop the presentation + loop: false, + + // Change the presentation direction to be RTL + rtl: false, + + // Randomizes the order of slides each time the presentation loads + shuffle: false, + + // Turns fragments on and off globally + fragments: true, + + // Flags if the presentation is running in an embedded mode, + // i.e. contained within a limited portion of the screen + embedded: false, + + // Flags if we should show a help overlay when the questionmark + // key is pressed + help: true, + + // Flags if speaker notes should be visible to all viewers + showNotes: false, + + // Number of milliseconds between automatically proceeding to the + // next slide, disabled when set to 0, this value can be overwritten + // by using a data-autoslide attribute on your slides + autoSlide: 0, + + // Stop auto-sliding after user input + autoSlideStoppable: true, + + // Use this method for navigation when auto-sliding + autoSlideMethod: Reveal.navigateNext, + + // Enable slide navigation via mouse wheel + mouseWheel: false, + + // Hides the address bar on mobile devices + hideAddressBar: true, + + // Opens links in an iframe preview overlay + previewLinks: false, + + // Transition style + transition: 'default', // none/fade/slide/convex/concave/zoom + + // Transition speed + transitionSpeed: 'default', // default/fast/slow + + // Transition style for full page slide backgrounds + backgroundTransition: 'default', // none/fade/slide/convex/concave/zoom + + // Number of slides away from the current that are visible + viewDistance: 3, + + // Parallax background image + parallaxBackgroundImage: '', // e.g. "'https://s3.amazonaws.com/hakim-static/reveal-js/reveal-parallax-1.jpg'" + + // Parallax background size + parallaxBackgroundSize: '', // CSS syntax, e.g. "2100px 900px" + + // Number of pixels to move the parallax background per slide + // - Calculated automatically unless specified + // - Set to 0 to disable movement along an axis + parallaxBackgroundHorizontal: null, + parallaxBackgroundVertical: null + +}); +``` + + +The configuration can be updated after initialization using the ```configure``` method: + +```javascript +// Turn autoSlide off +Reveal.configure({ autoSlide: 0 }); + +// Start auto-sliding every 5s +Reveal.configure({ autoSlide: 5000 }); +``` + + +### Presentation Size + +All presentations have a normal size, that is the resolution at which they are authored. The framework will automatically scale presentations uniformly based on this size to ensure that everything fits on any given display or viewport. + +See below for a list of configuration options related to sizing, including default values: + +```javascript +Reveal.initialize({ + + ... + + // The "normal" size of the presentation, aspect ratio will be preserved + // when the presentation is scaled to fit different resolutions. Can be + // specified using percentage units. + width: 960, + height: 700, + + // Factor of the display size that should remain empty around the content + margin: 0.1, + + // Bounds for smallest/largest possible scale to apply to content + minScale: 0.2, + maxScale: 1.5 + +}); +``` + + +### Dependencies + +Reveal.js doesn't _rely_ on any third party scripts to work but a few optional libraries are included by default. These libraries are loaded as dependencies in the order they appear, for example: + +```javascript +Reveal.initialize({ + dependencies: [ + // Cross-browser shim that fully implements classList - https://github.com/eligrey/classList.js/ + { src: 'lib/js/classList.js', condition: function() { return !document.body.classList; } }, + + // Interpret Markdown in
    elements + { src: 'plugin/markdown/marked.js', condition: function() { return !!document.querySelector( '[data-markdown]' ); } }, + { src: 'plugin/markdown/markdown.js', condition: function() { return !!document.querySelector( '[data-markdown]' ); } }, + + // Syntax highlight for elements + { src: 'plugin/highlight/highlight.js', async: true, callback: function() { hljs.initHighlightingOnLoad(); } }, + + // Zoom in and out with Alt+click + { src: 'plugin/zoom-js/zoom.js', async: true }, + + // Speaker notes + { src: 'plugin/notes/notes.js', async: true }, + + // MathJax + { src: 'plugin/math/math.js', async: true } + ] +}); +``` + +You can add your own extensions using the same syntax. The following properties are available for each dependency object: +- **src**: Path to the script to load +- **async**: [optional] Flags if the script should load after reveal.js has started, defaults to false +- **callback**: [optional] Function to execute when the script has loaded +- **condition**: [optional] Function which must return true for the script to be loaded + + +### Ready Event + +A 'ready' event is fired when reveal.js has loaded all non-async dependencies and is ready to start navigating. To check if reveal.js is already 'ready' you can call `Reveal.isReady()`. + +```javascript +Reveal.addEventListener( 'ready', function( event ) { + // event.currentSlide, event.indexh, event.indexv +} ); +``` + + +### Auto-sliding + +Presentations can be configured to progress through slides automatically, without any user input. To enable this you will need to tell the framework how many milliseconds it should wait between slides: + +```javascript +// Slide every five seconds +Reveal.configure({ + autoSlide: 5000 +}); +``` +When this is turned on a control element will appear that enables users to pause and resume auto-sliding. Alternatively, sliding can be paused or resumed by pressing »a« on the keyboard. Sliding is paused automatically as soon as the user starts navigating. You can disable these controls by specifying ```autoSlideStoppable: false``` in your reveal.js config. + +You can also override the slide duration for individual slides and fragments by using the ```data-autoslide``` attribute: + +```html +
    +

    After 2 seconds the first fragment will be shown.

    +

    After 10 seconds the next fragment will be shown.

    +

    Now, the fragment is displayed for 2 seconds before the next slide is shown.

    +
    +``` + +To override the method used for navigation when auto-sliding, you can specify the ```autoSlideMethod``` setting. To only navigate along the top layer and ignore vertical slides, set this to ```Reveal.navigateRight```. + +Whenever the auto-slide mode is resumed or paused the ```autoslideresumed``` and ```autoslidepaused``` events are fired. + + +### Keyboard Bindings + +If you're unhappy with any of the default keyboard bindings you can override them using the ```keyboard``` config option: + +```javascript +Reveal.configure({ + keyboard: { + 13: 'next', // go to the next slide when the ENTER key is pressed + 27: function() {}, // do something custom when ESC is pressed + 32: null // don't do anything when SPACE is pressed (i.e. disable a reveal.js default binding) + } +}); +``` + +### Touch Navigation + +You can swipe to navigate through a presentation on any touch-enabled device. Horizontal swipes change between horizontal slides, vertical swipes change between vertical slides. If you wish to disable this you can set the `touch` config option to false when initializing reveal.js. + +If there's some part of your content that needs to remain accessible to touch events you'll need to highlight this by adding a `data-prevent-swipe` attribute to the element. One common example where this is useful is elements that need to be scrolled. + + +### Lazy Loading + +When working on presentation with a lot of media or iframe content it's important to load lazily. Lazy loading means that reveal.js will only load content for the few slides nearest to the current slide. The number of slides that are preloaded is determined by the `viewDistance` configuration option. + +To enable lazy loading all you need to do is change your "src" attributes to "data-src" as shown below. This is supported for image, video, audio and iframe elements. Lazy loaded iframes will also unload when the containing slide is no longer visible. + +```html +
    + + + +
    +``` + + +### API + +The ``Reveal`` object exposes a JavaScript API for controlling navigation and reading state: + +```javascript +// Navigation +Reveal.slide( indexh, indexv, indexf ); +Reveal.left(); +Reveal.right(); +Reveal.up(); +Reveal.down(); +Reveal.prev(); +Reveal.next(); +Reveal.prevFragment(); +Reveal.nextFragment(); + +// Randomize the order of slides +Reveal.shuffle(); + +// Toggle presentation states, optionally pass true/false to force on/off +Reveal.toggleOverview(); +Reveal.togglePause(); +Reveal.toggleAutoSlide(); + +// Change a config value at runtime +Reveal.configure({ controls: true }); + +// Returns the present configuration options +Reveal.getConfig(); + +// Fetch the current scale of the presentation +Reveal.getScale(); + +// Retrieves the previous and current slide elements +Reveal.getPreviousSlide(); +Reveal.getCurrentSlide(); + +Reveal.getIndices(); // { h: 0, v: 0 } } +Reveal.getProgress(); // 0-1 +Reveal.getTotalSlides(); + +// Returns the speaker notes for the current slide +Reveal.getSlideNotes(); + +// State checks +Reveal.isFirstSlide(); +Reveal.isLastSlide(); +Reveal.isOverview(); +Reveal.isPaused(); +Reveal.isAutoSliding(); +``` + +### Slide Changed Event + +A 'slidechanged' event is fired each time the slide is changed (regardless of state). The event object holds the index values of the current slide as well as a reference to the previous and current slide HTML nodes. + +Some libraries, like MathJax (see [#226](https://github.com/hakimel/reveal.js/issues/226#issuecomment-10261609)), get confused by the transforms and display states of slides. Often times, this can be fixed by calling their update or render function from this callback. + +```javascript +Reveal.addEventListener( 'slidechanged', function( event ) { + // event.previousSlide, event.currentSlide, event.indexh, event.indexv +} ); +``` + +### Presentation State + +The presentation's current state can be fetched by using the `getState` method. A state object contains all of the information required to put the presentation back as it was when `getState` was first called. Sort of like a snapshot. It's a simple object that can easily be stringified and persisted or sent over the wire. + +```javascript +Reveal.slide( 1 ); +// we're on slide 1 + +var state = Reveal.getState(); + +Reveal.slide( 3 ); +// we're on slide 3 + +Reveal.setState( state ); +// we're back on slide 1 +``` + +### Slide States + +If you set ``data-state="somestate"`` on a slide ``
    ``, "somestate" will be applied as a class on the document element when that slide is opened. This allows you to apply broad style changes to the page based on the active slide. + +Furthermore you can also listen to these changes in state via JavaScript: + +```javascript +Reveal.addEventListener( 'somestate', function() { + // TODO: Sprinkle magic +}, false ); +``` + +### Slide Backgrounds + +Slides are contained within a limited portion of the screen by default to allow them to fit any display and scale uniformly. You can apply full page backgrounds outside of the slide area by adding a ```data-background``` attribute to your ```
    ``` elements. Four different types of backgrounds are supported: color, image, video and iframe. Below are a few examples. + +```html +
    +

    All CSS color formats are supported, like rgba() or hsl().

    +
    +
    +

    This slide will have a full-size background image.

    +
    +
    +

    This background image will be sized to 100px and repeated.

    +
    +
    +

    Video. Multiple sources can be defined using a comma separated list. Video will loop when the data-background-video-loop attribute is provided and can be muted with the data-background-video-muted attribute.

    +
    +
    +

    Embeds a web page as a background. Note that the page won't be interactive.

    +
    +``` + +Backgrounds transition using a fade animation by default. This can be changed to a linear sliding transition by passing ```backgroundTransition: 'slide'``` to the ```Reveal.initialize()``` call. Alternatively you can set ```data-background-transition``` on any section with a background to override that specific transition. + + +### Parallax Background + +If you want to use a parallax scrolling background, set the first two config properties below when initializing reveal.js (the other two are optional). + +```javascript +Reveal.initialize({ + + // Parallax background image + parallaxBackgroundImage: '', // e.g. "https://s3.amazonaws.com/hakim-static/reveal-js/reveal-parallax-1.jpg" + + // Parallax background size + parallaxBackgroundSize: '', // CSS syntax, e.g. "2100px 900px" - currently only pixels are supported (don't use % or auto) + + // Number of pixels to move the parallax background per slide + // - Calculated automatically unless specified + // - Set to 0 to disable movement along an axis + parallaxBackgroundHorizontal: 200, + parallaxBackgroundVertical: 50 + +}); +``` + +Make sure that the background size is much bigger than screen size to allow for some scrolling. [View example](http://lab.hakim.se/reveal-js/?parallaxBackgroundImage=https%3A%2F%2Fs3.amazonaws.com%2Fhakim-static%2Freveal-js%2Freveal-parallax-1.jpg¶llaxBackgroundSize=2100px%20900px). + + + +### Slide Transitions +The global presentation transition is set using the ```transition``` config value. You can override the global transition for a specific slide by using the ```data-transition``` attribute: + +```html +
    +

    This slide will override the presentation transition and zoom!

    +
    + +
    +

    Choose from three transition speeds: default, fast or slow!

    +
    +``` + +You can also use different in and out transitions for the same slide: + +```html +
    + The train goes on … +
    +
    + and on … +
    +
    + and stops. +
    +
    + (Passengers entering and leaving) +
    +
    + And it starts again. +
    +``` + + +### Internal links + +It's easy to link between slides. The first example below targets the index of another slide whereas the second targets a slide with an ID attribute (```
    ```): + +```html +Link +Link +``` + +You can also add relative navigation links, similar to the built in reveal.js controls, by appending one of the following classes on any element. Note that each element is automatically given an ```enabled``` class when it's a valid navigation route based on the current slide. + +```html + + + + + + +``` + + +### Fragments +Fragments are used to highlight individual elements on a slide. Every element with the class ```fragment``` will be stepped through before moving on to the next slide. Here's an example: http://lab.hakim.se/reveal-js/#/fragments + +The default fragment style is to start out invisible and fade in. This style can be changed by appending a different class to the fragment: + +```html +
    +

    grow

    +

    shrink

    +

    fade-out

    +

    fade-up (also down, left and right!)

    +

    visible only once

    +

    blue only once

    +

    highlight-red

    +

    highlight-green

    +

    highlight-blue

    +
    +``` + +Multiple fragments can be applied to the same element sequentially by wrapping it, this will fade in the text on the first step and fade it back out on the second. + +```html +
    + + I'll fade in, then out + +
    +``` + +The display order of fragments can be controlled using the ```data-fragment-index``` attribute. + +```html +
    +

    Appears last

    +

    Appears first

    +

    Appears second

    +
    +``` + +### Fragment events + +When a slide fragment is either shown or hidden reveal.js will dispatch an event. + +Some libraries, like MathJax (see #505), get confused by the initially hidden fragment elements. Often times this can be fixed by calling their update or render function from this callback. + +```javascript +Reveal.addEventListener( 'fragmentshown', function( event ) { + // event.fragment = the fragment DOM element +} ); +Reveal.addEventListener( 'fragmenthidden', function( event ) { + // event.fragment = the fragment DOM element +} ); +``` + +### Code syntax highlighting + +By default, Reveal is configured with [highlight.js](https://highlightjs.org/) for code syntax highlighting. Below is an example with clojure code that will be syntax highlighted. When the `data-trim` attribute is present, surrounding whitespace is automatically removed. HTML will be escaped by default. To avoid this, for example if you are using `` to call out a line of code, add the `data-noescape` attribute to the `` element. + +```html +
    + 
    
+(def lazy-fib
+  (concat
+   [0 1]
+   ((fn rfib [a b]
+        (lazy-cons (+ a b) (rfib b (+ a b)))) 0 1)))
+
    +
    +``` + +### Slide number +If you would like to display the page number of the current slide you can do so using the ```slideNumber``` configuration value. + +```javascript +// Shows the slide number using default formatting +Reveal.configure({ slideNumber: true }); + +// Slide number formatting can be configured using these variables: +// "h.v": horizontal . vertical slide number (default) +// "h/v": horizontal / vertical slide number +// "c": flattened slide number +// "c/t": flattened slide number / total slides +Reveal.configure({ slideNumber: 'c/t' }); + +``` + + +### Overview mode + +Press "Esc" or "o" keys to toggle the overview mode on and off. While you're in this mode, you can still navigate between slides, +as if you were at 1,000 feet above your presentation. The overview mode comes with a few API hooks: + +```javascript +Reveal.addEventListener( 'overviewshown', function( event ) { /* ... */ } ); +Reveal.addEventListener( 'overviewhidden', function( event ) { /* ... */ } ); + +// Toggle the overview mode programmatically +Reveal.toggleOverview(); +``` + +### Fullscreen mode +Just press »F« on your keyboard to show your presentation in fullscreen mode. Press the »ESC« key to exit fullscreen mode. + + +### Embedded media +Embedded HTML5 `
    @@ -46,23 +46,39 @@

    What We Bring to the Table

    - Our core is based on cutting-edge JIT compiler - research, written in Python. + A JIT compiler & GC for your language
    -

    Written in Python

    - Being written in Python means we can quickly - try out (and reject or incorporate) new ideas. - For instance, our STM work has - uncovered subtle bugs in CLANG and GCC, or the fact that we can easily try out - new register allocation strategies. +

    Written in Python

    + Quickly try out (and reject or incorporate) new ideas. +
    + +
    +

    PyPy STM

    +

    PyPy STM, Accept the GIL, but allow concurrent and correct execution

    +

    Uncovered subtle bugs in CLANG and GCC

    + Done by Armin Rigo & Remi Meier +
    +
    +

    VecOpt

    +

    Optimization to use SIMD instructions in a loop

    + Done by Richard Plangger +
    +
    +

    CPyExt Emulation

    +

    Support a rather complete set of CPyExt

    + Done by Matti Picus & Armin Rigo +
    +
    +

    VMProf, Reverse Python Debugger, CFFI, Sandboxing, several GCs

    +

    And many more in the pipeline ...

    -

    Took us some time too

    +

    Took us some time ...

    ... but runs on all major platforms
    Linux, Windows, MacOS, ...
    including four different CPU architectures
    @@ -75,12 +91,22 @@

    Flow graphs → Annotation → RTyping → Code generation

      -
    • + Whole program optimizations (take that, C)
      • +
    • + Whole program optimizations
    • + Deliver features quickly, without sacrificing speed
    • + Loosely coupled (JIT, Interp., RPython)
    • - Takes a while to translate
    + +
    +

    Tracing JIT compiler

    +

    Start off in loops and record one iteration

    +

    Build a tree of traces

    +
      +
    • + Simple and efficient type specialization
      • +
    • + Optimization pass easier
      • +
    • - Code duplication
      • +
    @@ -93,13 +119,6 @@
    -

    PyPy is extremly "under-funded"

    - We are not pushing data science on PyPy at all,
    - slowly moving forward on a volunteer basis. -

    → C-API compatibility

    -
    - -

    Does PyPy run out of the box?

    More help in identifying and resolving small issues
    @@ -107,11 +126,18 @@
    +

    PyPy is extremly "under-funded"

    + We are not pushing data science on PyPy at all,
    +

    → C-API compatibility

    + Slowly moving forward on a volunteer basis +
    + +

    NumPy

    - We are also working on Micro NumPy, which provides the kernels for numerical operations. - It is very usable, but still some features are missing. We would love to have a - partner/company that would help us to complete NumPy for PyPy. +

    Several limitations made us to redo part of the work

    +

    Usable, still features missing

    + Slowly moving forward on a volunteer basis
    @@ -124,14 +150,27 @@
    -

    Opinionated aspects

    +

    Some Aspects

    +

    with strong opinion

    +
    +
    +

    Let's only apply

    + static compiler techniques to speed up dynamic languages +
    +
    +

    You need static type information

    + to optimize your program +
    +
    +

    Method based JIT compilers

    + is the only concept that works *cough*

    Writing a program in statically compiled language

    does not mean the dev time / speedup ratio is appropriate
    - neither does it mean you program is fast + neither does it mean you program is "fast"

    Exposing details of your VM impl.

    @@ -141,20 +180,12 @@ and caused PyPy a lot of troubles
    -

    Dynamic typed languages

    +

    Dynamically typed languages

    are slow?
    and cannot be used for big programs?
    -
    -

    Method based JIT compilers

    - is the only concept that works *cough* -
    -
    -

    Let's apply

    - static compiler techniques to speed up dynamic langues -

    Questions?

    From pypy.commits at gmail.com Mon Jul 11 12:01:32 2016 From: pypy.commits at gmail.com (raffael_t) Date: Mon, 11 Jul 2016 09:01:32 -0700 (PDT) Subject: [pypy-commit] pypy py3.5-async: Throw SyntaxError if yield inside async function Message-ID: <5783c2dc.071fc20a.d3764.331c@mx.google.com> Author: Raffael Tfirst Branch: py3.5-async Changeset: r85657:8a1e4cca3a66 Date: 2016-07-11 18:00 +0200 http://bitbucket.org/pypy/pypy/changeset/8a1e4cca3a66/ Log: Throw SyntaxError if yield inside async function diff --git a/pypy/interpreter/astcompiler/symtable.py b/pypy/interpreter/astcompiler/symtable.py --- a/pypy/interpreter/astcompiler/symtable.py +++ b/pypy/interpreter/astcompiler/symtable.py @@ -282,6 +282,15 @@ if (self.has_free or self.child_has_free) and not self.optimized: raise AssertionError("unknown reason for unoptimization") +class AsyncFunctionScope(FunctionScope): + + def __init__(self, name, lineno, col_offset): + FunctionScope.__init__(self, name, lineno, col_offset) + + def note_yield(self, yield_node): + """Called when a yield is found.""" + raise SyntaxError("'yield' inside async function", yield_node.lineno, + yield_node.col_offset) class ClassScope(Scope): @@ -383,7 +392,7 @@ self.visit_kwonlydefaults(args.kw_defaults) self._visit_annotations(func) self.visit_sequence(func.decorator_list) - new_scope = FunctionScope(func.name, func.lineno, func.col_offset) + new_scope = AsyncFunctionScope(func.name, func.lineno, func.col_offset) self.push_scope(new_scope, func) func.args.walkabout(self) self.visit_sequence(func.body) From pypy.commits at gmail.com Mon Jul 11 12:18:29 2016 From: pypy.commits at gmail.com (raffael_t) Date: Mon, 11 Jul 2016 09:18:29 -0700 (PDT) Subject: [pypy-commit] pypy py3.5-async: Throw SyntaxError if 'yield from' inside async function (fixes test_invalid_7 in test_syntax) Message-ID: <5783c6d5.e9f2c20a.374cd.3b37@mx.google.com> Author: Raffael Tfirst Branch: py3.5-async Changeset: r85658:e779f313fab5 Date: 2016-07-11 18:17 +0200 http://bitbucket.org/pypy/pypy/changeset/e779f313fab5/ Log: Throw SyntaxError if 'yield from' inside async function (fixes test_invalid_7 in test_syntax) diff --git a/pypy/interpreter/astcompiler/symtable.py b/pypy/interpreter/astcompiler/symtable.py --- a/pypy/interpreter/astcompiler/symtable.py +++ b/pypy/interpreter/astcompiler/symtable.py @@ -88,6 +88,11 @@ """Called when a yield is found.""" raise SyntaxError("'yield' outside function", yield_node.lineno, yield_node.col_offset) + + def note_yieldFrom(self, yield_node): + """Called when a yield from is found.""" + raise SyntaxError("'yield' outside function", yield_node.lineno, + yield_node.col_offset) def note_return(self, ret): """Called when a return statement is found.""" @@ -249,6 +254,11 @@ self.is_generator = True if self._in_try_body_depth > 0: self.has_yield_inside_try = True + + def note_yieldFrom(self, yield_node): + self.is_generator = True + if self._in_try_body_depth > 0: + self.has_yield_inside_try = True def note_return(self, ret): if ret.value: @@ -288,10 +298,13 @@ FunctionScope.__init__(self, name, lineno, col_offset) def note_yield(self, yield_node): - """Called when a yield is found.""" raise SyntaxError("'yield' inside async function", yield_node.lineno, yield_node.col_offset) + def note_yieldFrom(self, yield_node): + raise SyntaxError("'yield from' inside async function", yield_node.lineno, + yield_node.col_offset) + class ClassScope(Scope): _hide_bound_from_nested_scopes = True @@ -448,7 +461,7 @@ ast.GenericASTVisitor.visit_Yield(self, yie) def visit_YieldFrom(self, yfr): - self.scope.note_yield(yfr) + self.scope.note_yieldFrom(yfr) ast.GenericASTVisitor.visit_YieldFrom(self, yfr) def visit_Global(self, glob): From pypy.commits at gmail.com Mon Jul 11 13:33:51 2016 From: pypy.commits at gmail.com (raffael_t) Date: Mon, 11 Jul 2016 10:33:51 -0700 (PDT) Subject: [pypy-commit] pypy py3.5-async: Throw SyntaxError if 'await' outside [async] function (fixes test_invalid_2, 4, 5, 8 in test_syntax) Message-ID: <5783d87f.e9f2c20a.374cd.57c9@mx.google.com> Author: Raffael Tfirst Branch: py3.5-async Changeset: r85659:3700b8062272 Date: 2016-07-11 19:33 +0200 http://bitbucket.org/pypy/pypy/changeset/3700b8062272/ Log: Throw SyntaxError if 'await' outside [async] function (fixes test_invalid_2,4,5,8 in test_syntax) diff --git a/pypy/interpreter/astcompiler/symtable.py b/pypy/interpreter/astcompiler/symtable.py --- a/pypy/interpreter/astcompiler/symtable.py +++ b/pypy/interpreter/astcompiler/symtable.py @@ -89,10 +89,15 @@ raise SyntaxError("'yield' outside function", yield_node.lineno, yield_node.col_offset) - def note_yieldFrom(self, yield_node): + def note_yieldFrom(self, yieldFrom_node): """Called when a yield from is found.""" - raise SyntaxError("'yield' outside function", yield_node.lineno, - yield_node.col_offset) + raise SyntaxError("'yield' outside function", yieldFrom_node.lineno, + yieldFrom_node.col_offset) + + def note_await(self, await_node): + """Called when await is found.""" + raise SyntaxError("'await' outside function", await_node.lineno, + await_node.col_offset) def note_return(self, ret): """Called when a return statement is found.""" @@ -229,6 +234,10 @@ def __init__(self): Scope.__init__(self, "top") + def note_await(self, await_node): + raise SyntaxError("'await' outside async function", await_node.lineno, + await_node.col_offset) + class FunctionScope(Scope): @@ -259,6 +268,10 @@ self.is_generator = True if self._in_try_body_depth > 0: self.has_yield_inside_try = True + + def note_await(self, await_node): + raise SyntaxError("'await' outside async function", await_node.lineno, + await_node.col_offset) def note_return(self, ret): if ret.value: @@ -304,6 +317,10 @@ def note_yieldFrom(self, yield_node): raise SyntaxError("'yield from' inside async function", yield_node.lineno, yield_node.col_offset) + + def note_await(self, await_node): + pass + class ClassScope(Scope): @@ -410,6 +427,10 @@ func.args.walkabout(self) self.visit_sequence(func.body) self.pop_scope() + + def visit_Await(self, aw): + self.scope.note_await(aw) + ast.GenericASTVisitor.visit_Await(self, aw) def visit_Return(self, ret): self.scope.note_return(ret) From pypy.commits at gmail.com Mon Jul 11 14:08:11 2016 From: pypy.commits at gmail.com (wlav) Date: Mon, 11 Jul 2016 11:08:11 -0700 (PDT) Subject: [pypy-commit] pypy cling-support: from Aditi: implementation of lazy lookup of global/namespaced methods Message-ID: <5783e08b.c8a61c0a.ad1e7.4a68@mx.google.com> Author: Wim Lavrijsen Branch: cling-support Changeset: r85660:c3d56777b481 Date: 2016-07-11 11:05 -0700 http://bitbucket.org/pypy/pypy/changeset/c3d56777b481/ Log: from Aditi: implementation of lazy lookup of global/namespaced methods diff --git a/pypy/module/cppyy/src/clingcwrapper.cxx b/pypy/module/cppyy/src/clingcwrapper.cxx --- a/pypy/module/cppyy/src/clingcwrapper.cxx +++ b/pypy/module/cppyy/src/clingcwrapper.cxx @@ -1240,9 +1240,39 @@ } cppyy_index_t* cppyy_method_indices_from_name(cppyy_scope_t scope, const char* name){ -//NEED TO DO: - return (cppyy_index_t*)0; -// return (cppyy_index_t*)Cppyy::GetMethodsFromName(scope, name); + std::vector result; + TClassRef& cr = type_from_handle(scope); + if (cr.GetClass()) { + gInterpreter->UpdateListOfMethods(cr.GetClass()); + int imeth = 0; + TFunction* func; + TIter next(cr->GetListOfMethods()); + while ((func = (TFunction*)next())) { + if (strcmp(name, func->GetName()) == 0) { + if (func->Property() & Cppyy::IsPublicMethod((cppyy_method_t)func)) + result.push_back((cppyy_index_t)imeth); + } + ++imeth; + } + } else if (scope == (cppyy_scope_t)GLOBAL_HANDLE) { + TCollection* funcs = gROOT->GetListOfGlobalFunctions(kTRUE); + TFunction* func = 0; + TIter ifunc(funcs); + while ((func = (TFunction*)ifunc.Next())) { + if (strcmp(name, func->GetName()) == 0) { + g_globalfuncs.push_back(*func); + result.push_back((cppyy_index_t)func); + } + } + } + + if (result.empty()) + return (cppyy_index_t*)0; + + cppyy_index_t* llresult = (cppyy_index_t*)malloc(sizeof(cppyy_index_t)*(result.size()+1)); + for (int i = 0; i < (int)result.size(); ++i) llresult[i] = result[i]; + llresult[result.size()] = -1; + return llresult; } char* cppyy_method_name(cppyy_scope_t scope, cppyy_index_t idx) { From pypy.commits at gmail.com Mon Jul 11 15:03:26 2016 From: pypy.commits at gmail.com (arigo) Date: Mon, 11 Jul 2016 12:03:26 -0700 (PDT) Subject: [pypy-commit] pypy use-mmap: Trying a slightly different approach with "mreset" instead of munmap. Message-ID: <5783ed7e.c213c20a.fe443.740c@mx.google.com> Author: Armin Rigo Branch: use-mmap Changeset: r85661:9247cf241910 Date: 2016-07-11 20:47 +0200 http://bitbucket.org/pypy/pypy/changeset/9247cf241910/ Log: Trying a slightly different approach with "mreset" instead of munmap. This should avoid all fragmentation of the 256-KB mmaped regions by not returning them to the OS, but returning only the memory within. diff --git a/rpython/memory/gc/incminimark.py b/rpython/memory/gc/incminimark.py --- a/rpython/memory/gc/incminimark.py +++ b/rpython/memory/gc/incminimark.py @@ -739,9 +739,9 @@ self.major_collection_step() else: self.minor_and_major_collection() - if gen >= 3: - self.ac.kill_dying_arenas() self.rrc_invoke_callback() + if gen >= 3: + self.ac.mreset_dead_arenas() def minor_collection_with_major_progress(self, extrasize=0): diff --git a/rpython/memory/gc/minimark.py b/rpython/memory/gc/minimark.py --- a/rpython/memory/gc/minimark.py +++ b/rpython/memory/gc/minimark.py @@ -634,11 +634,13 @@ return llmemory.cast_adr_to_ptr(obj, llmemory.GCREF) - def collect(self, gen=1): + def collect(self, gen=2): """Do a minor (gen=0) or major (gen>0) collection.""" self.minor_collection() if gen > 0: self.major_collection() + if gen > 1: + self.ac.mreset_dead_arenas() def move_nursery_top(self, totalsize): size = self.nursery_cleanup diff --git a/rpython/memory/gc/minimarkpage.py b/rpython/memory/gc/minimarkpage.py --- a/rpython/memory/gc/minimarkpage.py +++ b/rpython/memory/gc/minimarkpage.py @@ -124,20 +124,31 @@ flavor='raw', zero=True, immortal=True) # - # two lists of completely free arenas. These arenas are - # pending to be returned to the OS. They are first added to - # the 'dying_arenas' list when the major collection runs. At - # the end, they are moved to the 'dead_arenas' list, and all + # three lists of completely free arenas, classified in terms of + # their age, measured as the number of "end major collection" + # since they are completely free: + # + # * 'dying_arenas': age = 0 + # * 'dead_arenas': age = 1 + # * 'mreset_arenas': age >= 2 + # + # The 'mreset_arenas' have been returned to the OS with + # llarena.arena_mreset(). + # + # The idea is that free arenas are first added to the + # 'dying_arenas' list when the major collection runs. At the + # end, they are moved to the 'dead_arenas' list, and all # arenas that were already in the 'dead_arenas' list at that # point (from the previous major collection) are really - # returned to the OS. - # Memory usage goes down quickly during the incremental - # major collection and up slowly the rest of the time. The - # point of these two lists is to avoid constantly freeing and + # returned to the OS, and moved to the 'mreset_arenas' list. + # Memory usage goes down quickly during the incremental major + # collection and up slowly the rest of the time. The point of + # these three lists is to avoid constantly freeing and # re-allocating arenas: we return to the OS the arenas that # have been unused for a complete cycle already. self.dying_arenas = ARENA_NULL self.dead_arenas = ARENA_NULL + self.mreset_arenas = ARENA_NULL # # the arena currently consumed; it must have at least one page # available, or be NULL. The arena object that we point to is @@ -305,13 +316,16 @@ for a in self._all_arenas(): assert a.nfreepages == 0 # - # Maybe we have a dying or a dead arena. + # Maybe we have a dying or a dead or a mreset arena. if self.dying_arenas: arena = self.dying_arenas self.dying_arenas = arena.nextarena elif self.dead_arenas: arena = self.dead_arenas self.dead_arenas = arena.nextarena + elif self.mreset_arenas: + arena = self.mreset_arenas + self.mreset_arenas = arena.nextarena else: # # 'arena_base' points to the start of malloced memory. It might @@ -343,16 +357,20 @@ allocate_new_arena._dont_inline_ = True - def kill_dying_arenas(self): - """Return to the OS all dead arenas, and then move the 'dying' - arenas to the 'dead' arenas list. + def mreset_dead_arenas(self): + """Return to the OS all 'dead' arenas and move them to the + 'mreset' arenas list, and then move the 'dying' arenas to the + 'dead' arenas list. """ arena = self.dead_arenas + mreset_head = self.mreset_arenas while arena: nextarena = arena.nextarena - llarena.arena_munmap(arena.base, self.arena_size) - lltype.free(arena, flavor='raw', track_allocation=False) + llarena.arena_mreset(arena.base, self.arena_size) + arena.nextarena = mreset_head + mreset_head = arena arena = nextarena + self.mreset_arenas = mreset_head self.dead_arenas = self.dying_arenas self.dying_arenas = ARENA_NULL @@ -402,7 +420,7 @@ if size_class >= 0: self._rehash_arenas_lists() self.size_class_with_old_pages = -1 - self.kill_dying_arenas() + self.mreset_dead_arenas() # return True diff --git a/rpython/memory/gc/minimarktest.py b/rpython/memory/gc/minimarktest.py --- a/rpython/memory/gc/minimarktest.py +++ b/rpython/memory/gc/minimarktest.py @@ -57,5 +57,5 @@ res = self.mass_free_incremental(ok_to_free_func, sys.maxint) assert res - def kill_dying_arenas(self): + def mreset_dead_arenas(self): pass diff --git a/rpython/memory/gc/test/test_minimarkpage.py b/rpython/memory/gc/test/test_minimarkpage.py --- a/rpython/memory/gc/test/test_minimarkpage.py +++ b/rpython/memory/gc/test/test_minimarkpage.py @@ -421,12 +421,13 @@ print '-' * 80 ac.__class__.allocate_new_arena(ac) a = ac.current_arena.base.arena - def my_mark_freed(): - a.freed = True + def my_mreset(): + a.__class__.mreset(a) DoneTesting.counter += 1 if DoneTesting.counter > 3: raise DoneTesting - a.mark_freed = my_mark_freed + a.mark_freed = "not callable" + a.mreset = my_mreset ac.allocate_new_arena = my_allocate_new_arena def allocate_object(live_objects): diff --git a/rpython/rlib/rmmap.py b/rpython/rlib/rmmap.py --- a/rpython/rlib/rmmap.py +++ b/rpython/rlib/rmmap.py @@ -76,7 +76,8 @@ constant_names = ['PAGE_READONLY', 'PAGE_READWRITE', 'PAGE_WRITECOPY', 'FILE_MAP_READ', 'FILE_MAP_WRITE', 'FILE_MAP_COPY', 'DUPLICATE_SAME_ACCESS', 'MEM_COMMIT', 'MEM_RESERVE', - 'MEM_RELEASE', 'PAGE_EXECUTE_READWRITE', 'PAGE_NOACCESS'] + 'MEM_RELEASE', 'PAGE_EXECUTE_READWRITE', 'PAGE_NOACCESS', + 'MEM_RESET'] for name in constant_names: setattr(CConfig, name, rffi_platform.ConstantInteger(name)) @@ -703,9 +704,17 @@ res = rffi.cast(PTR, 0) return res - def free_memory_chunk(addr, map_size): + def reset_memory_chunk(addr, map_size): # used by the memory allocator (in llarena.py, from minimarkpage.py) - c_munmap_safe(addr, rffi.cast(size_t, map_size)) + # to release the memory currently held in a memory chunk, possibly + # zeroing it, but keep that memory chunk available for future use. + # should only be used on allocate_memory_chunk() memory. + flags = MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED + prot = PROT_READ | PROT_WRITE + if we_are_translated(): + flags = NonConstant(flags) + prot = NonConstant(prot) + c_mmap_safe(rffi.cast(PTR, addr), map_size, prot, flags, -1, 0) def alloc_hinted(hintp, map_size): flags = MAP_PRIVATE | MAP_ANONYMOUS @@ -912,10 +921,12 @@ case of a sandboxed process """ null = lltype.nullptr(rffi.VOIDP.TO) + flags = MEM_COMMIT | MEM_RESERVE prot = PAGE_EXECUTE_READWRITE if we_are_translated(): + flags = NonConstant(flags) prot = NonConstant(prot) - res = VirtualAlloc_safe(null, map_size, MEM_COMMIT | MEM_RESERVE, prot) + res = VirtualAlloc_safe(null, map_size, flags, prot) if not res: raise MemoryError arg = lltype.malloc(LPDWORD.TO, 1, zero=True, flavor='raw') @@ -931,12 +942,19 @@ def allocate_memory_chunk(map_size): # used by the memory allocator (in llarena.py, from minimarkpage.py) null = lltype.nullptr(rffi.VOIDP.TO) + flags = MEM_COMMIT | MEM_RESERVE prot = PAGE_READWRITE if we_are_translated(): + flags = NonConstant(flags) prot = NonConstant(prot) - res = VirtualAlloc_safe(null, map_size, MEM_COMMIT | MEM_RESERVE, prot) + res = VirtualAlloc_safe(null, map_size, flags, prot) return res - def free_memory_chunk(addr, map_size): + def reset_memory_chunk(addr, map_size): # used by the memory allocator (in llarena.py, from minimarkpage.py) - VirtualFree_safe(addr, 0, MEM_RELEASE) + flags = MEM_RESET + prot = PAGE_READWRITE + if we_are_translated(): + flags = NonConstant(flags) + prot = NonConstant(prot) + VirtualAlloc_safe(rffi.cast(rffi.VOIDP, addr), map_size, flags, prot) diff --git a/rpython/rtyper/lltypesystem/llarena.py b/rpython/rtyper/lltypesystem/llarena.py --- a/rpython/rtyper/lltypesystem/llarena.py +++ b/rpython/rtyper/lltypesystem/llarena.py @@ -133,6 +133,11 @@ def mark_freed(self): self.freed = True # this method is a hook for tests + def mreset(self): # this method is a hook for tests + assert self.mmaped + self.reset(False) + assert not self.objectptrs + def set_protect(self, inaccessible): if inaccessible: assert self.protect_inaccessible is None @@ -318,8 +323,9 @@ # arena_new_view(ptr) is a no-op when translated, returns fresh view # on previous arena when run on top of llinterp -# arena_mmap() and arena_munmap() are similar to arena_malloc(zero=True) -# and arena_free(), but use the OS's mmap() function. +# arena_mmap() is similar to arena_malloc(zero=True), but uses the +# OS's mmap() function. We can then use arena_mreset() on the result +# to reset memory content (i.e. mark as free, possibly reset to zero) def arena_malloc(nbytes, zero): """Allocate and return a new arena, optionally zero-initialized.""" @@ -327,12 +333,9 @@ def arena_free(arena_addr): """Release an arena.""" - _arena_free(arena_addr, mmaped=False) - -def _arena_free(arena_addr, mmaped): assert isinstance(arena_addr, fakearenaaddress) assert arena_addr.offset == 0 - assert arena_addr.arena.mmaped == mmaped + assert not arena_addr.arena.mmaped arena_addr.arena.reset(False) assert not arena_addr.arena.objectptrs arena_addr.arena.mark_freed() @@ -341,9 +344,15 @@ """Allocate a new arena using mmap().""" return Arena(nbytes, zero=True, mmaped=True).getaddr(0) -def arena_munmap(arena_addr, original_nbytes): - """Release an arena allocated with arena_mmap().""" - _arena_free(arena_addr, mmaped=True) +def arena_mreset(arena_addr, original_nbytes): + """Reset a complete arena allocated with arena_mmap(). + Equivalent to arena_reset(..., zero=0), but really tries + to return the memory to the OS until the next access. + """ + assert isinstance(arena_addr, fakearenaaddress) + assert arena_addr.offset == 0 + assert arena_addr.arena.nbytes == original_nbytes + arena_addr.arena.mreset() def arena_reset(arena_addr, size, zero): """Free all objects in the arena, which can then be reused. @@ -526,9 +535,9 @@ from rpython.rlib import rmmap return rffi.cast(llmemory.Address, rmmap.allocate_memory_chunk(nbytes)) -def llimpl_arena_munmap(addr, nbytes): +def llimpl_arena_mreset(addr, nbytes): from rpython.rlib import rmmap - rmmap.free_memory_chunk(rffi.cast(rmmap.PTR, addr), nbytes) + rmmap.reset_memory_chunk(rffi.cast(rmmap.PTR, addr), nbytes) register_external(arena_mmap, [int], llmemory.Address, 'll_arena.arena_mmap', @@ -536,10 +545,10 @@ llfakeimpl=arena_mmap, sandboxsafe=True) -register_external(arena_munmap, [llmemory.Address, int], None, - 'll_arena.arena_munmap', - llimpl=llimpl_arena_munmap, - llfakeimpl=arena_munmap, +register_external(arena_mreset, [llmemory.Address, int], None, + 'll_arena.arena_mreset', + llimpl=llimpl_arena_mreset, + llfakeimpl=arena_mreset, sandboxsafe=True) def llimpl_arena_reset(arena_addr, size, zero): From pypy.commits at gmail.com Mon Jul 11 16:22:06 2016 From: pypy.commits at gmail.com (arigo) Date: Mon, 11 Jul 2016 13:22:06 -0700 (PDT) Subject: [pypy-commit] pypy default: Avoid crashing obscurely if NonConstant() is called on something Message-ID: <5783ffee.0a2cc20a.18b92.ffff8f24@mx.google.com> Author: Armin Rigo Branch: Changeset: r85662:e8c967178a5d Date: 2016-07-11 22:23 +0200 http://bitbucket.org/pypy/pypy/changeset/e8c967178a5d/ Log: Avoid crashing obscurely if NonConstant() is called on something that is already not a constant diff --git a/rpython/rlib/nonconst.py b/rpython/rlib/nonconst.py --- a/rpython/rlib/nonconst.py +++ b/rpython/rlib/nonconst.py @@ -39,7 +39,4 @@ def specialize_call(self, hop): hop.exception_cannot_occur() - retval = Constant(hop.r_result.convert_const(hop.args_v[0].value)) - retval.concretetype = hop.r_result.lowleveltype - return retval - + return hop.inputarg(hop.r_result, arg=0) diff --git a/rpython/rlib/test/test_nonconst.py b/rpython/rlib/test/test_nonconst.py --- a/rpython/rlib/test/test_nonconst.py +++ b/rpython/rlib/test/test_nonconst.py @@ -7,6 +7,7 @@ from rpython.annotator.annrpython import RPythonAnnotator from rpython.conftest import option from rpython.annotator.model import SomeInstance +from rpython.rtyper.test.test_llinterp import interpret def test_nonconst(): def nonconst_f(): @@ -18,7 +19,6 @@ assert s.knowntype is int assert not hasattr(s, 'const') - def test_nonconst_list(): def nonconst_l(): a = NonConstant([1, 2, 3]) @@ -56,3 +56,8 @@ if option.view: a.translator.view() + +def test_already_not_const(): + def fn(x): + return NonConstant(x) + assert interpret(fn, [5]) == 5 From pypy.commits at gmail.com Mon Jul 11 16:28:30 2016 From: pypy.commits at gmail.com (arigo) Date: Mon, 11 Jul 2016 13:28:30 -0700 (PDT) Subject: [pypy-commit] pypy use-mmap: Windows translation fix Message-ID: <5784016e.e7acc20a.57068.ffff8f28@mx.google.com> Author: Armin Rigo Branch: use-mmap Changeset: r85663:2c06183fc692 Date: 2016-07-11 22:29 +0200 http://bitbucket.org/pypy/pypy/changeset/2c06183fc692/ Log: Windows translation fix diff --git a/rpython/rlib/nonconst.py b/rpython/rlib/nonconst.py --- a/rpython/rlib/nonconst.py +++ b/rpython/rlib/nonconst.py @@ -39,7 +39,4 @@ def specialize_call(self, hop): hop.exception_cannot_occur() - retval = Constant(hop.r_result.convert_const(hop.args_v[0].value)) - retval.concretetype = hop.r_result.lowleveltype - return retval - + return hop.inputarg(hop.r_result, arg=0) diff --git a/rpython/rlib/rmmap.py b/rpython/rlib/rmmap.py --- a/rpython/rlib/rmmap.py +++ b/rpython/rlib/rmmap.py @@ -924,6 +924,7 @@ flags = MEM_COMMIT | MEM_RESERVE prot = PAGE_EXECUTE_READWRITE if we_are_translated(): + null = NonConstant(null) flags = NonConstant(flags) prot = NonConstant(prot) res = VirtualAlloc_safe(null, map_size, flags, prot) @@ -945,6 +946,7 @@ flags = MEM_COMMIT | MEM_RESERVE prot = PAGE_READWRITE if we_are_translated(): + null = NonConstant(null) flags = NonConstant(flags) prot = NonConstant(prot) res = VirtualAlloc_safe(null, map_size, flags, prot) From pypy.commits at gmail.com Mon Jul 11 18:03:58 2016 From: pypy.commits at gmail.com (arigo) Date: Mon, 11 Jul 2016 15:03:58 -0700 (PDT) Subject: [pypy-commit] pypy use-mmap: arena_mmap() returns page-aligned memory, so we can get rid of this Message-ID: <578417ce.0c091c0a.e9298.ffffd663@mx.google.com> Author: Armin Rigo Branch: use-mmap Changeset: r85664:96909c906c52 Date: 2016-07-12 00:05 +0200 http://bitbucket.org/pypy/pypy/changeset/96909c906c52/ Log: arena_mmap() returns page-aligned memory, so we can get rid of this page-realignment indirection diff --git a/rpython/memory/gc/minimarkpage.py b/rpython/memory/gc/minimarkpage.py --- a/rpython/memory/gc/minimarkpage.py +++ b/rpython/memory/gc/minimarkpage.py @@ -18,11 +18,10 @@ ARENA_PTR = lltype.Ptr(lltype.ForwardReference()) ARENA = lltype.Struct('ArenaReference', - # -- The address of the arena, as returned by malloc() + # -- The address of the arena, as returned by arena_mmap() ('base', llmemory.Address), - # -- The number of free and the total number of pages in the arena + # -- The number of free pages in the arena ('nfreepages', lltype.Signed), - ('totalpages', lltype.Signed), # -- A chained list of free pages in the arena. Ends with NULL. ('freepages', llmemory.Address), # -- A linked list of arenas. See below. @@ -113,14 +112,14 @@ for i in range(1, length): self.nblocks_for_size[i] = (page_size - self.hdrsize) // (WORD * i) # - self.max_pages_per_arena = arena_size // page_size + self.pages_per_arena = arena_size // page_size self.arenas_lists = lltype.malloc(rffi.CArray(ARENA_PTR), - self.max_pages_per_arena, + self.pages_per_arena, flavor='raw', zero=True, immortal=True) # this is used in mass_free() only self.old_arenas_lists = lltype.malloc(rffi.CArray(ARENA_PTR), - self.max_pages_per_arena, + self.pages_per_arena, flavor='raw', zero=True, immortal=True) # @@ -158,7 +157,7 @@ self.current_arena = ARENA_NULL # # guarantee that 'arenas_lists[1:min_empty_nfreepages]' are all empty - self.min_empty_nfreepages = self.max_pages_per_arena + self.min_empty_nfreepages = self.pages_per_arena # # part of current_arena might still contain uninitialized pages self.num_uninitialized_pages = 0 @@ -284,7 +283,7 @@ # but > 0. Use caching with 'min_empty_nfreepages', which guarantees # that 'arenas_lists[1:min_empty_nfreepages]' are all empty. i = self.min_empty_nfreepages - while i < self.max_pages_per_arena: + while i < self.pages_per_arena: # if self.arenas_lists[i] != ARENA_NULL: # @@ -328,30 +327,21 @@ self.mreset_arenas = arena.nextarena else: # - # 'arena_base' points to the start of malloced memory. It might - # not be a page-aligned address depending on the underlying call - # (although with mmap() it should be). + # 'arena_base' points to the start of malloced memory. It is + # aligned to a physical page (although self.page_size is + # typically two physical pages on 64-bit). arena_base = llarena.arena_mmap(self.arena_size) if not arena_base: out_of_memory("out of memory: couldn't allocate the next arena") - arena_end = arena_base + self.arena_size - # - # 'firstpage' points to the first unused page - firstpage = start_of_page(arena_base + self.page_size - 1, - self.page_size) - # 'npages' is the number of full pages just allocated - npages = (arena_end - firstpage) // self.page_size # # Allocate an ARENA object and initialize it arena = lltype.malloc(ARENA, flavor='raw', track_allocation=False) arena.base = arena_base - arena.totalpages = npages # arena.nfreepages = 0 # they are all uninitialized pages - arena.freepages = start_of_page(arena.base + self.page_size - 1, - self.page_size) + arena.freepages = arena.base arena.nextarena = ARENA_NULL - self.num_uninitialized_pages = arena.totalpages + self.num_uninitialized_pages = self.pages_per_arena self.current_arena = arena # allocate_new_arena._dont_inline_ = True @@ -443,17 +433,17 @@ self.arenas_lists, self.old_arenas_lists) # i = 0 - while i < self.max_pages_per_arena: + while i < self.pages_per_arena: self.arenas_lists[i] = ARENA_NULL i += 1 # i = 0 - while i < self.max_pages_per_arena: + while i < self.pages_per_arena: arena = self.old_arenas_lists[i] while arena != ARENA_NULL: nextarena = arena.nextarena # - if arena.nfreepages == arena.totalpages: + if arena.nfreepages == self.pages_per_arena: # # The whole arena is empty. Move it to the dying list. arena.nextarena = self.dying_arenas @@ -465,8 +455,8 @@ else: # Insert 'arena' in the correct arenas_lists[n] n = arena.nfreepages - ll_assert(n < self.max_pages_per_arena, - "totalpages != nfreepages >= max_pages_per_arena") + ll_assert(n < self.pages_per_arena, + "nfreepages > pages_per_arena") arena.nextarena = self.arenas_lists[n] self.arenas_lists[n] = arena # @@ -543,7 +533,7 @@ """Free a whole page.""" # # Insert the freed page in the arena's 'freepages' list. - # If nfreepages == totalpages, then it will be freed at the + # If nfreepages == pages_per_arena, then it will be freed at the # end of mass_free(). arena = page.arena arena.nfreepages += 1 @@ -636,21 +626,6 @@ # ____________________________________________________________ # Helpers to go from a pointer to the start of its page -def start_of_page(addr, page_size): - """Return the address of the start of the page that contains 'addr'.""" - if we_are_translated(): - offset = llmemory.cast_adr_to_int(addr) % page_size - return addr - offset - else: - return _start_of_page_untranslated(addr, page_size) - -def _start_of_page_untranslated(addr, page_size): - assert isinstance(addr, llarena.fakearenaaddress) - shift = WORD # for testing, we assume that the whole arena is not - # on a page boundary - ofs = ((addr.offset - shift) // page_size) * page_size + shift - return llarena.fakearenaaddress(addr.arena, ofs) - def _dummy_size(size): if we_are_translated(): return size diff --git a/rpython/memory/gc/test/test_minimarkpage.py b/rpython/memory/gc/test/test_minimarkpage.py --- a/rpython/memory/gc/test/test_minimarkpage.py +++ b/rpython/memory/gc/test/test_minimarkpage.py @@ -7,19 +7,18 @@ from rpython.rtyper.lltypesystem.llmemory import cast_ptr_to_adr NULL = llmemory.NULL -SHIFT = WORD hdrsize = llmemory.raw_malloc_usage(llmemory.sizeof(PAGE_HEADER)) def test_allocate_arena(): - ac = ArenaCollection(SHIFT + 64*20, 64, 1) + ac = ArenaCollection(64*20, 64, 1) ac.allocate_new_arena() assert ac.num_uninitialized_pages == 20 upages = ac.current_arena.freepages upages + 64*20 # does not raise py.test.raises(llarena.ArenaError, "upages + 64*20 + 1") # - ac = ArenaCollection(SHIFT + 64*20 + 7, 64, 1) + ac = ArenaCollection(64*20 + 7, 64, 1) ac.allocate_new_arena() assert ac.num_uninitialized_pages == 20 upages = ac.current_arena.freepages From pypy.commits at gmail.com Mon Jul 11 18:55:36 2016 From: pypy.commits at gmail.com (wlav) Date: Mon, 11 Jul 2016 15:55:36 -0700 (PDT) Subject: [pypy-commit] pypy cling-support: from Aditi (edited): initial enum support Message-ID: <578423e8.c91d1c0a.42e11.1a52@mx.google.com> Author: Wim Lavrijsen Branch: cling-support Changeset: r85665:66ca03bf6d3c Date: 2016-07-11 15:52 -0700 http://bitbucket.org/pypy/pypy/changeset/66ca03bf6d3c/ Log: from Aditi (edited): initial enum support diff --git a/pypy/module/cppyy/converter.py b/pypy/module/cppyy/converter.py --- a/pypy/module/cppyy/converter.py +++ b/pypy/module/cppyy/converter.py @@ -719,8 +719,8 @@ "NOT_RPYTHON" # signed types (use strtoll in setting of default in __init__) type_info = ( - (rffi.SHORT, ("short", "short int"), 'h'), - (rffi.INT, ("int",), 'i'), + (rffi.SHORT, ("short", "short int"), 'h'), + (rffi.INT, ("int", "internal_enum_type_t"), 'i'), ) # constref converters exist only b/c the stubs take constref by value, whereas diff --git a/pypy/module/cppyy/executor.py b/pypy/module/cppyy/executor.py --- a/pypy/module/cppyy/executor.py +++ b/pypy/module/cppyy/executor.py @@ -314,7 +314,7 @@ (bool, capi.c_call_b, ("bool",)), (rffi.CHAR, capi.c_call_c, ("char", "unsigned char")), (rffi.SHORT, capi.c_call_h, ("short", "short int", "unsigned short", "unsigned short int")), - (rffi.INT, capi.c_call_i, ("int",)), + (rffi.INT, capi.c_call_i, ("int", "internal_enum_type_t")), (rffi.UINT, capi.c_call_l, ("unsigned", "unsigned int")), (rffi.LONG, capi.c_call_l, ("long", "long int")), (rffi.ULONG, capi.c_call_l, ("unsigned long", "unsigned long int")), diff --git a/pypy/module/cppyy/pythonify.py b/pypy/module/cppyy/pythonify.py --- a/pypy/module/cppyy/pythonify.py +++ b/pypy/module/cppyy/pythonify.py @@ -446,7 +446,7 @@ # install a type for enums to refer to # TODO: this is correct for C++98, not for C++11 and in general there will # be the same issue for all typedef'd builtin types - setattr(gbl, 'unsigned int', int) + setattr(gbl, 'internal_enum_type_t', int) # install nullptr as a unique reference setattr(gbl, 'nullptr', cppyy._get_nullptr()) diff --git a/pypy/module/cppyy/src/clingcwrapper.cxx b/pypy/module/cppyy/src/clingcwrapper.cxx --- a/pypy/module/cppyy/src/clingcwrapper.cxx +++ b/pypy/module/cppyy/src/clingcwrapper.cxx @@ -558,6 +558,7 @@ } Bool_t Cppyy::IsEnum( const std::string& type_name ) { + if ( type_name.empty() ) return kFALSE; return gInterpreter->ClassInfo_IsEnum( type_name.c_str() ); } @@ -1068,7 +1069,10 @@ } char* cppyy_resolve_name(const char* cppitem_name) { - return cppstring_to_cstring(Cppyy::ResolveName(cppitem_name)); + std::string str = cppstring_to_cstring(Cppyy::ResolveName(cppitem_name)); + if (Cppyy::IsEnum(str)) + return cppstring_to_cstring("internal_enum_type_t"); + return cppstring_to_cstring(str); } cppyy_scope_t cppyy_get_scope(const char* scope_name) { From pypy.commits at gmail.com Tue Jul 12 10:33:29 2016 From: pypy.commits at gmail.com (mattip) Date: Tue, 12 Jul 2016 07:33:29 -0700 (PDT) Subject: [pypy-commit] pypy default: test fix for np.array()[...] returning a view (njs) Message-ID: <5784ffb9.c8981c0a.9a107.ffffd368@mx.google.com> Author: Matti Picus Branch: Changeset: r85666:6b0ba4b3816d Date: 2016-07-12 09:32 -0500 http://bitbucket.org/pypy/pypy/changeset/6b0ba4b3816d/ Log: test fix for np.array()[...] returning a view (njs) diff --git a/pypy/module/micronumpy/ndarray.py b/pypy/module/micronumpy/ndarray.py --- a/pypy/module/micronumpy/ndarray.py +++ b/pypy/module/micronumpy/ndarray.py @@ -254,7 +254,7 @@ idx = space.str_w(w_idx) return self.getfield(space, idx) if space.is_w(w_idx, space.w_Ellipsis): - return self + return self.descr_view(space, space.type(self)) elif isinstance(w_idx, W_NDimArray) and w_idx.get_dtype().is_bool(): if w_idx.ndims() > 0: w_ret = self.getitem_filter(space, w_idx) diff --git a/pypy/module/micronumpy/test/test_ndarray.py b/pypy/module/micronumpy/test/test_ndarray.py --- a/pypy/module/micronumpy/test/test_ndarray.py +++ b/pypy/module/micronumpy/test/test_ndarray.py @@ -2614,17 +2614,11 @@ import numpy as np import sys a = np.array(1.5) - if '__pypy__' in sys.builtin_module_names: - assert a[...] is a - else: - assert a[...].base is a + assert a[...].base is a a[...] = 2.5 assert a == 2.5 a = np.array([1, 2, 3]) - if '__pypy__' in sys.builtin_module_names: - assert a[...] is a - else: - assert a[...].base is a + assert a[...].base is a a[...] = 4 assert (a == [4, 4, 4]).all() assert a[..., 0] == 4 From pypy.commits at gmail.com Tue Jul 12 10:53:27 2016 From: pypy.commits at gmail.com (plan_rich) Date: Tue, 12 Jul 2016 07:53:27 -0700 (PDT) Subject: [pypy-commit] extradoc extradoc: changed first slide a bit (suggestions made by fijal) Message-ID: <57850467.c8981c0a.9a107.ffffddd0@mx.google.com> Author: Richard Plangger Branch: extradoc Changeset: r5652:78b0da4925c1 Date: 2016-07-12 09:53 -0500 http://bitbucket.org/pypy/extradoc/changeset/78b0da4925c1/ Log: changed first slide a bit (suggestions made by fijal) diff --git a/talk/compiler-workshop-2016/slides/index.html b/talk/compiler-workshop-2016/slides/index.html --- a/talk/compiler-workshop-2016/slides/index.html +++ b/talk/compiler-workshop-2016/slides/index.html @@ -35,11 +35,11 @@

    PyPy's Niche

    Mature & production-ready Python VM
    Accelerating pure Python code
    - up to 7x on average + Speedup is ~7x on average (see our benchmarks)

    PyPy is used

    - In many differnt commercial settings + In many different commercial settings
    From pypy.commits at gmail.com Tue Jul 12 12:55:26 2016 From: pypy.commits at gmail.com (wlav) Date: Tue, 12 Jul 2016 09:55:26 -0700 (PDT) Subject: [pypy-commit] pypy cling-support: assertEqual -> assert Message-ID: <578520fe.4a2d1c0a.f34ef.072a@mx.google.com> Author: Wim Lavrijsen Branch: cling-support Changeset: r85667:8ab8c51edb73 Date: 2016-07-12 09:52 -0700 http://bitbucket.org/pypy/pypy/changeset/8ab8c51edb73/ Log: assertEqual -> assert diff --git a/pypy/module/cppyy/test/test_stltypes.py b/pypy/module/cppyy/test/test_stltypes.py --- a/pypy/module/cppyy/test/test_stltypes.py +++ b/pypy/module/cppyy/test/test_stltypes.py @@ -266,7 +266,7 @@ stringy_class = cppyy.gbl.stringy_class t0 = "aap\0noot" - self.assertEqual(t0, "aap\0noot") + assert t0 == "aap\0noot" c, s = stringy_class(""), std.string(t0, len(t0)) From pypy.commits at gmail.com Tue Jul 12 12:55:28 2016 From: pypy.commits at gmail.com (wlav) Date: Tue, 12 Jul 2016 09:55:28 -0700 (PDT) Subject: [pypy-commit] pypy cling-support: from Aditi: proper type codes for instances converters Message-ID: <57852100.021a1c0a.b958f.c33f@mx.google.com> Author: Wim Lavrijsen Branch: cling-support Changeset: r85668:979fabdcc11a Date: 2016-07-12 09:52 -0700 http://bitbucket.org/pypy/pypy/changeset/979fabdcc11a/ Log: from Aditi: proper type codes for instances converters diff --git a/pypy/module/cppyy/converter.py b/pypy/module/cppyy/converter.py --- a/pypy/module/cppyy/converter.py +++ b/pypy/module/cppyy/converter.py @@ -455,9 +455,10 @@ uses_local = True class InstanceRefConverter(TypeConverter): - _immutable_fields_ = ['libffitype', 'cppclass'] + _immutable_fields_ = ['libffitype', 'typecode', 'cppclass'] libffitype = jit_libffi.types.pointer + typecode = 'V' def __init__(self, space, cppclass): from pypy.module.cppyy.interp_cppyy import W_CPPClass @@ -480,7 +481,7 @@ x[0] = rffi.cast(rffi.VOIDP, self._unwrap_object(space, w_obj)) address = rffi.cast(capi.C_OBJECT, address) ba = rffi.cast(rffi.CCHARP, address) - ba[capi.c_function_arg_typeoffset(space)] = 'o' + ba[capi.c_function_arg_typeoffset(space)] = self.typecode def convert_argument_libffi(self, space, w_obj, address, call_local): x = rffi.cast(rffi.VOIDPP, address) @@ -502,6 +503,7 @@ class InstancePtrConverter(InstanceRefConverter): + typecode = 'o' def _unwrap_object(self, space, w_obj): try: diff --git a/pypy/module/cppyy/src/clingcwrapper.cxx b/pypy/module/cppyy/src/clingcwrapper.cxx --- a/pypy/module/cppyy/src/clingcwrapper.cxx +++ b/pypy/module/cppyy/src/clingcwrapper.cxx @@ -377,6 +377,7 @@ case 'D': /* long double */ vargs[i] = (void*)&args[i].fValue.fLongDouble; break; + case 'a': case 'o': case 'p': /* void* */ vargs[i] = (void*)&args[i].fValue.fVoidp; From pypy.commits at gmail.com Tue Jul 12 13:58:18 2016 From: pypy.commits at gmail.com (wlav) Date: Tue, 12 Jul 2016 10:58:18 -0700 (PDT) Subject: [pypy-commit] pypy cling-support: fix handling of abstract classes Message-ID: <57852fba.c8981c0a.9a107.3fd6@mx.google.com> Author: Wim Lavrijsen Branch: cling-support Changeset: r85669:cfe5c3380cce Date: 2016-07-12 10:55 -0700 http://bitbucket.org/pypy/pypy/changeset/cfe5c3380cce/ Log: fix handling of abstract classes diff --git a/pypy/module/cppyy/capi/builtin_capi.py b/pypy/module/cppyy/capi/builtin_capi.py --- a/pypy/module/cppyy/capi/builtin_capi.py +++ b/pypy/module/cppyy/capi/builtin_capi.py @@ -238,6 +238,13 @@ compilation_info=backend.eci) def c_is_namespace(space, scope): return _c_is_namespace(scope) +_c_is_abstract = rffi.llexternal( + "cppyy_is_abstract", + [C_SCOPE], rffi.INT, + releasegil=ts_reflect, + compilation_info=backend.eci) +def c_is_abstract(space, cpptype): + return _c_is_abstract(cpptype) _c_is_enum = rffi.llexternal( "cppyy_is_enum", [rffi.CCHARP], rffi.INT, diff --git a/pypy/module/cppyy/capi/loadable_capi.py b/pypy/module/cppyy/capi/loadable_capi.py --- a/pypy/module/cppyy/capi/loadable_capi.py +++ b/pypy/module/cppyy/capi/loadable_capi.py @@ -162,6 +162,7 @@ # scope reflection information 'is_namespace' : ([c_scope], c_int), + 'is_abstract' : ([c_type], c_int), 'is_enum' : ([c_ccharp], c_int), # type/class reflection information @@ -367,6 +368,8 @@ # scope reflection information ----------------------------------------------- def c_is_namespace(space, scope): return space.bool_w(call_capi(space, 'is_namespace', [_Arg(h=scope)])) +def c_is_abstract(space, scope): + return space.bool_w(call_capi(space, 'is_abstract', [_Arg(h=cpptype)])) def c_is_enum(space, name): return space.bool_w(call_capi(space, 'is_enum', [_Arg(s=name)])) diff --git a/pypy/module/cppyy/include/capi.h b/pypy/module/cppyy/include/capi.h --- a/pypy/module/cppyy/include/capi.h +++ b/pypy/module/cppyy/include/capi.h @@ -85,6 +85,8 @@ RPY_EXTERN int cppyy_is_namespace(cppyy_scope_t scope); RPY_EXTERN + int cppyy_is_abstract(cppyy_type_t type); + RPY_EXTERN int cppyy_is_enum(const char* type_name); /* class reflection information ------------------------------------------- */ diff --git a/pypy/module/cppyy/interp_cppyy.py b/pypy/module/cppyy/interp_cppyy.py --- a/pypy/module/cppyy/interp_cppyy.py +++ b/pypy/module/cppyy/interp_cppyy.py @@ -1078,16 +1078,13 @@ return None def instance__init__(self, args_w): - try: - constructor_overload = self.cppclass.get_overload(self.cppclass.name) - constructor_overload.call(self, args_w) - except OperationError as e: - if not e.match(self.space, self.space.w_AttributeError): - raise + if capi.c_is_abstract(self.space, self.cppclass.handle): raise oefmt(self.space.w_TypeError, "cannot instantiate abstract class '%s'", self.cppclass.name) - + constructor_overload = self.cppclass.get_overload(self.cppclass.name) + constructor_overload.call(self, args_w) + def instance__eq__(self, w_other): # special case: if other is None, compare pointer-style if self.space.is_w(w_other, self.space.w_None): diff --git a/pypy/module/cppyy/src/clingcwrapper.cxx b/pypy/module/cppyy/src/clingcwrapper.cxx --- a/pypy/module/cppyy/src/clingcwrapper.cxx +++ b/pypy/module/cppyy/src/clingcwrapper.cxx @@ -567,16 +567,18 @@ // class reflection information ---------------------------------------------- std::string Cppyy::GetFinalName( TCppType_t klass ) { - if ( klass == GLOBAL_HANDLE ) // due to CLING WORKAROUND in InitConverters_ + if ( klass == GLOBAL_HANDLE ) return ""; - // TODO: either this or GetScopedFinalName is wrong TClassRef& cr = type_from_handle( klass ); - return cr->GetName(); + std::string clName = cr->GetName(); + std::string::size_type pos = clName.substr( 0, clName.find( '<' ) ).rfind( "::" ); + if ( pos != std::string::npos ) + return clName.substr( pos + 2, std::string::npos ); + return clName; } std::string Cppyy::GetScopedFinalName( TCppType_t klass ) { - // TODO: either this or GetFinalName is wrong TClassRef& cr = type_from_handle( klass ); return cr->GetName(); } @@ -1198,6 +1200,10 @@ return (int)Cppyy::IsNamespace(scope); } +int cppyy_is_abstract(cppyy_type_t type){ + return (int)Cppyy::IsAbstract(type); +} + int cppyy_is_enum(const char* type_name){ return (int)Cppyy::IsEnum(type_name); } From pypy.commits at gmail.com Tue Jul 12 14:17:49 2016 From: pypy.commits at gmail.com (wlav) Date: Tue, 12 Jul 2016 11:17:49 -0700 (PDT) Subject: [pypy-commit] pypy cling-support: pass length when converting char* to std::string Message-ID: <5785344d.d48e1c0a.5f623.79ea@mx.google.com> Author: Wim Lavrijsen Branch: cling-support Changeset: r85670:76d09d04588d Date: 2016-07-12 11:15 -0700 http://bitbucket.org/pypy/pypy/changeset/76d09d04588d/ Log: pass length when converting char* to std::string diff --git a/pypy/module/cppyy/capi/builtin_capi.py b/pypy/module/cppyy/capi/builtin_capi.py --- a/pypy/module/cppyy/capi/builtin_capi.py +++ b/pypy/module/cppyy/capi/builtin_capi.py @@ -541,12 +541,12 @@ _c_charp2stdstring = rffi.llexternal( "cppyy_charp2stdstring", - [rffi.CCHARP], C_OBJECT, + [rffi.CCHARP, rffi.SIZE_T], C_OBJECT, releasegil=ts_helper, compilation_info=backend.eci) -def c_charp2stdstring(space, svalue): +def c_charp2stdstring(space, svalue, sz): charp = rffi.str2charp(svalue) - result = _c_charp2stdstring(charp) + result = _c_charp2stdstring(charp, sz) rffi.free_charp(charp) return result _c_stdstring2stdstring = rffi.llexternal( diff --git a/pypy/module/cppyy/capi/loadable_capi.py b/pypy/module/cppyy/capi/loadable_capi.py --- a/pypy/module/cppyy/capi/loadable_capi.py +++ b/pypy/module/cppyy/capi/loadable_capi.py @@ -216,7 +216,7 @@ 'strtoull' : ([c_ccharp], c_ullong), 'free' : ([c_voidp], c_void), - 'charp2stdstring' : ([c_ccharp], c_object), + 'charp2stdstring' : ([c_ccharp, c_size_t], c_object), 'stdstring2stdstring' : ([c_object], c_object), } @@ -517,8 +517,9 @@ c_free(space, rffi.cast(rffi.VOIDP, charp)) return pystr -def c_charp2stdstring(space, svalue): - return _cdata_to_cobject(space, call_capi(space, 'charp2stdstring', [_Arg(s=svalue)])) +def c_charp2stdstring(space, svalue, sz): + return _cdata_to_cobject( + space, call_capi(space, 'charp2stdstring', [_Arg(s=svalue), _Arg(l=sz)])) def c_stdstring2stdstring(space, cppobject): return _cdata_to_cobject(space, call_capi(space, 'stdstring2stdstring', [_Arg(h=cppobject)])) diff --git a/pypy/module/cppyy/converter.py b/pypy/module/cppyy/converter.py --- a/pypy/module/cppyy/converter.py +++ b/pypy/module/cppyy/converter.py @@ -563,7 +563,7 @@ arg = InstanceConverter._unwrap_object(self, space, w_obj) return capi.c_stdstring2stdstring(space, arg) else: - return capi.c_charp2stdstring(space, space.str_w(w_obj)) + return capi.c_charp2stdstring(space, space.str_w(w_obj), space.len_w(w_obj)) def to_memory(self, space, w_obj, w_value, offset): try: diff --git a/pypy/module/cppyy/include/capi.h b/pypy/module/cppyy/include/capi.h --- a/pypy/module/cppyy/include/capi.h +++ b/pypy/module/cppyy/include/capi.h @@ -177,7 +177,7 @@ void cppyy_free(void* ptr); RPY_EXTERN - cppyy_object_t cppyy_charp2stdstring(const char* str); + cppyy_object_t cppyy_charp2stdstring(const char* str, size_t sz); RPY_EXTERN cppyy_object_t cppyy_stdstring2stdstring(cppyy_object_t ptr); diff --git a/pypy/module/cppyy/src/clingcwrapper.cxx b/pypy/module/cppyy/src/clingcwrapper.cxx --- a/pypy/module/cppyy/src/clingcwrapper.cxx +++ b/pypy/module/cppyy/src/clingcwrapper.cxx @@ -1405,8 +1405,8 @@ free(ptr); } -cppyy_object_t cppyy_charp2stdstring(const char* str){ - return (cppyy_object_t)new std::string(str); +cppyy_object_t cppyy_charp2stdstring(const char* str, size_t sz){ + return (cppyy_object_t)new std::string(str, sz); } cppyy_object_t cppyy_stdstring2stdstring(cppyy_object_t ptr){ From pypy.commits at gmail.com Tue Jul 12 14:43:18 2016 From: pypy.commits at gmail.com (wlav) Date: Tue, 12 Jul 2016 11:43:18 -0700 (PDT) Subject: [pypy-commit] pypy cling-support: simpler template handling Message-ID: <57853a46.d48e1c0a.5f623.ffff8845@mx.google.com> Author: Wim Lavrijsen Branch: cling-support Changeset: r85671:7e5b01d3cab2 Date: 2016-07-12 11:40 -0700 http://bitbucket.org/pypy/pypy/changeset/7e5b01d3cab2/ Log: simpler template handling diff --git a/pypy/module/cppyy/capi/builtin_capi.py b/pypy/module/cppyy/capi/builtin_capi.py --- a/pypy/module/cppyy/capi/builtin_capi.py +++ b/pypy/module/cppyy/capi/builtin_capi.py @@ -53,13 +53,6 @@ compilation_info=backend.eci) def c_get_scope_opaque(space, name): return _c_get_scope_opaque(name) -_c_get_template = rffi.llexternal( - "cppyy_get_template", - [rffi.CCHARP], C_TYPE, - releasegil=ts_reflect, - compilation_info=backend.eci) -def c_get_template(space, name): - return _c_get_template(name) _c_actual_class = rffi.llexternal( "cppyy_actual_class", [C_TYPE, C_OBJECT], C_TYPE, @@ -238,6 +231,13 @@ compilation_info=backend.eci) def c_is_namespace(space, scope): return _c_is_namespace(scope) +_c_is_template = rffi.llexternal( + "cppyy_is_template", + [rffi.CCHARP], rffi.INT, + releasegil=ts_reflect, + compilation_info=backend.eci) +def c_is_template(space, name): + return _c_is_template(name) _c_is_abstract = rffi.llexternal( "cppyy_is_abstract", [C_SCOPE], rffi.INT, diff --git a/pypy/module/cppyy/capi/loadable_capi.py b/pypy/module/cppyy/capi/loadable_capi.py --- a/pypy/module/cppyy/capi/loadable_capi.py +++ b/pypy/module/cppyy/capi/loadable_capi.py @@ -126,7 +126,6 @@ 'resolve_name' : ([c_ccharp], c_ccharp), 'get_scope' : ([c_ccharp], c_scope), - 'get_template' : ([c_ccharp], c_type), 'actual_class' : ([c_type, c_object], c_type), # memory management @@ -162,6 +161,7 @@ # scope reflection information 'is_namespace' : ([c_scope], c_int), + 'is_template' : ([c_ccharp], c_int), 'is_abstract' : ([c_type], c_int), 'is_enum' : ([c_ccharp], c_int), @@ -291,8 +291,6 @@ return charp2str_free(space, call_capi(space, 'resolve_name', [_Arg(s=name)])) def c_get_scope_opaque(space, name): return rffi.cast(C_SCOPE, space.uint_w(call_capi(space, 'get_scope', [_Arg(s=name)]))) -def c_get_template(space, name): - return rffi.cast(C_TYPE, space.uint_w(call_capi(space, 'get_template', [_Arg(s=name)]))) def c_actual_class(space, cppclass, cppobj): args = [_Arg(h=cppclass.handle), _Arg(h=cppobj)] return rffi.cast(C_TYPE, space.uint_w(call_capi(space, 'actual_class', args))) @@ -368,6 +366,8 @@ # scope reflection information ----------------------------------------------- def c_is_namespace(space, scope): return space.bool_w(call_capi(space, 'is_namespace', [_Arg(h=scope)])) +def c_is_template(space, name): + return space.bool_w(call_capi(space, 'is_template', [_Arg(s=name)])) def c_is_abstract(space, scope): return space.bool_w(call_capi(space, 'is_abstract', [_Arg(h=cpptype)])) def c_is_enum(space, name): diff --git a/pypy/module/cppyy/include/capi.h b/pypy/module/cppyy/include/capi.h --- a/pypy/module/cppyy/include/capi.h +++ b/pypy/module/cppyy/include/capi.h @@ -26,8 +26,6 @@ RPY_EXTERN cppyy_scope_t cppyy_get_scope(const char* scope_name); RPY_EXTERN - cppyy_type_t cppyy_get_template(const char* template_name); - RPY_EXTERN cppyy_type_t cppyy_actual_class(cppyy_type_t klass, cppyy_object_t obj); /* memory management ------------------------------------------------------ */ @@ -85,6 +83,8 @@ RPY_EXTERN int cppyy_is_namespace(cppyy_scope_t scope); RPY_EXTERN + int cppyy_is_template(const char* template_name); + RPY_EXTERN int cppyy_is_abstract(cppyy_type_t type); RPY_EXTERN int cppyy_is_enum(const char* type_name); diff --git a/pypy/module/cppyy/include/cpp_cppyy.h b/pypy/module/cppyy/include/cpp_cppyy.h --- a/pypy/module/cppyy/include/cpp_cppyy.h +++ b/pypy/module/cppyy/include/cpp_cppyy.h @@ -6,21 +6,21 @@ #include #include -//ROOT types - typedef long Long_t; - typedef unsigned long ULong_t; - typedef long long Long64_t; - typedef unsigned long long ULong64_t; - typedef float Float_t; - typedef double Double_t; - typedef long double LongDouble_t; - typedef bool Bool_t; - typedef char Char_t; - typedef unsigned char UChar_t; - typedef short Short_t; - typedef unsigned short UShort_t; - typedef int Int_t; - typedef unsigned int UInt_t; +// ROOT types +typedef long Long_t; +typedef unsigned long ULong_t; +typedef long long Long64_t; +typedef unsigned long long ULong64_t; +typedef float Float_t; +typedef double Double_t; +typedef long double LongDouble_t; +typedef bool Bool_t; +typedef char Char_t; +typedef unsigned char UChar_t; +typedef short Short_t; +typedef unsigned short UShort_t; +typedef int Int_t; +typedef unsigned int UInt_t; namespace Cppyy { typedef ptrdiff_t TCppScope_t; @@ -36,7 +36,6 @@ std::string GetScopeName( TCppScope_t parent, TCppIndex_t iscope ); std::string ResolveName( const std::string& cppitem_name ); TCppScope_t GetScope( const std::string& scope_name ); - TCppType_t GetTemplate( const std::string& template_name ); TCppType_t GetActualClass( TCppType_t klass, TCppObject_t obj ); size_t SizeOf( TCppType_t klass ); @@ -78,6 +77,7 @@ // scope reflection information ---------------------------------------------- Bool_t IsNamespace( TCppScope_t scope ); + Bool_t IsTemplate( const std::string& template_name ); Bool_t IsAbstract( TCppType_t type ); Bool_t IsEnum( const std::string& type_name ); diff --git a/pypy/module/cppyy/interp_cppyy.py b/pypy/module/cppyy/interp_cppyy.py --- a/pypy/module/cppyy/interp_cppyy.py +++ b/pypy/module/cppyy/interp_cppyy.py @@ -103,10 +103,8 @@ except KeyError: pass - opaque_handle = capi.c_get_template(space, name) - assert lltype.typeOf(opaque_handle) == capi.C_TYPE - if opaque_handle: - cpptemplate = W_CPPTemplateType(space, name, opaque_handle) + if capi.c_is_template(space, name): + cpptemplate = W_CPPTemplateType(space, name) state.cpptemplate_cache[name] = cpptemplate return cpptemplate @@ -997,14 +995,12 @@ class W_CPPTemplateType(W_Root): - _attrs_ = ['space', 'name', 'handle'] - _immutable_fields = ['name', 'handle'] + _attrs_ = ['space', 'name'] + _immutable_fields = ['name'] - def __init__(self, space, name, opaque_handle): + def __init__(self, space, name): self.space = space self.name = name - assert lltype.typeOf(opaque_handle) == capi.C_TYPE - self.handle = opaque_handle @unwrap_spec(args_w='args_w') def __call__(self, args_w): diff --git a/pypy/module/cppyy/src/clingcwrapper.cxx b/pypy/module/cppyy/src/clingcwrapper.cxx --- a/pypy/module/cppyy/src/clingcwrapper.cxx +++ b/pypy/module/cppyy/src/clingcwrapper.cxx @@ -178,9 +178,9 @@ return (TCppScope_t)sz; } -Cppyy::TCppType_t Cppyy::GetTemplate( const std::string& /* template_name */ ) +Bool_t Cppyy::IsTemplate( const std::string& template_name ) { - return (TCppType_t)0; + return (Bool_t)gInterpreter->CheckClassTemplate( template_name.c_str() ); } Cppyy::TCppType_t Cppyy::GetActualClass( TCppType_t klass, TCppObject_t obj ) @@ -1082,10 +1082,6 @@ return cppyy_scope_t(Cppyy::GetScope(scope_name)); } -cppyy_type_t cppyy_get_template(const char* template_name) { - return cppyy_type_t(Cppyy::GetTemplate(template_name)); -} - cppyy_type_t cppyy_actual_class(cppyy_type_t klass, cppyy_object_t obj) { return cppyy_type_t(Cppyy::GetActualClass(klass, (void*)obj)); } @@ -1200,6 +1196,10 @@ return (int)Cppyy::IsNamespace(scope); } +int cppyy_is_template(const char* template_name) { + return (int)Cppyy::IsTemplate(template_name); +} + int cppyy_is_abstract(cppyy_type_t type){ return (int)Cppyy::IsAbstract(type); } From pypy.commits at gmail.com Tue Jul 12 16:39:56 2016 From: pypy.commits at gmail.com (raffael_t) Date: Tue, 12 Jul 2016 13:39:56 -0700 (PDT) Subject: [pypy-commit] pypy py3.5-async: Implement GET_YIELD_FROM_ITER pyopcode, 'yield from' works, write test for 'yield from' Message-ID: <5785559c.c3cb1c0a.d794b.ffff8c8d@mx.google.com> Author: Raffael Tfirst Branch: py3.5-async Changeset: r85672:74022d9a4bbc Date: 2016-07-12 22:39 +0200 http://bitbucket.org/pypy/pypy/changeset/74022d9a4bbc/ Log: Implement GET_YIELD_FROM_ITER pyopcode, 'yield from' works, write test for 'yield from' diff --git a/pypy/interpreter/astcompiler/assemble.py b/pypy/interpreter/astcompiler/assemble.py --- a/pypy/interpreter/astcompiler/assemble.py +++ b/pypy/interpreter/astcompiler/assemble.py @@ -646,6 +646,7 @@ ops.BEFORE_ASYNC_WITH: 1, ops.GET_AITER: 0, ops.GET_ANEXT: 1, + ops.GET_YIELD_FROM_ITER: 0, ops.LOAD_CONST: 1, diff --git a/pypy/interpreter/astcompiler/codegen.py b/pypy/interpreter/astcompiler/codegen.py --- a/pypy/interpreter/astcompiler/codegen.py +++ b/pypy/interpreter/astcompiler/codegen.py @@ -1022,7 +1022,7 @@ def visit_YieldFrom(self, yfr): self.update_position(yfr.lineno) yfr.value.walkabout(self) - self.emit_op(ops.GET_ITER) + self.emit_op(ops.GET_YIELD_FROM_ITER) self.load_const(self.space.w_None) self.emit_op(ops.YIELD_FROM) diff --git a/pypy/interpreter/pyopcode.py b/pypy/interpreter/pyopcode.py --- a/pypy/interpreter/pyopcode.py +++ b/pypy/interpreter/pyopcode.py @@ -418,6 +418,18 @@ self.YIELD_VALUE(oparg, next_instr) elif opcode == opcodedesc.YIELD_FROM.index: self.YIELD_FROM(oparg, next_instr) + elif opcode == opcodedesc.GET_YIELD_FROM_ITER.index: + self.GET_YIELD_FROM_ITER(oparg, next_instr) + elif opcode == opcodedesc.GET_AWAITABLE.index: + self.GET_AWAITABLE(oparg, next_instr) + elif opcode == opcodedesc.SETUP_ASYNC_WITH.index: + self.SETUP_ASYNC_WITH(oparg, next_instr) + elif opcode == opcodedesc.BEFORE_ASYNC_WITH.index: + self.BEFORE_ASYNC_WITH(oparg, next_instr) + elif opcode == opcodedesc.GET_AITER.index: + self.GET_AITER(oparg, next_instr) + elif opcode == opcodedesc.GET_ANEXT.index: + self.GET_ANEXT(oparg, next_instr) else: self.MISSING_OPCODE(oparg, next_instr) @@ -1403,19 +1415,24 @@ itemcount -= 1 self.pushvalue(w_dict) - def GET_AWAITABLE(self): + def GET_YIELD_FROM_ITER(self, oparg, next_instr): + w_iterable = self.popvalue() + w_iterator = self.space.iter(w_iterable) + self.pushvalue(w_iterator) + + def GET_AWAITABLE(self, oparg, next_instr): pass - def SETUP_ASYNC_WITH(self): + def SETUP_ASYNC_WITH(self, oparg, next_instr): pass - def BEFORE_ASYNC_WITH(self): + def BEFORE_ASYNC_WITH(self, oparg, next_instr): pass - def GET_AITER(self): + def GET_AITER(self, oparg, next_instr): pass - def GET_ANEXT(self): + def GET_ANEXT(self, oparg, next_instr): pass ### ____________________________________________________________ ### diff --git a/pypy/interpreter/test/test_compiler.py b/pypy/interpreter/test/test_compiler.py --- a/pypy/interpreter/test/test_compiler.py +++ b/pypy/interpreter/test/test_compiler.py @@ -612,6 +612,24 @@ space.exec_(code, w_d, w_d) w_res = space.getitem(w_d, space.wrap('res')) assert space.eq_w(w_res, space.wrap("var")) + + def test_(self): + space = self.space + snippet = str(py.code.Source(r''' + def f(): + def generator2(): + yield 8 + def generator(): + yield from generator2() + for i in generator(): return i + res = f() + ''')) + code = self.compiler.compile(snippet, '', 'exec', 0) + space = self.space + w_d = space.newdict() + space.exec_(code, w_d, w_d) + w_res = space.getitem(w_d, space.wrap('res')) + assert space.eq_w(w_res, space.wrap("8")) def test_dont_inherit_flag(self): # this test checks that compile() don't inherit the __future__ flags From pypy.commits at gmail.com Wed Jul 13 04:51:08 2016 From: pypy.commits at gmail.com (raffael_t) Date: Wed, 13 Jul 2016 01:51:08 -0700 (PDT) Subject: [pypy-commit] pypy py3.5-async: Add missing name for 'yield from' test Message-ID: <578600fc.46461c0a.2305c.ffff9cf5@mx.google.com> Author: Raffael Tfirst Branch: py3.5-async Changeset: r85673:8026b5c3d0c8 Date: 2016-07-13 10:50 +0200 http://bitbucket.org/pypy/pypy/changeset/8026b5c3d0c8/ Log: Add missing name for 'yield from' test diff --git a/pypy/interpreter/test/test_compiler.py b/pypy/interpreter/test/test_compiler.py --- a/pypy/interpreter/test/test_compiler.py +++ b/pypy/interpreter/test/test_compiler.py @@ -613,7 +613,7 @@ w_res = space.getitem(w_d, space.wrap('res')) assert space.eq_w(w_res, space.wrap("var")) - def test_(self): + def test_yield_from(self): space = self.space snippet = str(py.code.Source(r''' def f(): From pypy.commits at gmail.com Wed Jul 13 05:26:17 2016 From: pypy.commits at gmail.com (arigo) Date: Wed, 13 Jul 2016 02:26:17 -0700 (PDT) Subject: [pypy-commit] pypy default: update the faq Message-ID: <57860939.0dc11c0a.d2b73.76a0@mx.google.com> Author: Armin Rigo Branch: Changeset: r85674:02aa2e218f83 Date: 2016-07-13 11:27 +0200 http://bitbucket.org/pypy/pypy/changeset/02aa2e218f83/ Log: update the faq diff --git a/pypy/doc/faq.rst b/pypy/doc/faq.rst --- a/pypy/doc/faq.rst +++ b/pypy/doc/faq.rst @@ -379,8 +379,9 @@ give us some temporary ssh access to a machine where the bug can be reproduced. -* If giving us access would require us to sign a NDA, then we can - consider a commerical support contract for a small sum of money. +* If giving us access would require us to use other tools than ssh, + make appointments, or sign a NDA, then we can consider a commerical + support contract for a small sum of money. * If even that is not possible for you, then sorry, we can't help. From pypy.commits at gmail.com Wed Jul 13 05:55:08 2016 From: pypy.commits at gmail.com (arigo) Date: Wed, 13 Jul 2016 02:55:08 -0700 (PDT) Subject: [pypy-commit] pypy default: add vmware Message-ID: <57860ffc.632bc20a.1b561.0127@mx.google.com> Author: Armin Rigo Branch: Changeset: r85675:48ca27a04e61 Date: 2016-07-13 11:56 +0200 http://bitbucket.org/pypy/pypy/changeset/48ca27a04e61/ Log: add vmware diff --git a/pypy/doc/faq.rst b/pypy/doc/faq.rst --- a/pypy/doc/faq.rst +++ b/pypy/doc/faq.rst @@ -377,7 +377,8 @@ * If the bug involves Closed Source components, or just too many Open Source components to install them all ourselves, then maybe you can give us some temporary ssh access to a machine where the bug can be - reproduced. + reproduced. Or, maybe we can download a VirtualBox or VMWare + virtual machine where the problem occurs. * If giving us access would require us to use other tools than ssh, make appointments, or sign a NDA, then we can consider a commerical From pypy.commits at gmail.com Wed Jul 13 11:16:07 2016 From: pypy.commits at gmail.com (arigo) Date: Wed, 13 Jul 2016 08:16:07 -0700 (PDT) Subject: [pypy-commit] pypy use-mmap: Try to clean up rmmap and use hints for all internal mmap()s Message-ID: <57865b37.2e91c20a.f403d.ffff8957@mx.google.com> Author: Armin Rigo Branch: use-mmap Changeset: r85676:ae84c87cdd28 Date: 2016-07-13 15:46 +0100 http://bitbucket.org/pypy/pypy/changeset/ae84c87cdd28/ Log: Try to clean up rmmap and use hints for all internal mmap()s diff --git a/rpython/rlib/rmmap.py b/rpython/rlib/rmmap.py --- a/rpython/rlib/rmmap.py +++ b/rpython/rlib/rmmap.py @@ -694,15 +694,10 @@ def allocate_memory_chunk(map_size): # used by the memory allocator (in llarena.py, from minimarkpage.py) - flags = MAP_PRIVATE | MAP_ANONYMOUS - prot = PROT_READ | PROT_WRITE + hint = hint_regular if we_are_translated(): - flags = NonConstant(flags) - prot = NonConstant(prot) - res = c_mmap_safe(rffi.cast(PTR, 0), map_size, prot, flags, -1, 0) - if res == rffi.cast(PTR, -1): - res = rffi.cast(PTR, 0) - return res + hint = NonConstant(hint) + return mmap_hinted(hint, map_size) def reset_memory_chunk(addr, map_size): # used by the memory allocator (in llarena.py, from minimarkpage.py) @@ -715,14 +710,50 @@ flags = NonConstant(flags) prot = NonConstant(prot) c_mmap_safe(rffi.cast(PTR, addr), map_size, prot, flags, -1, 0) + # ignore unlikely errors - def alloc_hinted(hintp, map_size): + def mmap_hinted(hint, map_size): + from errno import ENOMEM + from rpython.rlib import debug + + if _CYGWIN: + # XXX: JIT memory should be using mmap MAP_PRIVATE with + # PROT_EXEC but Cygwin's fork() fails. mprotect() + # cannot be used, but seems to be unnecessary there. + # Just use malloc(). + return c_malloc_safe(map_size) + flags = MAP_PRIVATE | MAP_ANONYMOUS - prot = PROT_EXEC | PROT_READ | PROT_WRITE if we_are_translated(): flags = NonConstant(flags) - prot = NonConstant(prot) - return c_mmap_safe(hintp, map_size, prot, flags, -1, 0) + pos = hint.pos + if hint.direction < 0: + pos -= map_size + res = c_mmap_safe(rffi.cast(PTR, pos), map_size, + hint.prot, flags, -1, 0) + if res == rffi.cast(PTR, -1): + # some systems (some versions of OS/X?) complain if they + # are passed a non-zero address. Try again without a hint. + res = c_mmap_safe(rffi.cast(PTR, 0), map_size, + hint.prot, flags, -1, 0) + if res == rffi.cast(PTR, -1): + # ENOMEM simply raises MemoryError, but other errors are fatal + if rposix.get_saved_errno() != ENOMEM: + if hint.prot & PROT_EXEC: + debug.fatalerror_notb( + "Got an unexpected error trying to allocate some " + "memory for the JIT (tried to do mmap() with " + "PROT_EXEC|PROT_READ|PROT_WRITE). This can be caused " + "by a system policy like PAX. You need to find how " + "to work around the policy on your system.") + else: + debug.fatalerror_notb( + "Got an unexpected error trying to allocate memory " + "with mmap(). Cannot continue.") + return rffi.cast(PTR, 0) + else: + hint.pos += map_size * hint.direction + return res def clear_large_memory_chunk_aligned(addr, map_size): addr = rffi.cast(PTR, addr) @@ -734,47 +765,51 @@ res = c_mmap_safe(addr, map_size, prot, flags, -1, 0) return res == addr - # XXX is this really necessary? - class Hint: - if sys.maxint <= 2**32: - pos = -0x4fff0000 # for reproducible results - else: - pos = 0x4fde00000000 - hint = Hint() + # gives a hint to mmap(), for reproducible results, but also to + # avoid the following problem. We use both malloc() for all large + # objects, and direct mmap() of 512KB arenas for small objects. + # Without any hint, the mmap() tend to go just after the malloc() + # zone; and then the malloc() zone cannot grow and fragments. We + # avoid that by hinting the mmap()s to go somewhere else. + # + # The 'hint_regular' decrements. We also allocate a few smaller + # executable zones for the JIT's machine code; this is 'hint_jit' + # which increments. + # + # These hints are valid on Linux. XXX check on OS/X and possibly + # elsewhere. + # + # Using a 'Struct' instead of a regular 'class' as a way to work + # around the multiple steps of a translation. + + HINT = lltype.Struct('HINT', ('pos', lltype.Signed), + ('direction', lltype.Signed), + ('prot', lltype.Signed)) + + hint_regular = lltype.malloc(HINT, flavor='raw', immortal=True) + if sys.maxint <= 2**32: + hint_regular.pos = -0x20000000 + else: + hint_regular.pos = 0x4fde00000000 + hint_regular.direction = -1 + hint_regular.prot = PROT_READ | PROT_WRITE + + hint_jit = lltype.malloc(HINT, flavor='raw', immortal=True) + hint_jit.pos = hint_regular.pos + hint_jit.direction = +1 + hint_jit.prot = PROT_EXEC | PROT_READ | PROT_WRITE def alloc(map_size): """Allocate memory. This is intended to be used by the JIT, so the memory has the executable bit set and gets allocated internally in case of a sandboxed process. """ - from errno import ENOMEM - from rpython.rlib import debug - - if _CYGWIN: - # XXX: JIT memory should be using mmap MAP_PRIVATE with - # PROT_EXEC but Cygwin's fork() fails. mprotect() - # cannot be used, but seems to be unnecessary there. - res = c_malloc_safe(map_size) - if res == rffi.cast(PTR, 0): - raise MemoryError - return res - res = alloc_hinted(rffi.cast(PTR, hint.pos), map_size) - if res == rffi.cast(PTR, -1): - # some systems (some versions of OS/X?) complain if they - # are passed a non-zero address. Try again. - res = alloc_hinted(rffi.cast(PTR, 0), map_size) - if res == rffi.cast(PTR, -1): - # ENOMEM simply raises MemoryError, but other errors are fatal - if rposix.get_saved_errno() != ENOMEM: - debug.fatalerror_notb( - "Got an unexpected error trying to allocate some " - "memory for the JIT (tried to do mmap() with " - "PROT_EXEC|PROT_READ|PROT_WRITE). This can be caused " - "by a system policy like PAX. You need to find how " - "to work around the policy on your system.") - raise MemoryError - else: - hint.pos += map_size + hint = hint_jit + if we_are_translated(): + hint = NonConstant(hint) + res = mmap_hinted(hint, map_size) + if res == rffi.cast(PTR, 0): + raise MemoryError return res alloc._annenforceargs_ = (int,) @@ -909,11 +944,6 @@ m.map_handle = INVALID_HANDLE raise winerror - class Hint: - pos = -0x4fff0000 # for reproducible results - hint = Hint() - # XXX this has no effect on windows - def alloc(map_size): """Allocate memory. This is intended to be used by the JIT, so the memory has the executable bit set. diff --git a/rpython/rlib/test/test_rmmap.py b/rpython/rlib/test/test_rmmap.py --- a/rpython/rlib/test/test_rmmap.py +++ b/rpython/rlib/test/test_rmmap.py @@ -73,6 +73,11 @@ f.write(size*4096*"c") f.flush() + class Hint: + prot = mmap.PROT_EXEC | mmap.PROT_READ | mmap.PROT_WRITE + direction = -1 + hint = Hint() + def func(no): m = mmap.mmap(no, size*4096) m.unmap_range(left*4096, (right-left)*4096) @@ -84,7 +89,8 @@ return rffi.ptradd(m.data, offset) def as_num(ptr): return rffi.cast(lltype.Unsigned, ptr) - res = mmap.alloc_hinted(in_map(m, (left+right)/2 * 4096), 4096) + hint.pos = rffi.cast(lltype.Signed, in_map(m, (left+right)/2 * 4096)) + res = mmap.mmap_hinted(hint, 4096) assert as_num(in_map(m, left*4096)) <= as_num(res) < as_num(in_map(m, right*4096)) interpret(func, [f.fileno()]) f.close() From pypy.commits at gmail.com Wed Jul 13 11:16:09 2016 From: pypy.commits at gmail.com (arigo) Date: Wed, 13 Jul 2016 08:16:09 -0700 (PDT) Subject: [pypy-commit] pypy use-mmap: translation fix Message-ID: <57865b39.4d9a1c0a.71057.4463@mx.google.com> Author: Armin Rigo Branch: use-mmap Changeset: r85677:a1cac4893166 Date: 2016-07-13 16:23 +0100 http://bitbucket.org/pypy/pypy/changeset/a1cac4893166/ Log: translation fix diff --git a/rpython/rlib/rmmap.py b/rpython/rlib/rmmap.py --- a/rpython/rlib/rmmap.py +++ b/rpython/rlib/rmmap.py @@ -8,7 +8,7 @@ from rpython.rtyper.tool import rffi_platform from rpython.rtyper.lltypesystem import rffi, lltype -from rpython.rlib import rposix +from rpython.rlib import rposix, rgc, jit from rpython.translator.tool.cbuild import ExternalCompilationInfo from rpython.rlib.objectmodel import we_are_translated from rpython.rlib.nonconst import NonConstant @@ -109,8 +109,7 @@ else: HAVE_LARGEFILE_SUPPORT = False -def external(name, args, result, save_err_on_unsafe=0, save_err_on_safe=0, - **kwargs): +def external(name, args, result, save_err_on_unsafe=0, **kwargs): unsafe = rffi.llexternal(name, args, result, compilation_info=CConfig._compilation_info_, save_err=save_err_on_unsafe, @@ -118,7 +117,7 @@ safe = rffi.llexternal(name, args, result, compilation_info=CConfig._compilation_info_, sandboxsafe=True, releasegil=False, - save_err=save_err_on_safe, + _nowrapper=True, **kwargs) return unsafe, safe @@ -330,7 +329,8 @@ Per munmap(1), the offset must be a multiple of the page size, and the size will be rounded up to a multiple of the page size. """ - c_munmap_safe(self.getptr(offset), size) + c_munmap_safe(self.getptr(offset), + rffi.cast(size_t, size)) def close(self): if _MS_WINDOWS: @@ -704,14 +704,19 @@ # to release the memory currently held in a memory chunk, possibly # zeroing it, but keep that memory chunk available for future use. # should only be used on allocate_memory_chunk() memory. - flags = MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED + flags = MAP_FIXED | MAP_PRIVATE | MAP_ANONYMOUS prot = PROT_READ | PROT_WRITE - if we_are_translated(): - flags = NonConstant(flags) - prot = NonConstant(prot) - c_mmap_safe(rffi.cast(PTR, addr), map_size, prot, flags, -1, 0) - # ignore unlikely errors + addr = rffi.cast(PTR, addr) + res = c_mmap_safe(addr, + rffi.cast(size_t, map_size), + rffi.cast(rffi.INT, prot), + rffi.cast(rffi.INT, flags), + rffi.cast(rffi.INT, -1), + rffi.cast(off_t, 0)) + return res == addr + @rgc.no_collect + @jit.dont_look_inside def mmap_hinted(hint, map_size): from errno import ENOMEM from rpython.rlib import debug @@ -721,24 +726,30 @@ # PROT_EXEC but Cygwin's fork() fails. mprotect() # cannot be used, but seems to be unnecessary there. # Just use malloc(). - return c_malloc_safe(map_size) + return c_malloc_safe(rffi.cast(size_t, map_size)) flags = MAP_PRIVATE | MAP_ANONYMOUS - if we_are_translated(): - flags = NonConstant(flags) pos = hint.pos if hint.direction < 0: pos -= map_size - res = c_mmap_safe(rffi.cast(PTR, pos), map_size, - hint.prot, flags, -1, 0) + res = c_mmap_safe(rffi.cast(PTR, pos), + rffi.cast(size_t, map_size), + rffi.cast(rffi.INT, hint.prot), + rffi.cast(rffi.INT, flags), + rffi.cast(rffi.INT, -1), + rffi.cast(off_t, 0)) if res == rffi.cast(PTR, -1): # some systems (some versions of OS/X?) complain if they # are passed a non-zero address. Try again without a hint. - res = c_mmap_safe(rffi.cast(PTR, 0), map_size, - hint.prot, flags, -1, 0) + res = c_mmap_safe(rffi.cast(PTR, 0), + rffi.cast(size_t, map_size), + rffi.cast(rffi.INT, hint.prot), + rffi.cast(rffi.INT, flags), + rffi.cast(rffi.INT, -1), + rffi.cast(off_t, 0)) if res == rffi.cast(PTR, -1): # ENOMEM simply raises MemoryError, but other errors are fatal - if rposix.get_saved_errno() != ENOMEM: + if intmask(rposix._get_errno()) != ENOMEM: if hint.prot & PROT_EXEC: debug.fatalerror_notb( "Got an unexpected error trying to allocate some " @@ -756,14 +767,7 @@ return res def clear_large_memory_chunk_aligned(addr, map_size): - addr = rffi.cast(PTR, addr) - flags = MAP_FIXED | MAP_PRIVATE | MAP_ANONYMOUS - prot = PROT_READ | PROT_WRITE - if we_are_translated(): - flags = NonConstant(flags) - prot = NonConstant(prot) - res = c_mmap_safe(addr, map_size, prot, flags, -1, 0) - return res == addr + return reset_memory_chunk(rffi.cast(PTR, addr), map_size) # gives a hint to mmap(), for reproducible results, but also to # avoid the following problem. We use both malloc() for all large @@ -813,10 +817,12 @@ return res alloc._annenforceargs_ = (int,) - if _CYGWIN: - free = c_free_safe - else: - free = c_munmap_safe + def free(addr, map_size): + if _CYGWIN: + _free = c_free_safe + else: + _free = c_munmap_safe + _free(rffi.cast(PTR, addr), rffi.cast(size_t, map_size)) elif _MS_WINDOWS: def mmap(fileno, length, tagname="", access=_ACCESS_DEFAULT, offset=0): From pypy.commits at gmail.com Wed Jul 13 12:48:01 2016 From: pypy.commits at gmail.com (wlav) Date: Wed, 13 Jul 2016 09:48:01 -0700 (PDT) Subject: [pypy-commit] pypy cling-support: fix configuration and reduce (direct) dependency on cpyext tests Message-ID: <578670c1.075dc20a.5597b.ffffafa9@mx.google.com> Author: Wim Lavrijsen Branch: cling-support Changeset: r85678:2e0d2cee85c8 Date: 2016-07-13 09:45 -0700 http://bitbucket.org/pypy/pypy/changeset/2e0d2cee85c8/ Log: fix configuration and reduce (direct) dependency on cpyext tests diff --git a/pypy/module/cppyy/test/crossing.xml b/pypy/module/cppyy/test/crossing.xml --- a/pypy/module/cppyy/test/crossing.xml +++ b/pypy/module/cppyy/test/crossing.xml @@ -1,7 +1,6 @@ - - + diff --git a/pypy/module/cppyy/test/test_crossing.py b/pypy/module/cppyy/test/test_crossing.py --- a/pypy/module/cppyy/test/test_crossing.py +++ b/pypy/module/cppyy/test/test_crossing.py @@ -9,8 +9,6 @@ from pypy.module.cpyext import api from pypy.module.cpyext.state import State -from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase - currpath = py.path.local(__file__).dirpath() test_dct = str(currpath.join("crossingDict.so")) @@ -24,7 +22,7 @@ # from pypy/module/cpyext/test/test_cpyext.py; modified to accept more external # symbols and called directly instead of import_module -def compile_extension_module(space, modname, symbols, **kwds): +def compile_extension_module(space, modname, **kwds): """ Build an extension module and return the filename of the resulting native code file. @@ -38,19 +36,23 @@ state = space.fromcache(State) api_library = state.api_lib if sys.platform == 'win32': - kwds["libraries"] = [api_library] + kwds["libraries"] = []#[api_library] # '%s' undefined; assuming extern returning int kwds["compile_extra"] = ["/we4013"] + # prevent linking with PythonXX.lib + w_maj, w_min = space.fixedview(space.sys.get('version_info'), 5)[:2] + kwds["link_extra"] = ["/NODEFAULTLIB:Python%d%d.lib" % + (space.int_w(w_maj), space.int_w(w_min))] elif sys.platform == 'darwin': kwds["link_files"] = [str(api_library + '.dylib')] else: kwds["link_files"] = [str(api_library + '.so')] if sys.platform.startswith('linux'): - kwds["compile_extra"]=["-Werror=implicit-function-declaration"] + kwds["compile_extra"]=["-Werror", "-g", "-O0"] + kwds["link_extra"]=["-g"] modname = modname.split('.')[-1] eci = ExternalCompilationInfo( - #export_symbols=['init%s' % (modname,)]+symbols, include_dirs=api.include_dirs, **kwds ) @@ -65,28 +67,30 @@ soname.rename(pydname) return str(pydname) -class AppTestCrossing(AppTestCpythonExtensionBase): - spaceconfig = dict(usemodules=['cppyy', '_rawffi', 'itertools', 'cpyext']) +class AppTestCrossing: + spaceconfig = dict(usemodules=['cppyy', '_rawffi', 'itertools']) def setup_class(cls): - AppTestCpythonExtensionBase.setup_class.im_func(cls) # cppyy specific additions (note that test_dct is loaded late # to allow the generated extension module be loaded first) cls.w_test_dct = cls.space.wrap(test_dct) cls.w_pre_imports = cls.space.appexec([], """(): - import cppyy, cpyext, ctypes""") # prevents leak-checking complaints on ctypes + import ctypes, cppyy""") # prevents leak-checking complaints on ctypes' statics def setup_method(self, func): - AppTestCpythonExtensionBase.setup_method.im_func(self, func) - @unwrap_spec(name=str, init=str, body=str) - def create_cdll(space, name, init, body, w_symbols): + def create_cdll(space, name, init, body): # the following is loosely from test_cpyext.py import_module; it # is copied here to be able to tweak the call to # compile_extension_module and to get a different return result # than in that function code = """ #include + /* fix for cpython 2.7 Python.h if running tests with -A + since pypy compiles with -fvisibility-hidden */ + #undef PyMODINIT_FUNC + #define PyMODINIT_FUNC RPY_EXPORTED void + %(body)s PyMODINIT_FUNC @@ -95,8 +99,7 @@ } """ % dict(name=name, init=init, body=body) kwds = dict(separate_module_sources=[code]) - symbols = [space.str_w(w_item) for w_item in space.fixedview(w_symbols)] - mod = compile_extension_module(space, name, symbols, **kwds) + mod = compile_extension_module(space, name, **kwds) # explicitly load the module as a CDLL rather than as a module from pypy.module.imp.importing import get_so_extension @@ -106,17 +109,6 @@ self.w_create_cdll = self.space.wrap(interp2app(create_cdll)) - def test00_base_class(self): - """Test from cpyext; only here to see whether the imported class works""" - - import sys - init = """ - if (Py_IsInitialized()) - Py_InitModule("foo", NULL); - """ - self.import_module(name='foo', init=init) - assert 'foo' in sys.modules - def test01_build_bar_extension(self): """Test that builds the needed extension; runs as test to keep it loaded""" @@ -131,10 +123,12 @@ # note: only the symbols are needed for C, none for python body = """ + RPY_EXPORTED long bar_unwrap(PyObject* arg) { - return PyLong_AsLong(arg); + return 13;//PyLong_AsLong(arg); } + RPY_EXPORTED PyObject* bar_wrap(long l) { return PyLong_FromLong(l); @@ -146,8 +140,7 @@ # explicitly load the module as a CDLL rather than as a module import ctypes self.cmodule = ctypes.CDLL( - self.create_cdll(name, init, body, ['bar_unwrap', 'bar_wrap']), - ctypes.RTLD_GLOBAL) + self.create_cdll(name, init, body), ctypes.RTLD_GLOBAL) def test02_crossing_dict(self): """Test availability of all needed classes in the dict""" @@ -160,6 +153,7 @@ assert crossing.A == crossing.A + @py.test.mark.dont_track_allocations("fine when running standalone, though?!") def test03_send_pyobject(self): """Test sending a true pyobject to C++""" @@ -169,6 +163,7 @@ a = crossing.A() assert a.unwrap(13) == 13 + @py.test.mark.dont_track_allocations("fine when running standalone, though?!") def test04_send_and_receive_pyobject(self): """Test receiving a true pyobject from C++""" From pypy.commits at gmail.com Wed Jul 13 13:01:01 2016 From: pypy.commits at gmail.com (arigo) Date: Wed, 13 Jul 2016 10:01:01 -0700 (PDT) Subject: [pypy-commit] pypy issue2343: A branch for issue #2343 (PyPy hides TypeError raised by __instancecheck__) Message-ID: <578673cd.a861c20a.2c2cb.ffffb20d@mx.google.com> Author: Armin Rigo Branch: issue2343 Changeset: r85679:e85deda33720 Date: 2016-07-13 19:02 +0200 http://bitbucket.org/pypy/pypy/changeset/e85deda33720/ Log: A branch for issue #2343 (PyPy hides TypeError raised by __instancecheck__) From pypy.commits at gmail.com Wed Jul 13 13:01:29 2016 From: pypy.commits at gmail.com (arigo) Date: Wed, 13 Jul 2016 10:01:29 -0700 (PDT) Subject: [pypy-commit] pypy issue2343: Some failing tests Message-ID: <578673e9.0a43c20a.b7251.ffffb334@mx.google.com> Author: Armin Rigo Branch: issue2343 Changeset: r85680:17e4455c688a Date: 2016-07-13 19:02 +0200 http://bitbucket.org/pypy/pypy/changeset/17e4455c688a/ Log: Some failing tests diff --git a/pypy/module/__builtin__/test/test_abstractinst.py b/pypy/module/__builtin__/test/test_abstractinst.py --- a/pypy/module/__builtin__/test/test_abstractinst.py +++ b/pypy/module/__builtin__/test/test_abstractinst.py @@ -216,3 +216,11 @@ c = C() assert isinstance(c, C) assert not called + + def test_instancecheck_exception_not_eaten(self): + class M(object): + def __instancecheck__(self, obj): + raise TypeError("foobar") + + e = raises(TypeError, isinstance, 42, M()) + assert str(e.value) == "foobar" diff --git a/pypy/objspace/std/test/test_typeobject.py b/pypy/objspace/std/test/test_typeobject.py --- a/pypy/objspace/std/test/test_typeobject.py +++ b/pypy/objspace/std/test/test_typeobject.py @@ -1091,6 +1091,16 @@ C() # the lookup of '__new__' succeeds in 'int', # but the lookup of '__init__' fails + def test_instancecheck(self): + assert int.__instancecheck__(42) is True + assert int.__instancecheck__(42.0) is False + class Foo: + __class__ = int + assert int.__instancecheck__(Foo()) is False + class Bar(object): + __class__ = int + assert int.__instancecheck__(Bar()) is True + class AppTestWithMethodCacheCounter: spaceconfig = {"objspace.std.withmethodcachecounter": True} @@ -1290,5 +1300,3 @@ assert not self.compares_by_identity(X) del X.__eq__ assert self.compares_by_identity(X) - - From pypy.commits at gmail.com Wed Jul 13 13:12:04 2016 From: pypy.commits at gmail.com (arigo) Date: Wed, 13 Jul 2016 10:12:04 -0700 (PDT) Subject: [pypy-commit] pypy issue2343: Some similar tests for issubclass() Message-ID: <57867664.0a43c20a.b7251.ffffb75f@mx.google.com> Author: Armin Rigo Branch: issue2343 Changeset: r85681:50c243508a5c Date: 2016-07-13 19:13 +0200 http://bitbucket.org/pypy/pypy/changeset/50c243508a5c/ Log: Some similar tests for issubclass() diff --git a/pypy/module/__builtin__/test/test_abstractinst.py b/pypy/module/__builtin__/test/test_abstractinst.py --- a/pypy/module/__builtin__/test/test_abstractinst.py +++ b/pypy/module/__builtin__/test/test_abstractinst.py @@ -224,3 +224,18 @@ e = raises(TypeError, isinstance, 42, M()) assert str(e.value) == "foobar" + + def test_issubclass_exception_not_eaten(self): + class M(object): + def __subclasscheck__(self, subcls): + raise TypeError("foobar") + + e = raises(TypeError, issubclass, 42, M()) + assert str(e.value) == "foobar" + + def test_issubclass_no_fallback(self): + class M(object): + def __subclasscheck__(self, subcls): + return False + + assert issubclass(42, ()) is False diff --git a/pypy/objspace/std/test/test_typeobject.py b/pypy/objspace/std/test/test_typeobject.py --- a/pypy/objspace/std/test/test_typeobject.py +++ b/pypy/objspace/std/test/test_typeobject.py @@ -1101,6 +1101,19 @@ __class__ = int assert int.__instancecheck__(Bar()) is True + def test_subclasscheck(self): + assert int.__subclasscheck__(bool) is True + assert int.__subclasscheck__(float) is False + class Foo: + __class__ = int + assert int.__subclasscheck__(Foo) is False + class Bar(object): + __class__ = int + assert int.__subclasscheck__(Bar) is False + class AbstractClass(object): + __bases__ = (int,) + assert int.__subclasscheck__(AbstractClass()) is True + class AppTestWithMethodCacheCounter: spaceconfig = {"objspace.std.withmethodcachecounter": True} From pypy.commits at gmail.com Wed Jul 13 15:05:48 2016 From: pypy.commits at gmail.com (raffael_t) Date: Wed, 13 Jul 2016 12:05:48 -0700 (PDT) Subject: [pypy-commit] pypy py3.5-async: Implement GET_AWAITABLE, SETUP_ASYNC_WITH, BEFORE_ASYNC_WITH, GET_AITER, GET_ANEXT pyopcodes (temporarily without additional error checks), fix compiler test Message-ID: <5786910c.41c71c0a.47f38.cccc@mx.google.com> Author: Raffael Tfirst Branch: py3.5-async Changeset: r85682:e3c740229eb3 Date: 2016-07-13 21:05 +0200 http://bitbucket.org/pypy/pypy/changeset/e3c740229eb3/ Log: Implement GET_AWAITABLE, SETUP_ASYNC_WITH, BEFORE_ASYNC_WITH, GET_AITER, GET_ANEXT pyopcodes (temporarily without additional error checks), fix compiler test diff --git a/pypy/interpreter/pyopcode.py b/pypy/interpreter/pyopcode.py --- a/pypy/interpreter/pyopcode.py +++ b/pypy/interpreter/pyopcode.py @@ -1421,19 +1421,36 @@ self.pushvalue(w_iterator) def GET_AWAITABLE(self, oparg, next_instr): - pass + w_iterable = self.popvalue() + self.pushvalue(w_iterable) - def SETUP_ASYNC_WITH(self, oparg, next_instr): - pass + def SETUP_ASYNC_WITH(self, offsettoend, next_instr): + res = self.popvalue() + block = WithBlock(self.valuestackdepth, + next_instr + offsettoend, self.lastblock) + self.lastblock = block + self.pushvalue(res) def BEFORE_ASYNC_WITH(self, oparg, next_instr): - pass + w_manager = self.peekvalue() + w_enter = self.space.lookup(w_manager, "__aenter__") + w_descr = self.space.lookup(w_manager, "__aexit__") + if w_enter is None or w_descr is None: + raise oefmt(self.space.w_AttributeError, + "'%T' object is not a context manager (no __aenter__/" + "__aexit__ method)", w_manager) + w_exit = self.space.get(w_descr, w_manager) + self.settopvalue(w_exit) + w_result = self.space.get_and_call_function(w_enter, w_manager) + self.pushvalue(w_result) def GET_AITER(self, oparg, next_instr): - pass + w_iterable = self.popvalue() + self.pushvalue(w_iterable) def GET_ANEXT(self, oparg, next_instr): - pass + w_iterable = self.popvalue() + self.pushvalue(w_iterable) ### ____________________________________________________________ ### diff --git a/pypy/interpreter/test/test_compiler.py b/pypy/interpreter/test/test_compiler.py --- a/pypy/interpreter/test/test_compiler.py +++ b/pypy/interpreter/test/test_compiler.py @@ -621,7 +621,7 @@ yield 8 def generator(): yield from generator2() - for i in generator(): return i + return next(generator()) res = f() ''')) code = self.compiler.compile(snippet, '', 'exec', 0) @@ -629,7 +629,7 @@ w_d = space.newdict() space.exec_(code, w_d, w_d) w_res = space.getitem(w_d, space.wrap('res')) - assert space.eq_w(w_res, space.wrap("8")) + assert space.eq_w(w_res, space.wrap(8)) def test_dont_inherit_flag(self): # this test checks that compile() don't inherit the __future__ flags From pypy.commits at gmail.com Wed Jul 13 18:01:39 2016 From: pypy.commits at gmail.com (arigo) Date: Wed, 13 Jul 2016 15:01:39 -0700 (PDT) Subject: [pypy-commit] pypy use-madv-free: Giving up on use-mmap and implementing a different idea, which for Message-ID: <5786ba43.0a43c20a.b7251.14ca@mx.google.com> Author: Armin Rigo Branch: use-madv-free Changeset: r85683:32040348824c Date: 2016-07-13 23:36 +0200 http://bitbucket.org/pypy/pypy/changeset/32040348824c/ Log: Giving up on use-mmap and implementing a different idea, which for unknown reasons seems to give far better results: keep using malloc() for arenas but use MADV_DONTNEED / MADV_FREE just before free()ing them From pypy.commits at gmail.com Wed Jul 13 18:01:41 2016 From: pypy.commits at gmail.com (arigo) Date: Wed, 13 Jul 2016 15:01:41 -0700 (PDT) Subject: [pypy-commit] pypy use-madv-free: madvise_free(), which will call madvise() with either MADV_FREE or Message-ID: <5786ba45.10a81c0a.6f272.cc87@mx.google.com> Author: Armin Rigo Branch: use-madv-free Changeset: r85684:abadb1536a6c Date: 2016-07-14 00:02 +0200 http://bitbucket.org/pypy/pypy/changeset/abadb1536a6c/ Log: madvise_free(), which will call madvise() with either MADV_FREE or MADV_DONTNEED. The first option is better but doesn't exist everywhere (it does exist on OS/X and BSDs and Linux >= 4.5) diff --git a/rpython/rlib/rmmap.py b/rpython/rlib/rmmap.py --- a/rpython/rlib/rmmap.py +++ b/rpython/rlib/rmmap.py @@ -70,7 +70,13 @@ CConfig.MREMAP_MAYMOVE = ( rffi_platform.DefinedConstantInteger("MREMAP_MAYMOVE")) CConfig.has_mremap = rffi_platform.Has('mremap(NULL, 0, 0, 0)') - # a dirty hack, this is probably a macro + CConfig.has_madvise = rffi_platform.Has('madvise(NULL, 0, 0)') + # ^^ both are a dirty hack, this is probably a macro + + CConfig.MADV_DONTNEED = ( + rffi_platform.DefinedConstantInteger('MADV_DONTNEED')) + CConfig.MADV_FREE = ( + rffi_platform.DefinedConstantInteger('MADV_FREE')) elif _MS_WINDOWS: constant_names = ['PAGE_READONLY', 'PAGE_READWRITE', 'PAGE_WRITECOPY', @@ -144,6 +150,7 @@ if _POSIX: has_mremap = cConfig['has_mremap'] + has_madvise = cConfig['has_madvise'] c_mmap, c_mmap_safe = external('mmap', [PTR, size_t, rffi.INT, rffi.INT, rffi.INT, off_t], PTR, macro=True, save_err_on_unsafe=rffi.RFFI_SAVE_ERRNO) @@ -154,6 +161,9 @@ if has_mremap: c_mremap, _ = external('mremap', [PTR, size_t, size_t, rffi.ULONG], PTR) + if has_madvise: + _, c_madvise_safe = external('madvise', [PTR, size_t, rffi.INT], + rffi.INT, _nowrapper=True) # this one is always safe _pagesize = rffi_platform.getintegerfunctionresult('getpagesize', @@ -755,6 +765,39 @@ else: free = c_munmap_safe + if sys.platform.startswith('linux'): + assert has_madvise + assert MADV_DONTNEED is not None + if MADV_FREE is None: + MADV_FREE = 8 # from the kernel sources of Linux >= 4.5 + class CanUseMadvFree: + ok = -1 + can_use_madv_free = CanUseMadvFree() + def madvise_free(addr, map_size): + # We don't know if we are running on a recent enough kernel + # that supports MADV_FREE. Check that at runtime: if the + # first call to madvise(MADV_FREE) fails, we assume it's + # because of EINVAL and we fall back to MADV_DONTNEED. + if can_use_madv_free.ok != 0: + res = c_madvise_safe(rffi.cast(PTR, addr), + rffi.cast(size_t, map_size), + rffi.cast(rffi.INT, MADV_FREE)) + if can_use_madv_free.ok == -1: + can_use_madv_free.ok = (rffi.cast(lltype.Signed, res) == 0) + if can_use_madv_free.ok == 0: + c_madvise_safe(rffi.cast(PTR, addr), + rffi.cast(size_t, map_size), + rffi.cast(rffi.INT, MADV_DONTNEED)) + elif has_madvise and not (MADV_FREE is MADV_DONTNEED is None): + use_flag = MADV_FREE if MADV_FREE is not None else MADV_DONTNEED + def madvise_free(addr, map_size): + c_madvise_safe(rffi.cast(PTR, addr), + rffi.cast(size_t, map_size), + rffi.cast(rffi.INT, use_flag)) + else: + def madvice_free(addr, map_size): + "No madvice() on this platform" + elif _MS_WINDOWS: def mmap(fileno, length, tagname="", access=_ACCESS_DEFAULT, offset=0): # XXX flags is or-ed into access by now. @@ -907,4 +950,9 @@ def free(ptr, map_size): VirtualFree_safe(ptr, 0, MEM_RELEASE) + def madvice_free(addr, map_size): + """XXX find a Windows equivalent? + 'addr' is in the middle of memory obtained with the C malloc()... + """ + # register_external here? diff --git a/rpython/rlib/test/test_rmmap.py b/rpython/rlib/test/test_rmmap.py --- a/rpython/rlib/test/test_rmmap.py +++ b/rpython/rlib/test/test_rmmap.py @@ -5,6 +5,7 @@ from rpython.rlib.rarithmetic import intmask from rpython.rlib import rmmap as mmap from rpython.rlib.rmmap import RTypeError, RValueError, alloc, free +from rpython.rlib.rmmap import madvise_free class TestMMap: def setup_class(cls): @@ -490,6 +491,7 @@ data[i] = chr(i & 0xff) for i in range(0, map_size, 171): assert data[i] == chr(i & 0xff) + madvise_free(data, map_size) free(data, map_size) def test_compile_alloc_free(): From pypy.commits at gmail.com Thu Jul 14 04:07:01 2016 From: pypy.commits at gmail.com (arigo) Date: Thu, 14 Jul 2016 01:07:01 -0700 (PDT) Subject: [pypy-commit] pypy use-madv-free: Call madvise() from minimarkpage and llarena Message-ID: <57874825.11051c0a.d2c13.7632@mx.google.com> Author: Armin Rigo Branch: use-madv-free Changeset: r85685:78c7dacab7ab Date: 2016-07-14 10:08 +0200 http://bitbucket.org/pypy/pypy/changeset/78c7dacab7ab/ Log: Call madvise() from minimarkpage and llarena diff --git a/rpython/memory/gc/minimarkpage.py b/rpython/memory/gc/minimarkpage.py --- a/rpython/memory/gc/minimarkpage.py +++ b/rpython/memory/gc/minimarkpage.py @@ -395,6 +395,7 @@ if arena.nfreepages == arena.totalpages: # # The whole arena is empty. Free it. + llarena.arena_reset(arena.base, self.arena_size, 4) llarena.arena_free(arena.base) lltype.free(arena, flavor='raw', track_allocation=False) # diff --git a/rpython/rlib/rmmap.py b/rpython/rlib/rmmap.py --- a/rpython/rlib/rmmap.py +++ b/rpython/rlib/rmmap.py @@ -949,10 +949,3 @@ def free(ptr, map_size): VirtualFree_safe(ptr, 0, MEM_RELEASE) - - def madvice_free(addr, map_size): - """XXX find a Windows equivalent? - 'addr' is in the middle of memory obtained with the C malloc()... - """ - -# register_external here? diff --git a/rpython/rtyper/lltypesystem/llarena.py b/rpython/rtyper/lltypesystem/llarena.py --- a/rpython/rtyper/lltypesystem/llarena.py +++ b/rpython/rtyper/lltypesystem/llarena.py @@ -52,7 +52,7 @@ del self.objectptrs[offset] del self.objectsizes[offset] obj._free() - if zero and zero != 3: + if zero in (1, 2): initialbyte = "0" else: initialbyte = "#" @@ -335,6 +335,8 @@ * 1: clear, optimized for a very large area of memory * 2: clear, optimized for a small or medium area of memory * 3: fill with garbage + * 4: large area of memory that can benefit from MADV_FREE + (i.e. contains garbage, may be zero-filled or not) """ arena_addr = getfakearenaaddress(arena_addr) arena_addr.arena.reset(zero, arena_addr.offset, size) @@ -410,16 +412,19 @@ self.pagesize = 0 def _cleanup_(self): self.pagesize = 0 + def get(self): + pagesize = self.pagesize + if pagesize == 0: + pagesize = rffi.cast(lltype.Signed, legacy_getpagesize()) + self.pagesize = pagesize + return pagesize + posixpagesize = PosixPageSize() def clear_large_memory_chunk(baseaddr, size): from rpython.rlib import rmmap - pagesize = posixpagesize.pagesize - if pagesize == 0: - pagesize = rffi.cast(lltype.Signed, legacy_getpagesize()) - posixpagesize.pagesize = pagesize - + pagesize = posixpagesize.get() if size > 2 * pagesize: lowbits = rffi.cast(lltype.Signed, baseaddr) & (pagesize - 1) if lowbits: # clear the initial misaligned part, if any @@ -435,6 +440,17 @@ if size > 0: # clear the final misaligned part, if any llmemory.raw_memclear(baseaddr, size) + def madvise_arena_free(baseaddr, size): + from rpython.rlib import rmmap + + pagesize = posixpagesize.get() + baseaddr = rffi.cast(lltype.Signed, baseaddr) + aligned_addr = (baseaddr + pagesize - 1) & (pagesize - 1) + size -= (aligned_addr - baseaddr) + if size >= pagesize: + rmmap.madvise_free(rffi.cast(rmmap.PTR, aligned_addr), + size & (pagesize - 1)) + else: # XXX any better implementation on Windows? # Should use VirtualAlloc() to reserve the range of pages, @@ -443,6 +459,12 @@ # them immediately. clear_large_memory_chunk = llmemory.raw_memclear + def madvise_arena_free(baseaddr, size): + """XXX find a Windows equivalent? + 'baseaddr' is in the middle of memory obtained with the C malloc()... + """ + + if os.name == "posix": from rpython.translator.tool.cbuild import ExternalCompilationInfo _eci = ExternalCompilationInfo(includes=['sys/mman.h']) @@ -509,6 +531,8 @@ clear_large_memory_chunk(arena_addr, size) elif zero == 3: llop.raw_memset(lltype.Void, arena_addr, ord('#'), size) + elif zero == 4: + madvise_arena_free(arena_addr, size) else: llmemory.raw_memclear(arena_addr, size) llimpl_arena_reset._always_inline_ = True From pypy.commits at gmail.com Thu Jul 14 05:10:00 2016 From: pypy.commits at gmail.com (arigo) Date: Thu, 14 Jul 2016 02:10:00 -0700 (PDT) Subject: [pypy-commit] pypy use-madv-free: Test and fix Message-ID: <578756e8.4f8d1c0a.f67b.9583@mx.google.com> Author: Armin Rigo Branch: use-madv-free Changeset: r85686:83477b04d5ed Date: 2016-07-14 11:11 +0200 http://bitbucket.org/pypy/pypy/changeset/83477b04d5ed/ Log: Test and fix diff --git a/rpython/rtyper/lltypesystem/llarena.py b/rpython/rtyper/lltypesystem/llarena.py --- a/rpython/rtyper/lltypesystem/llarena.py +++ b/rpython/rtyper/lltypesystem/llarena.py @@ -445,11 +445,11 @@ pagesize = posixpagesize.get() baseaddr = rffi.cast(lltype.Signed, baseaddr) - aligned_addr = (baseaddr + pagesize - 1) & (pagesize - 1) + aligned_addr = (baseaddr + pagesize - 1) & ~(pagesize - 1) size -= (aligned_addr - baseaddr) if size >= pagesize: rmmap.madvise_free(rffi.cast(rmmap.PTR, aligned_addr), - size & (pagesize - 1)) + size & ~(pagesize - 1)) else: # XXX any better implementation on Windows? diff --git a/rpython/rtyper/lltypesystem/test/test_llarena.py b/rpython/rtyper/lltypesystem/test/test_llarena.py --- a/rpython/rtyper/lltypesystem/test/test_llarena.py +++ b/rpython/rtyper/lltypesystem/test/test_llarena.py @@ -1,6 +1,6 @@ -import py +import py, os -from rpython.rtyper.lltypesystem import lltype, llmemory +from rpython.rtyper.lltypesystem import lltype, llmemory, rffi, llarena from rpython.rtyper.lltypesystem.llarena import (arena_malloc, arena_reset, arena_reserve, arena_free, round_up_for_allocation, ArenaError, arena_new_view, arena_shrink_obj, arena_protect, has_protect) @@ -299,6 +299,29 @@ p.x = 125 assert p.x == 125 +def test_madvise_arena_free(): + from rpython.rlib import rmmap + + if os.name != 'posix': + py.test.skip("posix only") + pagesize = llarena.posixpagesize.get() + prev = rmmap.madvise_free + try: + seen = [] + def my_madvise_free(addr, size): + assert lltype.typeOf(addr) == rmmap.PTR + seen.append((addr, size)) + rmmap.madvise_free = my_madvise_free + llarena.madvise_arena_free( + rffi.cast(llmemory.Address, 123 * pagesize + 1), + pagesize * 7 - 2) + finally: + rmmap.madvise_free = prev + assert len(seen) == 1 + addr, size = seen[0] + assert rffi.cast(lltype.Signed, addr) == 124 * pagesize + assert size == pagesize * 5 + class TestStandalone(test_standalone.StandaloneTests): def test_compiled_arena_protect(self): From pypy.commits at gmail.com Thu Jul 14 06:07:24 2016 From: pypy.commits at gmail.com (arigo) Date: Thu, 14 Jul 2016 03:07:24 -0700 (PDT) Subject: [pypy-commit] pypy use-madv-free: ready to merge Message-ID: <5787645c.06a61c0a.807ab.5ef2@mx.google.com> Author: Armin Rigo Branch: use-madv-free Changeset: r85688:4bd0dcbf94e3 Date: 2016-07-14 12:01 +0200 http://bitbucket.org/pypy/pypy/changeset/4bd0dcbf94e3/ Log: ready to merge From pypy.commits at gmail.com Thu Jul 14 06:07:26 2016 From: pypy.commits at gmail.com (arigo) Date: Thu, 14 Jul 2016 03:07:26 -0700 (PDT) Subject: [pypy-commit] pypy default: hg merge use-madv-free Message-ID: <5787645e.45061c0a.8f3f1.b6ca@mx.google.com> Author: Armin Rigo Branch: Changeset: r85689:95f85a57f5c9 Date: 2016-07-14 12:04 +0200 http://bitbucket.org/pypy/pypy/changeset/95f85a57f5c9/ Log: hg merge use-madv-free Use madvise(MADV_FREE), or if that doesn't exist MADV_DONTNEED, on freed arenas. Fixes issue #2336. diff --git a/rpython/memory/gc/minimarkpage.py b/rpython/memory/gc/minimarkpage.py --- a/rpython/memory/gc/minimarkpage.py +++ b/rpython/memory/gc/minimarkpage.py @@ -395,6 +395,7 @@ if arena.nfreepages == arena.totalpages: # # The whole arena is empty. Free it. + llarena.arena_reset(arena.base, self.arena_size, 4) llarena.arena_free(arena.base) lltype.free(arena, flavor='raw', track_allocation=False) # diff --git a/rpython/rlib/rmmap.py b/rpython/rlib/rmmap.py --- a/rpython/rlib/rmmap.py +++ b/rpython/rlib/rmmap.py @@ -70,7 +70,13 @@ CConfig.MREMAP_MAYMOVE = ( rffi_platform.DefinedConstantInteger("MREMAP_MAYMOVE")) CConfig.has_mremap = rffi_platform.Has('mremap(NULL, 0, 0, 0)') - # a dirty hack, this is probably a macro + CConfig.has_madvise = rffi_platform.Has('madvise(NULL, 0, 0)') + # ^^ both are a dirty hack, this is probably a macro + + CConfig.MADV_DONTNEED = ( + rffi_platform.DefinedConstantInteger('MADV_DONTNEED')) + CConfig.MADV_FREE = ( + rffi_platform.DefinedConstantInteger('MADV_FREE')) elif _MS_WINDOWS: constant_names = ['PAGE_READONLY', 'PAGE_READWRITE', 'PAGE_WRITECOPY', @@ -144,6 +150,7 @@ if _POSIX: has_mremap = cConfig['has_mremap'] + has_madvise = cConfig['has_madvise'] c_mmap, c_mmap_safe = external('mmap', [PTR, size_t, rffi.INT, rffi.INT, rffi.INT, off_t], PTR, macro=True, save_err_on_unsafe=rffi.RFFI_SAVE_ERRNO) @@ -154,6 +161,9 @@ if has_mremap: c_mremap, _ = external('mremap', [PTR, size_t, size_t, rffi.ULONG], PTR) + if has_madvise: + _, c_madvise_safe = external('madvise', [PTR, size_t, rffi.INT], + rffi.INT, _nowrapper=True) # this one is always safe _pagesize = rffi_platform.getintegerfunctionresult('getpagesize', @@ -755,6 +765,39 @@ else: free = c_munmap_safe + if sys.platform.startswith('linux'): + assert has_madvise + assert MADV_DONTNEED is not None + if MADV_FREE is None: + MADV_FREE = 8 # from the kernel sources of Linux >= 4.5 + class CanUseMadvFree: + ok = -1 + can_use_madv_free = CanUseMadvFree() + def madvise_free(addr, map_size): + # We don't know if we are running on a recent enough kernel + # that supports MADV_FREE. Check that at runtime: if the + # first call to madvise(MADV_FREE) fails, we assume it's + # because of EINVAL and we fall back to MADV_DONTNEED. + if can_use_madv_free.ok != 0: + res = c_madvise_safe(rffi.cast(PTR, addr), + rffi.cast(size_t, map_size), + rffi.cast(rffi.INT, MADV_FREE)) + if can_use_madv_free.ok == -1: + can_use_madv_free.ok = (rffi.cast(lltype.Signed, res) == 0) + if can_use_madv_free.ok == 0: + c_madvise_safe(rffi.cast(PTR, addr), + rffi.cast(size_t, map_size), + rffi.cast(rffi.INT, MADV_DONTNEED)) + elif has_madvise and not (MADV_FREE is MADV_DONTNEED is None): + use_flag = MADV_FREE if MADV_FREE is not None else MADV_DONTNEED + def madvise_free(addr, map_size): + c_madvise_safe(rffi.cast(PTR, addr), + rffi.cast(size_t, map_size), + rffi.cast(rffi.INT, use_flag)) + else: + def madvice_free(addr, map_size): + "No madvice() on this platform" + elif _MS_WINDOWS: def mmap(fileno, length, tagname="", access=_ACCESS_DEFAULT, offset=0): # XXX flags is or-ed into access by now. @@ -906,5 +949,3 @@ def free(ptr, map_size): VirtualFree_safe(ptr, 0, MEM_RELEASE) - -# register_external here? diff --git a/rpython/rlib/test/test_rmmap.py b/rpython/rlib/test/test_rmmap.py --- a/rpython/rlib/test/test_rmmap.py +++ b/rpython/rlib/test/test_rmmap.py @@ -5,6 +5,12 @@ from rpython.rlib.rarithmetic import intmask from rpython.rlib import rmmap as mmap from rpython.rlib.rmmap import RTypeError, RValueError, alloc, free +try: + from rpython.rlib.rmmap import madvise_free +except ImportError: + def madvise_free(addr, size): + "Not available" + class TestMMap: def setup_class(cls): @@ -490,6 +496,7 @@ data[i] = chr(i & 0xff) for i in range(0, map_size, 171): assert data[i] == chr(i & 0xff) + madvise_free(data, map_size) free(data, map_size) def test_compile_alloc_free(): diff --git a/rpython/rtyper/lltypesystem/llarena.py b/rpython/rtyper/lltypesystem/llarena.py --- a/rpython/rtyper/lltypesystem/llarena.py +++ b/rpython/rtyper/lltypesystem/llarena.py @@ -52,7 +52,7 @@ del self.objectptrs[offset] del self.objectsizes[offset] obj._free() - if zero and zero != 3: + if zero in (1, 2): initialbyte = "0" else: initialbyte = "#" @@ -335,6 +335,8 @@ * 1: clear, optimized for a very large area of memory * 2: clear, optimized for a small or medium area of memory * 3: fill with garbage + * 4: large area of memory that can benefit from MADV_FREE + (i.e. contains garbage, may be zero-filled or not) """ arena_addr = getfakearenaaddress(arena_addr) arena_addr.arena.reset(zero, arena_addr.offset, size) @@ -410,16 +412,19 @@ self.pagesize = 0 def _cleanup_(self): self.pagesize = 0 + def get(self): + pagesize = self.pagesize + if pagesize == 0: + pagesize = rffi.cast(lltype.Signed, legacy_getpagesize()) + self.pagesize = pagesize + return pagesize + posixpagesize = PosixPageSize() def clear_large_memory_chunk(baseaddr, size): from rpython.rlib import rmmap - pagesize = posixpagesize.pagesize - if pagesize == 0: - pagesize = rffi.cast(lltype.Signed, legacy_getpagesize()) - posixpagesize.pagesize = pagesize - + pagesize = posixpagesize.get() if size > 2 * pagesize: lowbits = rffi.cast(lltype.Signed, baseaddr) & (pagesize - 1) if lowbits: # clear the initial misaligned part, if any @@ -435,6 +440,17 @@ if size > 0: # clear the final misaligned part, if any llmemory.raw_memclear(baseaddr, size) + def madvise_arena_free(baseaddr, size): + from rpython.rlib import rmmap + + pagesize = posixpagesize.get() + baseaddr = rffi.cast(lltype.Signed, baseaddr) + aligned_addr = (baseaddr + pagesize - 1) & ~(pagesize - 1) + size -= (aligned_addr - baseaddr) + if size >= pagesize: + rmmap.madvise_free(rffi.cast(rmmap.PTR, aligned_addr), + size & ~(pagesize - 1)) + else: # XXX any better implementation on Windows? # Should use VirtualAlloc() to reserve the range of pages, @@ -443,6 +459,12 @@ # them immediately. clear_large_memory_chunk = llmemory.raw_memclear + def madvise_arena_free(baseaddr, size): + """XXX find a Windows equivalent? + 'baseaddr' is in the middle of memory obtained with the C malloc()... + """ + + if os.name == "posix": from rpython.translator.tool.cbuild import ExternalCompilationInfo _eci = ExternalCompilationInfo(includes=['sys/mman.h']) @@ -509,6 +531,8 @@ clear_large_memory_chunk(arena_addr, size) elif zero == 3: llop.raw_memset(lltype.Void, arena_addr, ord('#'), size) + elif zero == 4: + madvise_arena_free(arena_addr, size) else: llmemory.raw_memclear(arena_addr, size) llimpl_arena_reset._always_inline_ = True diff --git a/rpython/rtyper/lltypesystem/test/test_llarena.py b/rpython/rtyper/lltypesystem/test/test_llarena.py --- a/rpython/rtyper/lltypesystem/test/test_llarena.py +++ b/rpython/rtyper/lltypesystem/test/test_llarena.py @@ -1,6 +1,6 @@ -import py +import py, os -from rpython.rtyper.lltypesystem import lltype, llmemory +from rpython.rtyper.lltypesystem import lltype, llmemory, rffi, llarena from rpython.rtyper.lltypesystem.llarena import (arena_malloc, arena_reset, arena_reserve, arena_free, round_up_for_allocation, ArenaError, arena_new_view, arena_shrink_obj, arena_protect, has_protect) @@ -299,6 +299,29 @@ p.x = 125 assert p.x == 125 +def test_madvise_arena_free(): + from rpython.rlib import rmmap + + if os.name != 'posix': + py.test.skip("posix only") + pagesize = llarena.posixpagesize.get() + prev = rmmap.madvise_free + try: + seen = [] + def my_madvise_free(addr, size): + assert lltype.typeOf(addr) == rmmap.PTR + seen.append((addr, size)) + rmmap.madvise_free = my_madvise_free + llarena.madvise_arena_free( + rffi.cast(llmemory.Address, 123 * pagesize + 1), + pagesize * 7 - 2) + finally: + rmmap.madvise_free = prev + assert len(seen) == 1 + addr, size = seen[0] + assert rffi.cast(lltype.Signed, addr) == 124 * pagesize + assert size == pagesize * 5 + class TestStandalone(test_standalone.StandaloneTests): def test_compiled_arena_protect(self): From pypy.commits at gmail.com Thu Jul 14 06:07:22 2016 From: pypy.commits at gmail.com (arigo) Date: Thu, 14 Jul 2016 03:07:22 -0700 (PDT) Subject: [pypy-commit] pypy use-madv-free: Windows test fix Message-ID: <5787645a.0dc11c0a.97ced.b297@mx.google.com> Author: Armin Rigo Branch: use-madv-free Changeset: r85687:10d990c7213f Date: 2016-07-14 12:00 +0200 http://bitbucket.org/pypy/pypy/changeset/10d990c7213f/ Log: Windows test fix diff --git a/rpython/rlib/test/test_rmmap.py b/rpython/rlib/test/test_rmmap.py --- a/rpython/rlib/test/test_rmmap.py +++ b/rpython/rlib/test/test_rmmap.py @@ -5,7 +5,12 @@ from rpython.rlib.rarithmetic import intmask from rpython.rlib import rmmap as mmap from rpython.rlib.rmmap import RTypeError, RValueError, alloc, free -from rpython.rlib.rmmap import madvise_free +try: + from rpython.rlib.rmmap import madvise_free +except ImportError: + def madvise_free(addr, size): + "Not available" + class TestMMap: def setup_class(cls): From pypy.commits at gmail.com Thu Jul 14 06:07:28 2016 From: pypy.commits at gmail.com (arigo) Date: Thu, 14 Jul 2016 03:07:28 -0700 (PDT) Subject: [pypy-commit] pypy default: Document branch Message-ID: <57876460.531d1c0a.8d005.6099@mx.google.com> Author: Armin Rigo Branch: Changeset: r85690:66bb0cd7b0eb Date: 2016-07-14 12:08 +0200 http://bitbucket.org/pypy/pypy/changeset/66bb0cd7b0eb/ Log: Document branch diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -64,3 +64,11 @@ Avoid exhausting the stack in the JIT due to successive guard failures in the same Python function ending up as successive levels of RPython functions, while at app-level the traceback is very short + +.. branch: use-madv-free + +Try harder to memory to the OS. See e.g. issue #2336. Note that it does +not show up as a reduction of the VIRT column in ``top``, and the RES +column might also not show the reduction, particularly on Linux >= 4.5 or +on OS/X: it uses MADV_FREE, which only marks the pages as returnable to +the OS if the memory is low. From pypy.commits at gmail.com Thu Jul 14 06:35:36 2016 From: pypy.commits at gmail.com (arigo) Date: Thu, 14 Jul 2016 03:35:36 -0700 (PDT) Subject: [pypy-commit] pypy.org extradoc: Document the memory usage behavior of PyPy, because it is not very clear Message-ID: <57876af8.d11b1c0a.7c48b.c926@mx.google.com> Author: Armin Rigo Branch: extradoc Changeset: r766:c4d464e272a9 Date: 2016-07-14 12:37 +0200 http://bitbucket.org/pypy/pypy.org/changeset/c4d464e272a9/ Log: Document the memory usage behavior of PyPy, because it is not very clear that memory usage *does* go down in systems with MADV_FREE. diff --git a/compat.html b/compat.html --- a/compat.html +++ b/compat.html @@ -124,6 +124,15 @@ as on CPython: they run “some time later” in PyPy (or not at all if the program finishes running in the meantime). See     more details here.

    +

    Note that PyPy returns unused memory to the operating system if there +is a madvise() system call (at least Linux, OS/X, BSD). It is +important to realize that you may not see this in top. The unused +pages are marked with MADV_FREE, which tells the system “if you +need more memory at some point, grab this page”. As long as memory is +plentiful, the RES column in top remains high. (Exceptions to +this rule are systems with no MADV_FREE, where we use +MADV_DONTNEED, which forcefully lowers the RES. This includes +Linux <= 4.4.)

    A more complete list of known differences is available at our dev site.